alx/mu4e
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mu-server: use strings, not sexps object (optimization)

Browse Source 
When passing messages to mu, often we got a (parsed from string)
message-sexp from the message document; then appended some more
properties ("build_message_sexp").

Instead, we can do it in terms of the strings; this is _a little_
inelegant, but also much faster; compare:

(base)
[mu4e] Found 500 matching messages; 0 hidden; search: 1298.0 ms (2.60 ms/msg); render: 642.1 ms (1.28 ms/msg)

(with temp-file optimization (earlier commit)
[mu4e] Found 500 matching messages; 0 hidden; search: 1152.7 ms (2.31 ms/msg); render: 270.1 ms (0.54 ms/msg)

(with temp file optimize _and_ the string opt (this commit)
[mu4e] Found 500 matching messages; 0 hidden; search: 266.0 ms (0.53 ms/msg); render: 199.7 ms (0.40 ms/msg)
This commit is contained in:
Dirk-Jan C. Binnema
2023-07-31 23:53:29 +03:00
parent 1018f0f0a1
commit aea95b5be0
4 changed files with 137 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
lib/message/mu-document.hh
View File

@ -144,13 +144,19 @@ public:
void remove(Field::Id field_id); void remove(Field::Id field_id);
/** /**
* Get the cached s-expression useful for changing * Get the cached s-expression
* it (call update_sexp_cache() when done)
* *
* @return the cached s-expression * @return the cached s-expression
*/ */
const Sexp& sexp() const { return cached_sexp(); } const Sexp& sexp() const { return cached_sexp(); }
/**
* Get the message s-expression as a string
*
* @return message s-expression string
*/
std::string sexp_str() const { return xdoc_.get_data(); }
/** /**
* Generically adds an optional value, if set, to the document * Generically adds an optional value, if set, to the document
* *

213
lib/mu-server.cc
View File

@ -84,10 +84,12 @@ struct Server::Private {
// //
// output // output
// void output(const std::string& str, Server::OutputFlags flags = {}) const {
void output_sexp(const Sexp& sexp, Server::OutputFlags flags = {}) const {
if (output_) if (output_)
output_(sexp, flags); output_(str, flags);
}
void output_sexp(const Sexp& sexp, Server::OutputFlags flags = {}) const {
output(sexp.to_string(), flags);
} }
size_t output_results(const QueryResults& qres, size_t batch_size) const; size_t output_results(const QueryResults& qres, size_t batch_size) const;
@ -112,11 +114,6 @@ struct Server::Private {
void view_handler(const Command& cmd); void view_handler(const Command& cmd);
private: private:
// helpers
Sexp build_message_sexp(const Message& msg,
Store::Id docid,
const Option<QueryMatch&> qm) const;
void move_docid(Store::Id docid, Option<std::string> flagstr, void move_docid(Store::Id docid, Option<std::string> flagstr,
bool new_name, bool no_view); bool new_name, bool no_view);
@ -140,54 +137,66 @@ private:
std::string tmp_dir_; std::string tmp_dir_;
}; };
static Sexp static void
build_metadata(const QueryMatch& qmatch) append_metadata(std::string& str, const QueryMatch& qmatch)
{ {
const auto td{::atoi(qmatch.thread_date.c_str())}; const auto td{::atoi(qmatch.thread_date.c_str())};
auto mdata = Sexp().put_props(":path", qmatch.thread_path,
":level", qmatch.thread_level,
":date", qmatch.thread_date,
":data-tstamp", Sexp().add(static_cast<unsigned>(td >> 16),
static_cast<unsigned>(td & 0xffff),
0));
if (qmatch.has_flag(QueryMatch::Flags::Root))
mdata.put_props(":root", Sexp::t_sym);
if (qmatch.has_flag(QueryMatch::Flags::Related))
mdata.put_props(":related", Sexp::t_sym);
if (qmatch.has_flag(QueryMatch::Flags::First))
mdata.put_props(":first-child", Sexp::t_sym);
if (qmatch.has_flag(QueryMatch::Flags::Last))
mdata.put_props(":last-child", Sexp::t_sym);
if (qmatch.has_flag(QueryMatch::Flags::Orphan))
mdata.put_props(":orphan", Sexp::t_sym);
if (qmatch.has_flag(QueryMatch::Flags::Duplicate))
mdata.put_props(":duplicate", Sexp::t_sym);
if (qmatch.has_flag(QueryMatch::Flags::HasChild))
mdata.put_props(":has-child", Sexp::t_sym);
if (qmatch.has_flag(QueryMatch::Flags::ThreadSubject))
mdata.put_props(":thread-subject", Sexp::t_sym);
return mdata; str += mu_format(" :meta (:path \"{}\" :level {} :date \"{}\" "
":data-tstamp ({} {} 0)",
qmatch.thread_path,
qmatch.thread_level,
qmatch.thread_date,
static_cast<unsigned>(td >> 16),
static_cast<unsigned>(td & 0xffff));
if (qmatch.has_flag(QueryMatch::Flags::Root))
str += " :root t";
if (qmatch.has_flag(QueryMatch::Flags::Related))
str += " :related t";
if (qmatch.has_flag(QueryMatch::Flags::First))
str += " :first-child t";
if (qmatch.has_flag(QueryMatch::Flags::Last))
str += " :last-child t";
if (qmatch.has_flag(QueryMatch::Flags::Orphan))
str += " :orphan t";
if (qmatch.has_flag(QueryMatch::Flags::Duplicate))
str += " :duplicate t";
if (qmatch.has_flag(QueryMatch::Flags::HasChild))
str += " :has-child t";
if (qmatch.has_flag(QueryMatch::Flags::ThreadSubject))
str += " :thread-subject t";
str += ')';
} }
/* /*
* A message here consists of a message s-expression with optionally a :docid * A message here consists of a message s-expression with optionally a :docid
* and/or :meta expression added. * and/or :meta expression added.
*
* Note, for speed reasons, we build is as a _string_, without any further
* parsing.
*/ */
Sexp static std::string
Server::Private::build_message_sexp(const Message& msg, msg_sexp_str(const Message& msg, Store::Id docid, const Option<QueryMatch&> qm)
Store::Id docid,
const Option<QueryMatch&> qm) const
{ {
Sexp sexp{msg.sexp()}; // copy auto sexpstr{msg.document().sexp_str()};
if (docid != 0) sexpstr.reserve(sexpstr.size () + (docid == 0 ? 0 : 16) + (qm ? 64 : 0));
sexp.put_props(":docid", docid);
if (qm)
sexp.put_props(":meta", build_metadata(*qm));
return sexp; // remove the closing ( ... )
sexpstr.erase(sexpstr.end() - 1);
if (docid != 0)
sexpstr += " :docid " + to_string(docid);
if (qm)
append_metadata(sexpstr, *qm);
sexpstr += ')'; // ... end close it again.
return sexpstr;
} }
CommandHandler::CommandInfoMap CommandHandler::CommandInfoMap
Server::Private::make_command_map() Server::Private::make_command_map()
{ {
@ -413,8 +422,8 @@ Server::Private::add_handler(const Command& cmd)
throw Error(Error::Code::Store, throw Error(Error::Code::Store,
"failed to get message at {} (docid={})", *path, docid); "failed to get message at {} (docid={})", *path, docid);
output_sexp(Sexp().put_props(":update", output(mu_format("(:update {})",
build_message_sexp(msg_res.value(), docid, {}))); msg_sexp_str(msg_res.value(), docid, {})));
} }
/* 'compose' produces the un-changed *original* message sexp (ie., the message /* 'compose' produces the un-changed *original* message sexp (ie., the message
@ -423,8 +432,8 @@ Server::Private::add_handler(const Command& cmd)
* edit/resend), and 'docid' for the message to reply to. Note, type:new does * edit/resend), and 'docid' for the message to reply to. Note, type:new does
* not have an original message, and therefore does not need a docid * not have an original message, and therefore does not need a docid
* *
* In returns a (:compose <type> [:original <original-msg>] [:include] ) * In returns a (:compose <type> [:original <original-msg>] [:include] ) message
* message (detals: see code below) * (details: see code below)
* *
* Note ':include' t or nil determines whether to include attachments * Note ':include' t or nil determines whether to include attachments
*/ */
@ -476,8 +485,8 @@ Server::Private::compose_handler(const Command& cmd)
if (!msg) if (!msg)
throw Error{Error::Code::Store, "failed to get message {}", docid}; throw Error{Error::Code::Store, "failed to get message {}", docid};
comp_lst.put_props(":original", build_message_sexp(msg.value(), docid, {})); auto msg_sexp = unwrap(Sexp::parse(msg_sexp_str(msg.value(), docid, {})));
comp_lst.put_props(":original", msg_sexp);
if (ctype == "forward") { if (ctype == "forward") {
// when forwarding, attach any attachment in the orig // when forwarding, attach any attachment in the orig
size_t index{}; size_t index{};
@ -510,8 +519,8 @@ Server::Private::make_temp_file_stream() const
if (!output.good()) if (!output.good())
throw Mu::Error{Error::Code::File, "failed to create temp-file"}; throw Mu::Error{Error::Code::File, "failed to create temp-file"};
return make_pair<std::ofstream, std::string>(std::move(output), return std::make_pair<std::ofstream, std::string>(std::move(output),
std::move(tmp_eld)); std::move(tmp_eld));
} }
@ -531,37 +540,48 @@ Server::Private::contacts_handler(const Command& cmd)
mu_debug("find {} contacts last seen >= {} (tstamp: {})", mu_debug("find {} contacts last seen >= {} (tstamp: {})",
personal ? "personal" : "any", time_to_string("%c", after), tstamp); personal ? "personal" : "any", time_to_string("%c", after), tstamp);
auto n{0}; auto match_contact = [&](const Contact& ci)->bool {
Sexp contacts;
store().contacts_cache().for_each([&](const Contact& ci) {
/* since the last time we got some contacts */
if (tstamp > ci.tstamp) if (tstamp > ci.tstamp)
return true; return false; /* already seen? */
/* (maybe) only include 'personal' contacts */ else if (personal && !ci.personal)
if (personal && !ci.personal) return false; /* not personal? */
return true; else if (after > ci.message_date)
/* only include newer-than-x contacts */ return true; /* too old? */
if (after > ci.message_date) else
return true; return false;
n++; };
contacts.add(ci.display_name());
return maxnum == 0 || n < maxnum;
});
/* dump the contacts cache as a giant sexp */ auto n{0};
mu_debug("sending {} of {} contact(s)", n, store().contacts_cache().size());
if (options_.allow_temp_file) { if (options_.allow_temp_file) {
auto&& [output, tmp_eld] = make_temp_file_stream(); // fast way: put all the contacts in a temp-file
output << contacts; auto&& [tmp_file, tmp_file_name] = make_temp_file_stream();
output_sexp(Sexp{":tstamp"_sym, mu_format("{}", g_get_monotonic_time()), tmp_file << '(';
":contacts-temp-file"_sym, tmp_eld}); store().contacts_cache().for_each([&](const Contact& ci) {
if (!match_contact(ci))
return true; // continue
tmp_file << quote(ci.display_name()) << "\n";
++n;
return maxnum == 0 || n < maxnum;
});
tmp_file << ')';
output_sexp(Sexp{":tstamp"_sym, mu_format("\"{}\"", g_get_monotonic_time()),
":contacts-temp-file"_sym, tmp_file_name});
} else { } else {
Sexp contacts;
store().contacts_cache().for_each([&](const Contact& ci) {
if (!match_contact(ci))
return true; // continue;
contacts.add(ci.display_name());
++n;
return maxnum == 0 || n < maxnum;
});
Sexp seq; Sexp seq;
seq.put_props(":contacts", contacts, seq.put_props(":contacts", std::move(contacts),
":tstamp", mu_format("{}", g_get_monotonic_time())); ":tstamp", mu_format("\"{}\"", g_get_monotonic_time()));
output_sexp(seq, Server::OutputFlags::SplitList); output_sexp(seq);
} }
mu_debug("sent {} of {} contact(s)", n, store().contacts_cache().size());
} }
/* /*
@ -601,7 +621,7 @@ determine_docids(const Store& store, const Command& cmd)
size_t size_t
Server::Private::output_results(const QueryResults& qres, size_t batch_size) const Server::Private::output_results(const QueryResults& qres, size_t batch_size) const
{ {
size_t n{}; size_t n{};
Sexp headers; Sexp headers;
const auto output_batch = [&](Sexp&& hdrs) { const auto output_batch = [&](Sexp&& hdrs) {
@ -617,7 +637,7 @@ Server::Private::output_results(const QueryResults& qres, size_t batch_size) con
++n; ++n;
// construct sexp for a single header. // construct sexp for a single header.
auto qm{mi.query_match()}; auto qm{mi.query_match()};
auto msgsexp{build_message_sexp(*msg, mi.doc_id(), qm)}; auto msgsexp{unwrap(Sexp::parse(msg_sexp_str(*msg, mi.doc_id(), qm)))};
headers.add(std::move(msgsexp)); headers.add(std::move(msgsexp));
// we output up-to-batch-size lists of messages. It's much // we output up-to-batch-size lists of messages. It's much
// faster (on the emacs side) to handle such batches than single // faster (on the emacs side) to handle such batches than single
@ -641,8 +661,8 @@ Server::Private::output_results_temp_file(const QueryResults& qres, size_t batch
// create an output stream with a file name // create an output stream with a file name
size_t n{}; size_t n{};
auto&& [output, tmp_eld] = make_temp_file_stream(); auto&& [tmp_file, tmp_file_name] = make_temp_file_stream();
output << '('; tmp_file << '(';
for(auto&& mi: qres) { for(auto&& mi: qres) {
auto msg{mi.message()}; auto msg{mi.message()};
@ -650,20 +670,21 @@ Server::Private::output_results_temp_file(const QueryResults& qres, size_t batch
continue; continue;
auto qm{mi.query_match()}; // construct sexp for a single header. auto qm{mi.query_match()}; // construct sexp for a single header.
output << build_message_sexp(*msg, mi.doc_id(), qm); tmp_file << msg_sexp_str(*msg, mi.doc_id(), qm) << '\n';
++n; ++n;
if (n % batch_size == 0) { if (n % batch_size == 0) {
output << ')'; tmp_file << ')';
output_sexp(Sexp{":headers-temp-file"_sym, tmp_eld}); batch_size = 1000;
output_sexp(Sexp{":headers-temp-file"_sym, tmp_file_name});
auto new_stream{make_temp_file_stream()}; auto new_stream{make_temp_file_stream()};
output = std::move(new_stream.first); tmp_file = std::move(new_stream.first);
tmp_eld = std::move(new_stream.second); tmp_file_name = std::move(new_stream.second);
output << '('; tmp_file << '(';
} }
} }
output << ')'; tmp_file << ')';
output_sexp(Sexp{":headers-temp-file"_sym, tmp_eld}); output_sexp(Sexp{":headers-temp-file"_sym, tmp_file_name});
return n; return n;
} }
@ -855,18 +876,18 @@ Server::Private::perform_move(Store::Id docid,
throw idmsgvec.error(); throw idmsgvec.error();
for (auto&&[id, msg]: *idmsgvec) { for (auto&&[id, msg]: *idmsgvec) {
Sexp sexp{":update"_sym, build_message_sexp(idmsgvec->at(0).second, id, {})}; auto sexpstr = "(:update " + msg_sexp_str(idmsgvec->at(0).second, id, {});
/* note, the :move t thing is a hint to the frontend that it /* note, the :move t thing is a hint to the frontend that it
* could remove the particular header */ * could remove the particular header */
if (different_mdir) if (different_mdir)
sexp.put_props(":move", Sexp::t_sym); sexpstr += " :move t";
if (!no_view && id == docid) if (!no_view && id == docid)
sexp.put_props(":maybe-view", Sexp::t_sym); sexpstr += " :maybe-view t";
output_sexp(std::move(sexp)); sexpstr += ')';
output(std::move(sexpstr));
} }
} }
static Flags static Flags
calculate_message_flags(const Message& msg, Option<std::string> flagopt) calculate_message_flags(const Message& msg, Option<std::string> flagopt)
{ {
@ -1042,8 +1063,8 @@ Server::Private::view_mark_as_read(Store::Id docid, Message&& msg, bool rename)
throw move_res.error(); throw move_res.error();
for (auto&& [id, msg]: move_res.value()) for (auto&& [id, msg]: move_res.value())
output_sexp(Sexp{id == docid ? ":view"_sym : ":update"_sym, output(mu_format("({} {})", id == docid ? ":view" : ":update",
build_message_sexp(msg, id, {})}); msg_sexp_str(msg, id, {})));
} }
void void
@ -1067,7 +1088,7 @@ Server::Private::view_handler(const Command& cmd)
/* if the message should not be marked-as-read, we're done. */ /* if the message should not be marked-as-read, we're done. */
if (!mark_as_read) if (!mark_as_read)
output_sexp(Sexp().put_props(":view", build_message_sexp(msg, docid, {}))); output(mu_format("(:view {})", msg_sexp_str(msg, docid, {})));
else else
view_mark_as_read(docid, std::move(msg), rename); view_mark_as_read(docid, std::move(msg), rename);
/* otherwise, mark message and and possible dups as read */ /* otherwise, mark message and and possible dups as read */

9
lib/mu-server.hh
View File

@ -36,19 +36,16 @@ class Server {
public: public:
enum struct OutputFlags { enum struct OutputFlags {
None = 0, None = 0,
SplitList = 1 << 0, Flush = 1 << 0, /**< flush output buffer after */
/**< insert newlines between list items */
Flush = 1 << 1,
/**< flush output buffer after */
}; };
/** /**
* Prototype for output function * Prototype for output function
* *
* @param sexp an s-expression * @param str a string
* @param flags flags that influence the behavior * @param flags flags that influence the behavior
*/ */
using Output = std::function<void(const Sexp& sexp, OutputFlags flags)>; using Output = std::function<void(const std::string& str, OutputFlags flags)>;
struct Options { struct Options {
bool allow_temp_file; /**< temp file optimization allowed? */ bool allow_temp_file; /**< temp file optimization allowed? */

23
mu/mu-cmd-server.cc
View File

@ -80,32 +80,27 @@ cookie(size_t n)
::printf(COOKIE_PRE "%x" COOKIE_POST, num); ::printf(COOKIE_PRE "%x" COOKIE_POST, num);
} }
static void
output_sexp_stdout(const Sexp& sexp, Server::OutputFlags flags)
{
/* if requested, insert \n between list elements; note:
* is _not_ inherited by children */
Sexp::Format fopts{};
if (any_of(flags & Server::OutputFlags::SplitList))
fopts |= Sexp::Format::SplitList;
const auto str{sexp.to_string(fopts)};
static void
output_stdout(const std::string& str, Server::OutputFlags flags)
{
cookie(str.size() + 1); cookie(str.size() + 1);
if (G_UNLIKELY(::puts(str.c_str()) < 0)) { if (G_UNLIKELY(::puts(str.c_str()) < 0)) {
mu_critical("failed to write output '{}'", str); mu_critical("failed to write output '{}'", str);
::raise(SIGTERM); /* terminate ourselves */ ::raise(SIGTERM); /* terminate ourselves */
} }
if (any_of(flags & Server::OutputFlags::Flush)) if (any_of(flags & Server::OutputFlags::Flush))
std::fflush(stdout); std::fflush(stdout);
} }
static void static void
report_error(const Mu::Error& err) noexcept report_error(const Mu::Error& err) noexcept
{ {
output_sexp_stdout(Sexp(":error"_sym, Error::error_number(err.code()), output_stdout(Sexp(":error"_sym, Error::error_number(err.code()),
":message"_sym, err.what()), ":message"_sym, err.what()).to_string(),
Server::OutputFlags::Flush); Server::OutputFlags::Flush);
} }
@ -120,7 +115,7 @@ Mu::mu_cmd_server(const Mu::Options& opts) try {
Server::Options sopts{}; Server::Options sopts{};
sopts.allow_temp_file = opts.server.allow_temp_file; sopts.allow_temp_file = opts.server.allow_temp_file;
Server server{*store, sopts, output_sexp_stdout}; Server server{*store, sopts, output_stdout};
mu_message("created server with store @ {}; maildir @ {}; debug-mode {};" mu_message("created server with store @ {}; maildir @ {}; debug-mode {};"
"readline: {}", "readline: {}",
store->path(), store->root_maildir(), store->path(), store->root_maildir(),