diff --git a/lib/message/mu-document.cc b/lib/message/mu-document.cc index debf6372..6d090fc4 100644 --- a/lib/message/mu-document.cc +++ b/lib/message/mu-document.cc @@ -1,5 +1,5 @@ /* -** Copyright (C) 2022-2025 Dirk-Jan C. Binnema +** Copyright (C) 2022-2026 Dirk-Jan C. Binnema ** ** This program is free software; you can redistribute it and/or modify it ** under the terms of the GNU General Public License as published by the @@ -87,20 +87,37 @@ make_term_generator(Xapian::Document& doc, Document::Options opts) return termgen; } -static void -add_search_term(Xapian::Document& doc, const Field& field, const std::string& val, - Document::Options opts) +/* build a term from an already-flattened value; equivalent to + * Field::xapian_term(), minus the second flatten-pass. */ +static std::string +prefixed_term(const Field& field, const std::string& flat) +{ + std::string term(1U, field.xapian_prefix()); + term += flat; + if (G_UNLIKELY(term.size() > MaxTermLength)) + term.erase(MaxTermLength); + + return term; +} + +void +Document::add_search_term(const Field& field, const std::string& val) { if (field.is_normal_term() || field.is_phrasable_term()) { const auto flat{utf8_flatten(val)}; if (field.is_normal_term()) - doc.add_term(field.xapian_term(flat)); + xdoc_.add_term(prefixed_term(field, flat)); if (field.is_phrasable_term()) { - auto termgen{make_term_generator(doc, opts)}; + auto termgen{make_term_generator(xdoc_, options_)}; + /* continue after the previous text, with a gap, so + * phrases cannot match across unrelated texts */ + termgen.set_termpos(termpos_); termgen.index_text(flat, 1, field.xapian_term()); + termgen.increase_termpos(); + termpos_ = termgen.get_termpos(); } } else if (field.is_boolean_term()) { - doc.add_boolean_term(field.xapian_term(val)); + xdoc_.add_boolean_term(field.xapian_term(val)); } else throw std::logic_error("not a search term"); } @@ -114,7 +131,7 @@ Document::add(Field::Id id, const std::string& val) xdoc_.add_value(field.value_no(), val); if (field.is_searchable()) - add_search_term(xdoc_, field, val, options_); + add_search_term(field, val); sexp_put_prop(field, val); @@ -130,7 +147,7 @@ Document::add(Field::Id id, const std::vector& vals) if (field.is_searchable()) std::for_each(vals.begin(), vals.end(), [&](const auto& val) { - add_search_term(xdoc_, field, val, options_); }); + add_search_term(field, val); }); if (field.include_in_sexp()) { Sexp elms{}; @@ -170,9 +187,11 @@ Document::add(Field::Id id, const Contacts& contacts) const auto field{field_from_id(id)}; std::vector cvec; + Sexp contacts_sexp; const std::string sepa2(1, SepaChar2); auto&& termgen{make_term_generator(xdoc_, options_)}; + termgen.set_termpos(termpos_); for (auto&& contact: contacts) { @@ -180,7 +199,7 @@ Document::add(Field::Id id, const Contacts& contacts) if (!cfield_id || *cfield_id != id) continue; - const auto e{contact.email}; + const auto& e{contact.email}; xdoc_.add_term(field.xapian_term(e)); /* allow searching for address components, too */ @@ -190,16 +209,28 @@ Document::add(Field::Id id, const Contacts& contacts) xdoc_.add_term(field.xapian_term(e.substr(atpos + 1))); } - if (!contact.name.empty()) + if (!contact.name.empty()) { termgen.index_text(utf8_flatten(contact.name), 1, field.xapian_term()); - cvec.emplace_back(contact.email + sepa2 + contact.name); + /* keep a gap between the contacts, so phrases cannot + * match across them */ + termgen.increase_termpos(); + } + cvec.emplace_back(e + sepa2 + contact.name); + + Sexp contact_sexp(":email"_sym, e); + if (!contact.name.empty()) + contact_sexp.add(":name"_sym, contact.name); + contacts_sexp.add(std::move(contact_sexp)); } + termpos_ = termgen.get_termpos(); - if (!cvec.empty()) + if (!cvec.empty()) { xdoc_.add_value(field.value_no(), join(cvec, SepaChar1)); - - sexp_put_prop(field, make_contacts_sexp(contacts)); + /* only the contacts matching the field-id, consistent with + * the terms & values above */ + sexp_put_prop(field, std::move(contacts_sexp)); + } } Contacts @@ -470,6 +501,60 @@ test_to() } +static std::vector +term_positions(const Xapian::Document& xdoc, const std::string& term) +{ + std::vector positions; + + auto it{xdoc.termlist_begin()}; + it.skip_to(term); + if (it == xdoc.termlist_end() || *it != term) + return positions; + + for (auto pit = it.positionlist_begin(); pit != it.positionlist_end(); ++pit) + positions.emplace_back(*pit); + + return positions; +} + +static void +test_termpos_contacts() +{ + Document doc; + doc.add(Field::Id::To, {{ + Contact{"lars@example.com", "Lars Foo", Contact::Type::To}, + Contact{"kirk@example.com", "Kirk Bar", Contact::Type::To} + }}); + + /* "foo" is the last word of the first contact, "kirk" the first word + * of the second; without a gap between contacts, the phrase + * to:"foo kirk" would match. */ + const auto foo_pos{term_positions(doc.xapian_document(), "Tfoo")}; + const auto kirk_pos{term_positions(doc.xapian_document(), "Tkirk")}; + g_assert_cmpuint(foo_pos.size(), ==, 1); + g_assert_cmpuint(kirk_pos.size(), ==, 1); + g_assert_cmpuint(kirk_pos.front(), >, foo_pos.front() + 1); +} + +static void +test_termpos_texts() +{ + Document doc; + doc.add(Field::Id::BodyText, std::string{"hello world"}); + doc.add(Field::Id::BodyText, std::string{"foo bar"}); + + /* separately-added texts must not overlap position-wise; otherwise + * e.g. the phrase "hello bar" could match. */ + const auto hello_pos{term_positions(doc.xapian_document(), "Bhello")}; + const auto world_pos{term_positions(doc.xapian_document(), "Bworld")}; + const auto foo_pos{term_positions(doc.xapian_document(), "Bfoo")}; + g_assert_cmpuint(hello_pos.size(), ==, 1); + g_assert_cmpuint(world_pos.size(), ==, 1); + g_assert_cmpuint(foo_pos.size(), ==, 1); + g_assert_cmpuint(world_pos.front(), ==, hello_pos.front() + 1); + g_assert_cmpuint(foo_pos.front(), >, world_pos.front() + 1); +} + static void test_size() { @@ -496,6 +581,8 @@ main(int argc, char* argv[]) g_test_add_func("/message/document/from", test_from); g_test_add_func("/message/document/to", test_to); + g_test_add_func("/message/document/termpos-contacts", test_termpos_contacts); + g_test_add_func("/message/document/termpos-texts", test_termpos_texts); g_test_add_func("/message/document/size", test_size); return g_test_run(); diff --git a/lib/message/mu-document.hh b/lib/message/mu-document.hh index 03200aeb..2a9bc505 100644 --- a/lib/message/mu-document.hh +++ b/lib/message/mu-document.hh @@ -1,4 +1,4 @@ -/** Copyright (C) 2022-2025 Dirk-Jan C. Binnema +/** Copyright (C) 2022-2026 Dirk-Jan C. Binnema ** ** This program is free software; you can redistribute it and/or modify it ** under the terms of the GNU General Public License as published by the @@ -165,7 +165,10 @@ public: * * @return message s-expression string */ - std::string sexp_str() const { return xdoc_.get_data(); } + std::string sexp_str() const { + /* xapian_document() flushes any dirty cached sexp first */ + return xapian_document().get_data(); + } /** * Generically adds an optional value, if set, to the document @@ -242,6 +245,7 @@ public: private: template void sexp_put_prop(const Field& field, SexpType&& val); void sexp_remove_prop(const Field& field); + void add_search_term(const Field& field, const std::string& val); Sexp& cached_sexp() const { if (cached_sexp_.empty()) @@ -251,9 +255,13 @@ private: } mutable Xapian::Document xdoc_; - Options options_; - mutable Sexp cached_sexp_; + Options options_; + mutable Sexp cached_sexp_; mutable bool dirty_sexp_{}; /* xdoc's sexp is outdated */ + Xapian::termcount termpos_{}; /* next index term-position; + * tracked so phrases cannot + * match across unrelated + * texts */ }; MU_ENABLE_BITOPS(Document::Options);