lib/query: refactor & rework

- Move the lib/query/ stuff up a level into lib/
- Associate directly with the Query object
- Rework the Query object to be C++ rather than mixed with C
- Update all dependencies, tests

lib/Makefile.am

@@ -18,7 +18,7 @@
 # before descending into tests/
 include $(top_srcdir)/gtest.mk
 
-SUBDIRS= utils query index
+SUBDIRS= utils index
 
 TESTDEFS=							\
 	-DMU_TESTMAILDIR=\"${abs_srcdir}/testdir\"		\
@@ -34,9 +34,7 @@ AM_CFLAGS=							\
 	$(GMIME_CFLAGS)						\
 	$(GLIB_CFLAGS)						\
 	$(GUILE_CFLAGS)						\
-	$(JSON_GLIB_CFLAGS)					\
 	$(ASAN_CFLAGS)						\
-	$(json_flag)						\
 	$(CODE_COVERAGE_CFLAGS)					\
 	$(TESTDEFS)						\
 	-Wno-format-nonliteral					\
@@ -48,8 +46,6 @@ AM_CXXFLAGS=							\
 	$(GMIME_CFLAGS)						\
 	$(GLIB_CFLAGS)						\
 	$(GUILE_CFLAGS)						\
-	$(JSON_GLIB_CFLAGS)					\
-	$(json_flag)						\
 	$(WARN_CXXFLAGS)					\
 	$(XAPIAN_CXXFLAGS)					\
 	$(ASAN_CXXFLAGS)					\
@@ -59,13 +55,6 @@ AM_CXXFLAGS=							\
 AM_CPPFLAGS=							\
 	$(CODE_COVERAGE_CPPFLAGS)
 
-# don't use -Werror, as it might break on other compilers
-# use -Wno-unused-parameters, because some callbacks may not
-# really need all the params they get
-# AM_CFLAGS=-Wall -Wextra -Wno-unused-parameter			\
-# -Wdeclaration-after-statement -Wno-variadic-macros
-# AM_CXXFLAGS=-Wall -Wextra -Wno-unused-parameter
-
 noinst_LTLIBRARIES=						\
 	libmu.la
 
@@ -76,8 +65,9 @@ libmu_la_SOURCES=						\
 	mu-contacts.hh						\
 	mu-container.c						\
 	mu-container.h						\
-	mu-flags.h						\
+	mu-data.hh						\
 	mu-flags.c						\
+	mu-flags.h						\
 	mu-maildir.c						\
 	mu-maildir.h						\
 	mu-msg-crypto.c						\
@@ -89,7 +79,6 @@ libmu_la_SOURCES=						\
 	mu-msg-file.h						\
 	mu-msg-iter.cc						\
 	mu-msg-iter.h						\
-	$(json_srcs)						\
 	mu-msg-part.c						\
 	mu-msg-part.h						\
 	mu-msg-prio.c						\
@@ -99,8 +88,10 @@ libmu_la_SOURCES=						\
 	mu-msg.c						\
 	mu-msg.h						\
 	mu-msg.h						\
+	mu-parser.cc						\
+	mu-parser.hh						\
 	mu-query.cc						\
-	mu-query.h						\
+	mu-query.hh						\
 	mu-runtime.cc						\
 	mu-runtime.h						\
 	mu-script.c						\
@@ -110,40 +101,54 @@ libmu_la_SOURCES=						\
 	mu-store.cc						\
 	mu-store.hh						\
 	mu-threader.c						\
-	mu-threader.h
+	mu-threader.h						\
+	mu-tokenizer.cc						\
+	mu-tokenizer.hh						\
+	mu-tree.hh						\
+	mu-xapian.cc						\
+	mu-xapian.hh
 
 libmu_la_LIBADD=						\
 	$(XAPIAN_LIBS)						\
 	$(GMIME_LIBS)						\
 	$(GLIB_LIBS)						\
 	$(GUILE_LIBS)						\
-	$(JSON_GLIB_LIBS)					\
-	${builddir}/query/libmu-query.la			\
 	${builddir}/index/libmu-index.la			\
 	$(CODE_COVERAGE_LIBS)
 
 libmu_la_LDFLAGS=						\
 	$(ASAN_LDFLAGS)
 
+noinst_PROGRAMS=						\
+	tokenize
+
+tokenize_SOURCES=						\
+	tokenize.cc
+
+tokenize_LDADD=							\
+	$(WARN_LDFLAGS)						\
+	libmu.la						\
+	utils/libmu-utils.la
+
 EXTRA_DIST=							\
 	mu-msg-crypto.c						\
 	doxyfile.in
 
-noinst_PROGRAMS= $(TEST_PROGS)
+noinst_PROGRAMS+=$(TEST_PROGS)
 
 noinst_LTLIBRARIES+=						\
 	libtestmucommon.la
 
 TEST_PROGS += test-mu-maildir
-test_mu_maildir_SOURCES= test-mu-maildir.c dummy.cc
+test_mu_maildir_SOURCES= test-mu-maildir.cc
 test_mu_maildir_LDADD=  libtestmucommon.la
 
 TEST_PROGS += test-mu-msg-fields
-test_mu_msg_fields_SOURCES= test-mu-msg-fields.c dummy.cc
+test_mu_msg_fields_SOURCES= test-mu-msg-fields.cc
 test_mu_msg_fields_LDADD=  libtestmucommon.la
 
 TEST_PROGS += test-mu-msg
-test_mu_msg_SOURCES= test-mu-msg.c dummy.cc
+test_mu_msg_SOURCES= test-mu-msg.cc
 test_mu_msg_LDADD=  libtestmucommon.la
 
 TEST_PROGS += test-mu-store
@@ -151,29 +156,32 @@ test_mu_store_SOURCES= test-mu-store.cc
 test_mu_store_LDADD= libtestmucommon.la
 
 TEST_PROGS += test-mu-flags
-test_mu_flags_SOURCES= test-mu-flags.c dummy.cc
+test_mu_flags_SOURCES= test-mu-flags.cc
 test_mu_flags_LDADD=  libtestmucommon.la
 
 TEST_PROGS += test-mu-container
-test_mu_container_SOURCES= test-mu-container.c dummy.cc
+test_mu_container_SOURCES= test-mu-container.cc
 test_mu_container_LDADD= libtestmucommon.la
 
 TEST_PROGS += test-mu-contacts
 test_mu_contacts_SOURCES= test-mu-contacts.cc
 test_mu_contacts_LDADD= libtestmucommon.la
 
-# we need to use dummy.cc to enforce c++ linking...
+TEST_PROGS+=test-mu-tokenizer
-BUILT_SOURCES=							\
+test_mu_tokenizer_SOURCES=test-tokenizer.cc
-	dummy.cc
+test_mu_tokenizer_LDADD=libtestmucommon.la
 
-dummy.cc:
+# TEST_PROGS+=test-mu-parser
-	touch dummy.cc
+# test_mu_parser_SOURCES=test-parser.cc
+# test_mu_parser_LDADD=libtestmucommon.la
 
 libtestmucommon_la_SOURCES=					\
-	test-mu-common.c					\
+	test-mu-common.cc					\
-	test-mu-common.h
+	test-mu-common.hh
+
 libtestmucommon_la_LIBADD=					\
-	libmu.la
+	libmu.la						\
+	utils/libmu-utils.la
 
 # note the question marks; make does not like files with ':', so we
 # use the (also supported) version with '!' instead. We could escape

lib/index/mu-indexer.cc

@@ -205,7 +205,7 @@ Indexer::Private::cleanup()
         g_debug ("starting cleanup");
 
         std::vector<Store::Id> orphans_; // store messages without files.
-        store_.for_each([&](Store::Id id, const std::string &path) {
+        store_.for_each_message_path([&](Store::Id id, const std::string &path) {
 
                 if (clean_done_)
                         return false;

lib/mu-container.h

@@ -23,6 +23,8 @@
 #include <glib.h>
 #include <mu-msg.h>
 
+G_BEGIN_DECLS
+
 enum _MuContainerFlag {
 	MU_CONTAINER_FLAG_NONE    = 0,
 	MU_CONTAINER_FLAG_DELETE  = 1 << 0,
@@ -221,4 +223,6 @@ MuContainer* mu_container_sort (MuContainer *c, MuMsgFieldId mfid,
 GHashTable* mu_container_thread_info_hash_new (MuContainer *root_set,
 					       size_t matchnum);
 
+G_END_DECLS
+
 #endif /*__MU_CONTAINER_H__*/

lib/mu-parser.cc

@@ -0,0 +1,527 @@
+/*
+**  Copyright (C) 2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+**  This library is free software; you can redistribute it and/or
+**  modify it under the terms of the GNU Lesser General Public License
+**  as published by the Free Software Foundation; either version 2.1
+**  of the License, or (at your option) any later version.
+**
+**  This library is distributed in the hope that it will be useful,
+**  but WITHOUT ANY WARRANTY; without even the implied warranty of
+**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+**  Lesser General Public License for more details.
+**
+**  You should have received a copy of the GNU Lesser General Public
+**  License along with this library; if not, write to the Free
+**  Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA
+**  02110-1301, USA.
+*/
+#include "mu-parser.hh"
+#include "mu-tokenizer.hh"
+#include "utils/mu-utils.hh"
+#include "utils/mu-error.hh"
+#include <algorithm>
+
+using namespace Mu;
+
+// 3 precedence levels: units (NOT,()) > factors (OR) > terms (AND)
+
+// query	->      <term-1> | ε
+// <term-1>	->	<factor-1> <term-2> | ε
+// <term-2>	->	OR|XOR <term-1> | ε
+// <factor-1>	->	<unit> <factor-2> | ε
+// <factor-2>	->	[AND]|AND NOT <factor-1> | ε
+// <unit>	->	[NOT] <term-1> | ( <term-1> ) | <data>
+// <data>       ->      <value> | <range> | <regex>
+// <value>      ->      [field:]value
+// <range>      ->      [field:][lower]..[upper]
+// <regex>      ->      [field:]/regex/
+
+
+#define BUG(...) Mu::Error (Error::Code::Internal, format("%u: BUG: ",__LINE__)	\
+				     + format(__VA_ARGS__))
+
+
+
+/**
+ * Get the "shortcut"/internal fields for the the given fieldstr or empty if there is none
+ *
+ * @param fieldstr a fieldstr, e.g "subject" or "s" for the subject field
+ *
+ * @return a vector with "exploded" values, with a code and a fullname. E.g. "s" might map
+ * to [<"S","subject">], while "recip" could map to [<"to", "T">, <"cc", "C">, <"bcc", "B">]
+ */
+struct FieldInfo {
+        const std::string	field;
+        const std::string	prefix;
+        bool			supports_phrase;
+        unsigned		id;
+};
+using FieldInfoVec = std::vector<FieldInfo>;
+
+struct Parser::Private {
+        Private(const Store& store): store_{store} {}
+
+        std::vector<std::string> process_regex (const std::string& field,
+                                                const std::regex& rx) const;
+
+        Mu::Tree term_1 (Mu::Tokens& tokens,  WarningVec& warnings) const;
+        Mu::Tree term_2 (Mu::Tokens& tokens, Node::Type& op, WarningVec& warnings) const;
+        Mu::Tree factor_1 (Mu::Tokens& tokens, WarningVec& warnings) const;
+        Mu::Tree factor_2 (Mu::Tokens& tokens, Node::Type& op, WarningVec& warnings) const;
+        Mu::Tree unit (Mu::Tokens& tokens, WarningVec& warnings) const;
+        Mu::Tree data (Mu::Tokens& tokens, WarningVec& warnings) const;
+        Mu::Tree range (const FieldInfoVec& fields, const std::string& lower,
+                        const std::string& upper, size_t pos, WarningVec& warnings) const;
+        Mu::Tree regex (const FieldInfoVec& fields, const std::string& v,
+                        size_t pos,  WarningVec& warnings) const;
+        Mu::Tree value (const FieldInfoVec& fields, const std::string& v,
+                        size_t pos, WarningVec& warnings) const;
+private:
+        const Store& store_;
+};
+
+static MuMsgFieldId
+field_id (const std::string& field)
+{
+
+        if (field.empty())
+                return MU_MSG_FIELD_ID_NONE;
+
+        MuMsgFieldId id = mu_msg_field_id_from_name (field.c_str(), FALSE);
+        if (id != MU_MSG_FIELD_ID_NONE)
+                return id;
+        else if (field.length() == 1)
+                return mu_msg_field_id_from_shortcut (field[0], FALSE);
+        else
+                return MU_MSG_FIELD_ID_NONE;
+}
+
+static std::string
+process_value (const std::string& field, const std::string& value)
+{
+        const auto id = field_id (field);
+        if (id == MU_MSG_FIELD_ID_NONE)
+                return value;
+        switch(id) {
+        case MU_MSG_FIELD_ID_PRIO: {
+                if (!value.empty())
+                        return std::string(1, value[0]);
+        } break;
+
+        case MU_MSG_FIELD_ID_FLAGS: {
+                const auto flag = mu_flag_char_from_name (value.c_str());
+                if (flag)
+                        return std::string(1, tolower(flag));
+        } break;
+
+        default:
+                break;
+        }
+
+        return value; // XXX prio/flags, etc. alias
+}
+
+static void
+add_field (std::vector<FieldInfo>& fields, MuMsgFieldId id)
+{
+        const auto shortcut = mu_msg_field_shortcut(id);
+        if (!shortcut)
+                return; // can't be searched
+
+        const auto name = mu_msg_field_name (id);
+        const auto pfx  = mu_msg_field_xapian_prefix (id);
+
+        if (!name || !pfx)
+                return;
+
+        fields.push_back ({{name}, {pfx},
+                        (bool)mu_msg_field_xapian_index(id),
+                        id});
+}
+
+static std::vector<FieldInfo>
+process_field (const std::string& field)
+{
+
+        std::vector<FieldInfo> fields;
+
+        if (field == "contact" || field == "recip") { // multi fields
+                add_field (fields, MU_MSG_FIELD_ID_TO);
+                add_field (fields, MU_MSG_FIELD_ID_CC);
+                add_field (fields, MU_MSG_FIELD_ID_BCC);
+                if (field == "contact")
+                        add_field (fields, MU_MSG_FIELD_ID_FROM);
+        } else if (field == "") {
+                add_field (fields, MU_MSG_FIELD_ID_TO);
+                add_field (fields, MU_MSG_FIELD_ID_CC);
+                add_field (fields, MU_MSG_FIELD_ID_BCC);
+                add_field (fields, MU_MSG_FIELD_ID_FROM);
+                add_field (fields, MU_MSG_FIELD_ID_SUBJECT);
+                add_field (fields, MU_MSG_FIELD_ID_BODY_TEXT);
+        } else {
+                const auto id = field_id (field.c_str());
+                if (id != MU_MSG_FIELD_ID_NONE)
+                        add_field (fields, id);
+        }
+
+        return fields;
+}
+
+static bool
+is_range_field (const std::string& field)
+{
+        const auto id = field_id (field.c_str());
+        if (id == MU_MSG_FIELD_ID_NONE)
+                return false;
+        else
+                return mu_msg_field_is_range_field (id);
+}
+
+struct MyRange {
+        std::string lower;
+        std::string upper;
+};
+
+static MyRange
+process_range (const std::string& field, const std::string& lower,
+               const std::string& upper)
+{
+        const auto id = field_id (field.c_str());
+        if (id == MU_MSG_FIELD_ID_NONE)
+                return { lower, upper };
+
+        std::string	l2 = lower;
+        std::string	u2 = upper;
+
+        if (id == MU_MSG_FIELD_ID_DATE) {
+                l2 = Mu::date_to_time_t_string (lower, true);
+                u2 = Mu::date_to_time_t_string (upper, false);
+        } else if (id == MU_MSG_FIELD_ID_SIZE) {
+                l2 = Mu::size_to_string (lower, true);
+                u2 = Mu::size_to_string (upper, false);
+        }
+
+        return { l2, u2 };
+}
+
+std::vector<std::string>
+Parser::Private::process_regex (const std::string& field, const std::regex& rx) const
+{
+        const auto id = field_id (field.c_str());
+        if (id == MU_MSG_FIELD_ID_NONE)
+                return {};
+
+        char pfx[] = {  mu_msg_field_xapian_prefix(id), '\0' };
+
+        std::vector<std::string> terms;
+        store_.for_each_term(pfx,[&](auto&& str){
+                if (std::regex_search(str.c_str() + 1, rx)) // avoid copy
+                        terms.emplace_back(str);
+                return true;
+        });
+
+        return terms;
+}
+
+static Token
+look_ahead (const Mu::Tokens& tokens)
+{
+	return tokens.front();
+}
+
+static Mu::Tree
+empty()
+{
+	return {{Node::Type::Empty}};
+}
+
+Mu::Tree
+Parser::Private::value (const FieldInfoVec& fields, const std::string& v,
+                        size_t pos, WarningVec& warnings) const
+{
+	auto val = utf8_flatten(v);
+
+	if (fields.empty())
+		throw BUG("expected one or more fields");
+
+	if (fields.size() == 1) {
+		const auto item = fields.front();
+		return Tree({Node::Type::Value,
+			     std::make_unique<Value>(
+				     item.field, item.prefix, item.id,
+				     process_value(item.field, val),
+				     item.supports_phrase)});
+	}
+
+	// a 'multi-field' such as "recip:"
+	Tree tree(Node{Node::Type::OpOr});
+	for (const auto& item: fields)
+		tree.add_child (Tree({Node::Type::Value,
+				      std::make_unique<Value>(
+					      item.field, item.prefix, item.id,
+					      process_value(item.field, val),
+					      item.supports_phrase)}));
+	return tree;
+}
+
+Mu::Tree
+Parser::Private::regex (const FieldInfoVec& fields, const std::string& v,
+                        size_t pos,  WarningVec& warnings) const
+{
+	if (v.length() < 2)
+		throw BUG("expected regexp, got '%s'", v.c_str());
+
+	const auto rxstr = utf8_flatten(v.substr(1, v.length()-2));
+
+ 	try {
+		Tree tree(Node{Node::Type::OpOr});
+		const auto rx = std::regex (rxstr);
+		for (const auto& field: fields) {
+			const auto terms = process_regex (field.field, rx);
+			for (const auto& term: terms) {
+				tree.add_child (Tree(
+					{Node::Type::Value,
+					 std::make_unique<Value>(field.field, "",
+								 field.id, term)}));
+			}
+		}
+
+		if (tree.children.empty())
+			return empty();
+		else
+			return tree;
+
+	} catch (...) {
+		// fallback
+		warnings.push_back ({pos, "invalid regexp"});
+		return value (fields, v, pos, warnings);
+	}
+}
+
+
+
+Mu::Tree
+Parser::Private::range (const FieldInfoVec& fields, const std::string& lower,
+                        const std::string& upper, size_t pos, WarningVec& warnings) const
+{
+	if (fields.empty())
+		throw BUG("expected field");
+
+	const auto& field = fields.front();
+	if (!is_range_field(field.field))
+		return value (fields, lower + ".." + upper, pos, warnings);
+
+	auto prange = process_range (field.field, lower, upper);
+	if (prange.lower > prange.upper)
+		prange = process_range (field.field, upper, lower);
+
+	return Tree({Node::Type::Range,
+                        std::make_unique<Range>(field.field, field.prefix, field.id,
+                                                prange.lower, prange.upper)});
+}
+
+Mu::Tree
+Parser::Private::data (Mu::Tokens& tokens, WarningVec& warnings) const
+{
+	const auto token = look_ahead(tokens);
+	if (token.type != Token::Type::Data)
+		warnings.push_back ({token.pos, "expected: value"});
+
+	tokens.pop_front();
+
+	std::string field, val;
+	const auto col = token.str.find (":");
+	if (col != 0 && col != std::string::npos && col != token.str.length()-1) {
+		field = token.str.substr(0, col);
+		val = token.str.substr(col + 1);
+	} else
+		val = token.str;
+
+	auto fields = process_field (field);
+	if (fields.empty()) {// not valid field...
+		warnings.push_back ({token.pos, format ("invalid field '%s'", field.c_str())});
+		fields = process_field ("");
+		// fallback, treat the whole of foo:bar as a value
+		return value (fields, field + ":" + val, token.pos, warnings);
+	}
+
+	// does it look like a regexp?
+	if (val.length() >=2 )
+		if (val[0] == '/' && val[val.length()-1] == '/')
+			return regex (fields, val, token.pos, warnings);
+
+	// does it look like a range?
+	const auto dotdot = val.find("..");
+	if (dotdot != std::string::npos)
+		return range(fields, val.substr(0, dotdot), val.substr(dotdot + 2),
+			     token.pos, warnings);
+	else if (is_range_field(fields.front().field)) {
+		// range field without a range - treat as field:val..val
+		return range (fields, val, val, token.pos, warnings);
+	}
+
+	// if nothing else, it's a value.
+	return value (fields, val, token.pos, warnings);
+}
+
+Mu::Tree
+Parser::Private::unit (Mu::Tokens& tokens, WarningVec& warnings) const
+{
+	if (tokens.empty()) {
+		warnings.push_back ({0, "expected: unit"});
+		return empty();
+	}
+
+	const auto token = look_ahead (tokens);
+
+	if (token.type == Token::Type::Not) {
+		tokens.pop_front();
+		Tree tree{{Node::Type::OpNot}};
+		tree.add_child(unit (tokens, warnings));
+		return tree;
+	}
+
+	if (token.type == Token::Type::Open) {
+		tokens.pop_front();
+		auto tree = term_1 (tokens, warnings);
+		if (tokens.empty())
+			warnings.push_back({token.pos, "expected: ')'"});
+		else {
+			const auto token2 = look_ahead(tokens);
+			if (token2.type == Token::Type::Close)
+				tokens.pop_front();
+			else {
+				warnings.push_back(
+				{token2.pos,
+				 std::string("expected: ')' but got ") +
+				 token2.str});
+			}
+
+		}
+		return tree;
+	}
+
+	return data (tokens, warnings);
+}
+
+Mu::Tree
+Parser::Private::factor_2 (Mu::Tokens& tokens, Node::Type& op,
+                           WarningVec& warnings) const
+{
+	if (tokens.empty())
+		return empty();
+
+	const auto token = look_ahead(tokens);
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored   "-Wswitch-enum"
+	switch (token.type) {
+	case Token::Type::And: {
+		tokens.pop_front();
+		op = Node::Type::OpAnd;
+	} break;
+
+	case Token::Type::Open:
+	case Token::Type::Data:
+	case Token::Type::Not:
+		op = Node::Type::OpAnd; // implicit AND
+		break;
+
+	default:
+		return empty();
+	}
+#pragma GCC diagnostic pop
+
+	return factor_1 (tokens, warnings);
+}
+
+
+Mu::Tree
+Parser::Private::factor_1 (Mu::Tokens& tokens, WarningVec& warnings) const
+{
+	Node::Type op { Node::Type::Invalid };
+
+	auto t  = unit (tokens, warnings);
+	auto a2 = factor_2 (tokens, op, warnings);
+
+	if (a2.empty())
+		return t;
+
+	Tree tree {{op}};
+	tree.add_child(std::move(t));
+	tree.add_child(std::move(a2));
+
+	return tree;
+}
+
+
+Mu::Tree
+Parser::Private::term_2 (Mu::Tokens& tokens, Node::Type& op, WarningVec& warnings) const
+{
+	if (tokens.empty())
+		return empty();
+
+	const auto token = look_ahead (tokens);
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored   "-Wswitch-enum"
+	switch (token.type) {
+	case Token::Type::Or:
+		op = Node::Type::OpOr;
+		break;
+	case Token::Type::Xor:
+		op = Node::Type::OpXor;
+		break;
+	default:
+		if (token.type != Token::Type::Close)
+			warnings.push_back({token.pos, "expected OR|XOR"});
+		return empty();
+	}
+#pragma GCC diagnostic pop
+
+	tokens.pop_front();
+
+	return term_1 (tokens, warnings);
+}
+
+Mu::Tree
+Parser::Private::term_1 (Mu::Tokens& tokens,  WarningVec& warnings) const
+{
+	Node::Type op { Node::Type::Invalid };
+
+	auto t  = factor_1 (tokens, warnings);
+	auto o2 = term_2 (tokens, op, warnings);
+
+	if (o2.empty())
+		return t;
+	else {
+		Tree tree {{op}};
+		tree.add_child(std::move(t));
+		tree.add_child(std::move(o2));
+		return tree;
+	}
+}
+
+Mu::Parser::Parser(const Store& store):
+        priv_{std::make_unique<Private>(store)}
+{}
+
+Mu::Parser::~Parser() = default;
+
+
+Mu::Tree
+Mu::Parser::parse (const std::string& expr, WarningVec& warnings) const
+{
+	try {
+		auto tokens = tokenize (expr);
+                if (tokens.empty())
+                        return empty ();
+                else
+                        return priv_->term_1 (tokens, warnings);
+
+	} catch (const std::runtime_error& ex) {
+		std::cerr << ex.what() << std::endl;
+		return empty();
+	}
+}

lib/mu-parser.hh

@@ -25,9 +25,9 @@
 #include <vector>
 #include <memory>
 
-#include <query/mu-data.hh>
+#include <mu-data.hh>
-#include <query/mu-tree.hh>
+#include <mu-tree.hh>
-#include <query/mu-proc-iface.hh>
+#include <mu-store.hh>
 
 // A simple recursive-descent parser for queries. Follows the Xapian syntax,
 // but better handles non-alphanum; also implements regexp
@@ -53,7 +53,7 @@ struct Warning {
 		return pos == rhs.pos && msg == rhs.msg;
 	}
 };
-
+using WarningVec=std::vector<Warning>;
 
 /**
  * operator<<
@@ -70,19 +70,34 @@ operator<< (std::ostream& os, const Warning& w)
 	return os;
 }
 
-/**
+class Parser {
- * Parse a query string
+public:
- *
+        /**
- * @param query a query string
+         * Construct a query parser object
- * @param warnings vec to receive warnings
+         *
- * @param proc a Processor object
+         * @param store a store object ptr, or none
- *
+         */
- * @return a parse-tree
+        Parser(const Store& store);
- */
+        /**
-using WarningVec=std::vector<Warning>;
+         * DTOR
-using ProcPtr = const std::unique_ptr<ProcIface>&;
+         *
-Tree parse (const std::string& query, WarningVec& warnings,
+         */
-	    ProcPtr proc = std::make_unique<DummyProc>());
+        ~Parser();
+
+        /**
+         * Parse a query string
+         *
+         * @param query a query string
+         * @param warnings vec to receive warnings
+         *
+         * @return a parse-tree
+         */
+
+        Tree parse (const std::string& query, WarningVec& warnings) const;
+private:
+        struct Private;
+        std::unique_ptr<Private> priv_;
+};
 
 } // namespace Mu
 

lib/mu-query.cc

@@ -1,5 +1,5 @@
 /*
-** Copyright (C) 2008-2017 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+** Copyright (C) 2008-2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
 **
 ** This program is free software; you can redistribute it and/or modify
 ** it under the terms of the GNU General Public License as published by
@@ -16,6 +16,7 @@
 ** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 **
 */
+#include <mu-query.hh>
 
 #include <stdexcept>
 #include <string>
@@ -27,7 +28,6 @@
 #include <xapian.h>
 #include <glib/gstdio.h>
 
-#include "mu-query.h"
 #include "mu-msg-fields.h"
 
 #include "mu-msg-iter.h"
@@ -36,273 +36,94 @@
 #include "utils/mu-date.h"
 #include <utils/mu-utils.hh>
 
-#include <query/mu-proc-iface.hh>
+#include <mu-xapian.hh>
-#include <query/mu-xapian.hh>
 
 using namespace Mu;
 
-struct MuProc: public Mu::ProcIface {
+struct Query::Private {
+        Private(const Store& store): store_{store},
+                                     parser_{store_} {}
 
-	MuProc (const Xapian::Database& db): db_{db} {}
+        Xapian::Query   make_query (const std::string& expr, GError **err) const;
+        Xapian::Enquire make_enquire (const std::string& expr, MuMsgFieldId sortfieldid,
+                                      bool descending, GError **err) const;
+        GHashTable* find_thread_ids (MuMsgIter *iter, GHashTable **orig_set) const;
 
-	static MuMsgFieldId field_id (const std::string& field) {
+        Xapian::Query make_related_query (MuMsgIter *iter, GHashTable **orig_set) const;
 
-		if (field.empty())
+        void find_related_messages (MuMsgIter **iter, int maxnum,
-			return MU_MSG_FIELD_ID_NONE;
+                                    MuMsgFieldId sortfieldid, Query::Flags flags,
+                                    Xapian::Query orig_query) const;
 
-		MuMsgFieldId id = mu_msg_field_id_from_name (field.c_str(), FALSE);
+        const Store& store_;
-		if (id != MU_MSG_FIELD_ID_NONE)
+        const Parser parser_;
-			return id;
-		else if (field.length() == 1)
-			return mu_msg_field_id_from_shortcut (field[0], FALSE);
-		else
-			return MU_MSG_FIELD_ID_NONE;
-	}
-
-	std::string
-	process_value (const std::string& field,
-		       const std::string& value) const override {
-		const auto id = field_id (field);
-		if (id == MU_MSG_FIELD_ID_NONE)
-			return value;
-		switch(id) {
-		case MU_MSG_FIELD_ID_PRIO: {
-			if (!value.empty())
-				return std::string(1, value[0]);
-		} break;
-
-		case MU_MSG_FIELD_ID_FLAGS: {
-			const auto flag = mu_flag_char_from_name (value.c_str());
-			if (flag)
-				return std::string(1, tolower(flag));
-		} break;
-
-		default:
-			break;
-		}
-
-		return value; // XXX prio/flags, etc. alias
-	}
-
-	void add_field (std::vector<FieldInfo>& fields, MuMsgFieldId id) const {
-
-		const auto shortcut = mu_msg_field_shortcut(id);
-		if (!shortcut)
-			return; // can't be searched
-
-		const auto name = mu_msg_field_name (id);
-		const auto pfx  = mu_msg_field_xapian_prefix (id);
-
-		if (!name || !pfx)
-			return;
-
-		fields.push_back ({{name}, {pfx},
-				   (bool)mu_msg_field_xapian_index(id),
-				   id});
-	}
-
-	std::vector<FieldInfo>
-	process_field (const std::string& field) const override {
-
-		std::vector<FieldInfo> fields;
-
-		if (field == "contact" || field == "recip") { // multi fields
-			add_field (fields, MU_MSG_FIELD_ID_TO);
-			add_field (fields, MU_MSG_FIELD_ID_CC);
-			add_field (fields, MU_MSG_FIELD_ID_BCC);
-			if (field == "contact")
-				add_field (fields, MU_MSG_FIELD_ID_FROM);
-		} else if (field == "") {
-			add_field (fields, MU_MSG_FIELD_ID_TO);
-			add_field (fields, MU_MSG_FIELD_ID_CC);
-			add_field (fields, MU_MSG_FIELD_ID_BCC);
-			add_field (fields, MU_MSG_FIELD_ID_FROM);
-			add_field (fields, MU_MSG_FIELD_ID_SUBJECT);
-			add_field (fields, MU_MSG_FIELD_ID_BODY_TEXT);
-		} else {
-			const auto id = field_id (field.c_str());
-			if (id != MU_MSG_FIELD_ID_NONE)
-				add_field (fields, id);
-		}
-
-		return fields;
-	}
-
-	bool is_range_field (const std::string& field) const override {
-		const auto id = field_id (field.c_str());
-		if (id == MU_MSG_FIELD_ID_NONE)
-			return false;
-		else
-			return mu_msg_field_is_range_field (id);
-	}
-
-	Range process_range (const std::string& field, const std::string& lower,
-			     const std::string& upper) const override {
-
-		const auto id = field_id (field.c_str());
-		if (id == MU_MSG_FIELD_ID_NONE)
-			return { lower, upper };
-
-		std::string	l2 = lower;
-		std::string	u2 = upper;
-
-		if (id == MU_MSG_FIELD_ID_DATE) {
-			l2 = Mu::date_to_time_t_string (lower, true);
-			u2 = Mu::date_to_time_t_string (upper, false);
-		} else if (id == MU_MSG_FIELD_ID_SIZE) {
-			l2 = Mu::size_to_string (lower, true);
-			u2 = Mu::size_to_string (upper, false);
-		}
-
-		return { l2, u2 };
-	}
-
-	std::vector<std::string>
-	process_regex (const std::string& field, const std::regex& rx) const override {
-
-		const auto id = field_id (field.c_str());
-		if (id == MU_MSG_FIELD_ID_NONE)
-			return {};
-
-		char pfx[] = {  mu_msg_field_xapian_prefix(id), '\0' };
-
-		std::vector<std::string> terms;
-		for (auto it = db_.allterms_begin(pfx); it != db_.allterms_end(pfx); ++it) {
-			if (std::regex_search((*it).c_str() + 1, rx)) // avoid copy
-				terms.push_back(*it);
-		}
-
-		return terms;
-	}
-
-	const Xapian::Database& db_;
 };
 
-struct _MuQuery {
-public:
-	_MuQuery (MuStore *store): _store(mu_store_ref(store)) {}
-	~_MuQuery () { mu_store_unref (_store); }
 
-	Xapian::Database& db() const {
+static constexpr MuMsgIterFlags
-		const auto db = reinterpret_cast<Xapian::Database*>
+msg_iter_flags (Query::Flags flags)
-			(mu_store_get_read_only_database (_store));
-		if (!db)
-			throw Mu::Error(Error::Code::NotFound, "no database");
-		return *db;
-	}
-private:
-	MuStore *_store;
-};
-
-static const Xapian::Query
-get_query (MuQuery *mqx, const char* searchexpr, bool raw, GError **err) try {
-
-	Mu::WarningVec warns;
-	const auto tree = Mu::parse (searchexpr, warns,
-				      std::make_unique<MuProc>(mqx->db()));
-	for (auto&& w: warns)
-		std::cerr << w << std::endl;
-
-	return Mu::xapian_query (tree);
-
-} catch (...) {
-	mu_util_g_set_error (err,MU_ERROR_XAPIAN_QUERY,
-			     "parse error in query");
-	throw;
-}
-
-MuQuery*
-mu_query_new (MuStore *store, GError **err)
 {
-	g_return_val_if_fail (store, NULL);
+	MuMsgIterFlags iflags{MU_MSG_ITER_FLAG_NONE};
 
-	try {
+	if (any_of(flags & Query::Flags::Descending))
-		return new MuQuery (store);
-	} MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN (err, MU_ERROR_XAPIAN, 0);
-
-	return 0;
-}
-
-void
-mu_query_destroy (MuQuery *self)
-{
-	try { delete self; } MU_XAPIAN_CATCH_BLOCK;
-}
-
-
-/* this function is for handling the case where a DatabaseModified
- * exception is raised. We try to reopen the database, and run the
- * query again. */
-static MuMsgIter *
-try_requery (MuQuery *self, const char* searchexpr, MuMsgFieldId sortfieldid,
-	     int maxnum, MuQueryFlags flags, GError **err)
-{
-	try {
-		/* let's assume that infinite regression is
-		 * impossible */
-		self->db().reopen();
-		g_message ("reopening db after modification");
-		return mu_query_run (self, searchexpr, sortfieldid,
-				     maxnum, flags, err);
-
-	} MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN (err, MU_ERROR_XAPIAN, 0);
-}
-
-
-static MuMsgIterFlags
-msg_iter_flags (MuQueryFlags flags)
-{
-	MuMsgIterFlags iflags;
-
-	iflags = MU_MSG_ITER_FLAG_NONE;
-
-	if (flags & MU_QUERY_FLAG_DESCENDING)
 		iflags |= MU_MSG_ITER_FLAG_DESCENDING;
-	if (flags & MU_QUERY_FLAG_SKIP_UNREADABLE)
+	if (any_of(flags & Query::Flags::SkipUnreadable))
 		iflags |= MU_MSG_ITER_FLAG_SKIP_UNREADABLE;
-	if (flags & MU_QUERY_FLAG_SKIP_DUPS)
+	if (any_of(flags & Query::Flags::SkipDups))
 		iflags |= MU_MSG_ITER_FLAG_SKIP_DUPS;
-	if (flags & MU_QUERY_FLAG_THREADS)
+	if (any_of(flags & Query::Flags::Threading))
 		iflags |= MU_MSG_ITER_FLAG_THREADS;
 
 	return iflags;
 }
 
+Xapian::Query
+Query::Private::make_query (const std::string& expr, GError **err) const try {
+
+	Mu::WarningVec warns;
+        const auto tree{parser_.parse(expr, warns)};
+	for (auto&& w: warns)
+                g_warning ("query warning: %s", to_string(w).c_str());
+
+	return Mu::xapian_query (tree);
+
+} catch (...) {
+	mu_util_g_set_error (err, MU_ERROR_XAPIAN_QUERY,
+			     "parse error in query");
+	throw;
+}
 
 
-static Xapian::Enquire
+Xapian::Enquire
-get_enquire (MuQuery *self, const char *searchexpr, MuMsgFieldId sortfieldid,
+Query::Private::make_enquire (const std::string& expr, MuMsgFieldId sortfieldid,
-	     bool descending, bool raw, GError **err)
+                              bool descending, GError **err) const
 {
-	Xapian::Enquire enq (self->db());
+	Xapian::Enquire enq{store_.database()};
 
-	try {
+        try {
-		if (raw)
+		if (!expr.empty() && expr != R"("")")
-			enq.set_query(Xapian::Query(Xapian::Query(searchexpr)));
+                        enq.set_query(make_query (expr, err));
-		else if (!mu_str_is_empty(searchexpr) &&
-		    g_strcmp0 (searchexpr, "\"\"") != 0) /* NULL or "" or """" */
-			enq.set_query(get_query (self, searchexpr, raw, err));
 		else/* empty or "" means "matchall" */
 			enq.set_query(Xapian::Query::MatchAll);
 	} catch (...) {
-		mu_util_g_set_error (err, MU_ERROR_XAPIAN_QUERY,
+		mu_util_g_set_error (err, MU_ERROR_XAPIAN_QUERY, "parse error in query");
-				     "parse error in query");
 		throw;
 	}
 
 	enq.set_cutoff(0,0);
-	return enq;
+
+        return enq;
 }
 
 /*
- * record all threadids for the messages; also 'orig_set' receives all
+ * record all thread-ids for the messages; also 'orig_set' receives all
  * original matches (a map msgid-->docid), so we can make sure the
  * originals are not seen as 'duplicates' later (when skipping
  * duplicates).  We want to favor the originals over the related
  * messages, when skipping duplicates.
  */
-static GHashTable*
+GHashTable*
-get_thread_ids (MuMsgIter *iter, GHashTable **orig_set)
+Query::Private::find_thread_ids (MuMsgIter *iter, GHashTable **orig_set) const
 {
 	GHashTable *ids;
 
@@ -332,8 +153,8 @@ get_thread_ids (MuMsgIter *iter, GHashTable **orig_set)
 }
 
 
-static Xapian::Query
+Xapian::Query
-get_related_query (MuMsgIter *iter, GHashTable **orig_set)
+Query::Private::make_related_query (MuMsgIter *iter, GHashTable **orig_set) const
 {
 	GHashTable *hash;
 	GList *id_list, *cur;
@@ -343,7 +164,7 @@ get_related_query (MuMsgIter *iter, GHashTable **orig_set)
 
 	/* orig_set receives the hash msgid->docid of the set of
 	 * original matches */
-	hash = get_thread_ids (iter, orig_set);
+	hash = find_thread_ids (iter, orig_set);
 	/* id_list now gets a list of all thread-ids seen in the query
 	 * results; either in the Message-Id field or in
 	 * References. */
@@ -363,18 +184,18 @@ get_related_query (MuMsgIter *iter, GHashTable **orig_set)
 }
 
 
-static void
+void
-get_related_messages (MuQuery *self, MuMsgIter **iter, int maxnum,
+Query::Private::find_related_messages (MuMsgIter **iter, int maxnum,
-                      MuMsgFieldId sortfieldid, MuQueryFlags flags,
+                                       MuMsgFieldId sortfieldid, Query::Flags flags,
-                      Xapian::Query orig_query)
+                                       Xapian::Query orig_query) const
 {
 	GHashTable *orig_set;
-	Xapian::Enquire enq (self->db());
+	Xapian::Enquire enq{store_.database()};
 	MuMsgIter *rel_iter;
-	const bool inc_related = flags & MU_QUERY_FLAG_INCLUDE_RELATED;
+	const bool inc_related{any_of(flags & Query::Flags::IncludeRelated)};
 
 	orig_set = NULL;
-	Xapian::Query new_query = get_related_query (*iter, &orig_set);
+	Xapian::Query new_query{make_related_query (*iter, &orig_set)};
 	/* If related message are not desired, filter out messages which would not
 	   have matched the original query.
 	 */
@@ -402,25 +223,28 @@ get_related_messages (MuQuery *self, MuMsgIter **iter, int maxnum,
 	*iter = rel_iter;
 }
 
+Query::Query(const Store& store):
+        priv_{std::make_unique<Private>(store)}
+{}
+
+Query::Query(Query&& other) = default;
+
+Query::~Query() = default;
+
 
 MuMsgIter*
-mu_query_run (MuQuery *self, const char *searchexpr, MuMsgFieldId sortfieldid,
+Query::run (const std::string& expr, MuMsgFieldId sortfieldid, Query::Flags flags,
-	      int maxnum, MuQueryFlags flags, GError **err)
+            size_t maxnum, GError **err) const
 {
-	g_return_val_if_fail (self, NULL);
-	g_return_val_if_fail (searchexpr, NULL);
 	g_return_val_if_fail (mu_msg_field_id_is_valid (sortfieldid)	||
 			      sortfieldid    == MU_MSG_FIELD_ID_NONE,
 			      NULL);
 	try {
 		MuMsgIter *iter;
-		MuQueryFlags first_flags;
+		const bool threads     = any_of(flags & Flags::Threading);
-		const bool threads     = flags & MU_QUERY_FLAG_THREADS;
+		const bool inc_related = any_of(flags & Flags::IncludeRelated);
-		const bool inc_related = flags & MU_QUERY_FLAG_INCLUDE_RELATED;
+		const bool descending  = any_of(flags & Flags::Descending);
-		const bool descending  = flags & MU_QUERY_FLAG_DESCENDING;
+		Xapian::Enquire enq (priv_->make_enquire(expr, sortfieldid, descending, err));
-		const bool raw         = flags & MU_QUERY_FLAG_RAW;
-		Xapian::Enquire enq (get_enquire(self, searchexpr, sortfieldid,
-						 descending, raw, err));
 
 		/* when we're doing a 'include-related query', wea're actually
 		 * doing /two/ queries; one to get the initial matches, and
@@ -429,12 +253,13 @@ mu_query_run (MuQuery *self, const char *searchexpr, MuMsgFieldId sortfieldid,
 		 */
 
 		/* get the 'real' maxnum if it was specified as < 0 */
-		maxnum = maxnum < 0 ? self->db().get_doccount() : maxnum;
+		maxnum = maxnum == 0 ? priv_->store_.size(): maxnum;
 		/* Calculating threads involves two queries, so do the calculation only in
 		 * the second query instead of in both.
 		 */
+                Query::Flags first_flags{};
 		if (threads)
-			first_flags = (MuQueryFlags)(flags & ~MU_QUERY_FLAG_THREADS);
+			first_flags = flags & ~Flags::Threading;
 		else
 			first_flags = flags;
 		/* Perform the initial query, returning up to max num results.
@@ -454,68 +279,47 @@ mu_query_run (MuQuery *self, const char *searchexpr, MuMsgFieldId sortfieldid,
 		 * the undesired related messages later.
 		 */
 		if(threads||inc_related)
-			get_related_messages (self, &iter, maxnum, sortfieldid, flags,
+			priv_->find_related_messages (&iter, maxnum, sortfieldid, flags,
-			                      enq.get_query());
+                                                      enq.get_query());
 
-		if (err && *err && (*err)->code == MU_ERROR_XAPIAN_MODIFIED) {
+                return iter;
-			g_clear_error (err);
-			return try_requery (self, searchexpr, sortfieldid,
-					    maxnum, flags, err);
-		} else
-			return iter;
 
 	} MU_XAPIAN_CATCH_BLOCK_G_ERROR_RETURN (err, MU_ERROR_XAPIAN, 0);
 }
 
 
 size_t
-mu_query_count_run (MuQuery *self, const char *searchexpr) try
+Query::count (const std::string& expr) const try
 {
-	g_return_val_if_fail (self, 0);
+        const auto enq{priv_->make_enquire(expr, MU_MSG_FIELD_ID_NONE, false, nullptr)};
-	g_return_val_if_fail (searchexpr, 0);
+        auto mset{enq.get_mset(0, priv_->store_.size())};
-
-        const auto enq{get_enquire(self, searchexpr,MU_MSG_FIELD_ID_NONE, false, false, NULL)};
-        auto mset(enq.get_mset(0, self->db().get_doccount()));
         mset.fetch();
 
         return mset.size();
 
-} MU_XAPIAN_CATCH_BLOCK_RETURN (0);
+}MU_XAPIAN_CATCH_BLOCK_RETURN (0);
 
-char*
+
-mu_query_internal_xapian (MuQuery *self, const char *searchexpr, GError **err)
+
+std::string
+Query::parse(const std::string& expr, bool xapian) const try
 {
-	g_return_val_if_fail (self, NULL);
+        if (xapian) {
-	g_return_val_if_fail (searchexpr, NULL);
+                GError *err{};
-
+                const auto descr{priv_->make_query(expr, &err).get_description()};
-	try {
+                if (err) {
-		Xapian::Query query (get_query(self, searchexpr, false, err));
+                        g_warning ("query error: %s", err->message);
-		return g_strdup(query.get_description().c_str());
+                        g_clear_error(&err);
-
+                }
-	} MU_XAPIAN_CATCH_BLOCK_RETURN(NULL);
+                return descr;
-}
+        } else {
-
-
-char*
-mu_query_internal (MuQuery *self, const char *searchexpr,
-		   gboolean warn, GError **err)
-{
-	g_return_val_if_fail (self, NULL);
-	g_return_val_if_fail (searchexpr, NULL);
-
-	try {
 		Mu::WarningVec warns;
-		const auto tree = Mu::parse (searchexpr, warns,
+		const auto tree = priv_->parser_.parse (expr, warns);
-					      std::make_unique<MuProc>(self->db()));
+                for (auto&& w: warns)
-		std::stringstream ss;
+                        g_warning ("query error: %s", to_string(w).c_str());
-		ss << tree;
 
-		if (warn) {
+                return to_string(tree);
-			for (auto&& w: warns)
-				std::cerr << w << std::endl;
-		}
 
-		return g_strdup(ss.str().c_str());
+	}
 
-	} MU_XAPIAN_CATCH_BLOCK_RETURN(NULL);
+} MU_XAPIAN_CATCH_BLOCK_RETURN("");
-}

lib/mu-query.h

@@ -1,136 +0,0 @@
-/*
-** Copyright (C) 2008-2017 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
-**
-** This program is free software; you can redistribute it and/or modify
-** it under the terms of the GNU General Public License as published by
-** the Free Software Foundation; either version 3 of the License, or
-** (at your option) any later version.
-**
-** This program is distributed in the hope that it will be useful,
-** but WITHOUT ANY WARRANTY; without even the implied warranty of
-** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-** GNU General Public License for more details.
-**
-** You should have received a copy of the GNU General Public License
-** along with this program; if not, write to the Free Software Foundation,
-** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-**
-*/
-
-#ifndef __MU_QUERY_H__
-#define __MU_QUERY_H__
-
-#include <glib.h>
-#include <mu-store.hh>
-#include <mu-msg-iter.h>
-#include <utils/mu-util.h>
-
-G_BEGIN_DECLS
-
-struct _MuQuery;
-typedef struct _MuQuery MuQuery;
-
-/**
- * create a new MuQuery instance.
- *
- * @param store a MuStore object
- * @param err receives error information (if there is any); if
- * function returns non-NULL, err will _not_be set. err can be NULL
- * possible errors (err->code) are MU_ERROR_XAPIAN_DIR and
- * MU_ERROR_XAPIAN_NOT_UPTODATE
- *
- * @return a new MuQuery instance, or NULL in case of error.
- * when the instance is no longer needed, use mu_query_destroy
- * to free it
- */
-MuQuery* mu_query_new  (MuStore *store, GError **err)
-      G_GNUC_MALLOC G_GNUC_WARN_UNUSED_RESULT;
-
-/**
- * destroy the MuQuery instance
- *
- * @param self a MuQuery instance, or NULL
- */
-void mu_query_destroy  (MuQuery *self);
-
-
-typedef enum {
-	MU_QUERY_FLAG_NONE            = 0 << 0, /**< no flags */
-	MU_QUERY_FLAG_DESCENDING      = 1 << 0,	/**< sort z->a */
-	MU_QUERY_FLAG_SKIP_UNREADABLE = 1 << 1,	/**< skip unreadable msgs */
-	MU_QUERY_FLAG_SKIP_DUPS       = 1 << 2,	/**< skip duplicate msgs */
-	MU_QUERY_FLAG_INCLUDE_RELATED = 1 << 3,	/**< include related msgs */
-	MU_QUERY_FLAG_THREADS         = 1 << 4, /**< calculate threading info */
-	MU_QUERY_FLAG_RAW             = 1 << 5  /**< don't parse the query */
-} MuQueryFlags;
-
-/**
- * run a Xapian query; for the syntax, please refer to the mu-query
- * manpage
- *
- * @param self a valid MuQuery instance
- * @param expr the search expression; use "" to match all messages
- * @param sortfield the field id to sort by or MU_MSG_FIELD_ID_NONE if
- * sorting is not desired
- * @param maxnum maximum number of search results to return, or <= 0 for
- * unlimited
- * @param flags bitwise OR'd flags to influence the query (see MuQueryFlags)
- * @param err receives error information (if there is any); if
- * function returns non-NULL, err will _not_be set. err can be NULL
- * possible error (err->code) is MU_ERROR_QUERY,
- *
- * @return a MuMsgIter instance you can iterate over, or NULL in
- * case of error
- */
-MuMsgIter* mu_query_run (MuQuery *self, const char* expr,
-			 MuMsgFieldId sortfieldid, int maxnum,
-			 MuQueryFlags flags, GError **err)
-    G_GNUC_MALLOC G_GNUC_WARN_UNUSED_RESULT;
-
-
-/**
- * run a Xapian query to count the number of matches; for the syntax, please
- * refer to the mu-query manpage
- *
- * @param self a valid MuQuery instance
- * @param expr the search expression; use "" to match all messages
- *
- * @return the number of matches
- */
-size_t mu_query_count_run (MuQuery *self, const char *searchexpr);
-
-/**
- * get Xapian's internal string representation of the query
- *
- * @param self a MuQuery instance
- * @param searchexpr a xapian search expression
- * @param warn print warnings to stderr
- * @param err receives error information (if there is any); if
- * function returns non-NULL, err will _not_be set. err can be NULL
- *
- * @return the string representation of the xapian query, or NULL in case of
- * error; free the returned value with g_free
- */
-char* mu_query_internal (MuQuery *self, const char *searchexpr,
-			 gboolean warn, GError **err)
-    G_GNUC_MALLOC G_GNUC_WARN_UNUSED_RESULT;
-
-/**
- * get Xapian's internal string representation of the query
- *
- * @param self a MuQuery instance
- * @param searchexpr a xapian search expression
- * @param err receives error information (if there is any); if
- * function returns non-NULL, err will _not_be set. err can be NULL
- *
- * @return the string representation of the xapian query, or NULL in case of
- * error; free the returned value with g_free
- */
-char* mu_query_internal_xapian (MuQuery *self, const char* searchexpr,
-				GError **err)
-    G_GNUC_MALLOC G_GNUC_WARN_UNUSED_RESULT;
-
-
-G_END_DECLS
-
-#endif /*__MU_QUERY_H__*/

lib/mu-query.hh

@@ -0,0 +1,120 @@
+/*
+** Copyright (C) 2008-2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify
+** it under the terms of the GNU General Public License as published by
+** the Free Software Foundation; either version 3 of the License, or
+** (at your option) any later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+**
+*/
+
+#ifndef __MU_QUERY_HH__
+#define __MU_QUERY_HH__
+
+#include <memory>
+
+#include <glib.h>
+#include <mu-store.hh>
+#include <mu-msg-iter.h>
+#include <utils/mu-utils.hh>
+
+namespace Mu {
+
+class Query {
+public:
+        /**
+         * Construct a new Query instance.
+         *
+         * @param store a MuStore object
+         */
+        Query (const Store& store);
+        /**
+         * DTOR
+         *
+         */
+        ~Query ();
+
+
+        /**
+         * Move CTOR
+         *
+         * @param other
+         */
+        Query(Query&& other);
+
+
+       enum struct Flags {
+                None           = 0, /**< no flags */
+                Descending     = 1 << 0, /**< sort z->a */
+                SkipUnreadable = 1 << 1, /**< skip unreadable msgs */
+                SkipDups       = 1 << 2, /**< skip duplicate msgs */
+                IncludeRelated = 1 << 3, /**< include related msgs */
+                Threading      = 1 << 4, /**< calculate threading info */
+        };
+
+
+        /**
+         * run a query; for the syntax, please refer to the mu-query manpage
+         *
+         * @param expr the search expression; use "" to match all messages
+         * @param sortfield the field id to sort by or MU_MSG_FIELD_ID_NONE if
+         * sorting is not desired
+         * @param flags bitwise OR'd flags to influence the query (see MuQueryFlags)
+         * @param maxnum maximum number of search results to return, or 0 for
+         * unlimited
+         * @param err receives error information (if there is any); if
+         * function returns non-NULL, err will _not_be set. err can be NULL
+         * possible error (err->code) is MU_ERROR_QUERY,
+         *
+         * @return a MuMsgIter instance you can iterate over, or NULL in
+         * case of error
+         */
+        MuMsgIter* run (const std::string& expr="",
+                        MuMsgFieldId sortfieldid=MU_MSG_FIELD_ID_NONE,
+                        Flags flags=Flags::None,
+                        size_t maxnum=0,
+                        GError **err=nullptr) const
+                G_GNUC_MALLOC G_GNUC_WARN_UNUSED_RESULT;
+
+
+        /**
+         * run a Xapian query to count the number of matches; for the syntax, please
+         * refer to the mu-query manpage
+         *
+         * @param expr the search expression; use "" to match all messages
+         *
+         * @return the number of matches
+         */
+        size_t count (const std::string& expr="") const;
+
+        /**
+         * For debugging, get the internal string representation of the parsed
+         * query
+         *
+         * @param expr a xapian search expression
+         * @param xapian if true, show Xapian's internal representation,
+         * otherwise, mu's.
+
+         * @return the string representation of the query
+         */
+        std::string parse (const std::string& expr, bool xapian) const;
+
+private:
+        struct Private;
+        std::unique_ptr<Private> priv_;
+
+};
+MU_ENABLE_BITOPS(Query::Flags);
+
+}
+
+#endif /*__MU_QUERY_HH__*/

lib/mu-server.cc

@@ -19,6 +19,7 @@
 
 #include "config.h"
 
+#include "mu-msg-fields.h"
 #include "mu-server.hh"
 
 #include <iostream>
@@ -36,7 +37,7 @@
 #include "mu-msg.h"
 #include "mu-runtime.h"
 #include "mu-maildir.h"
-#include "mu-query.h"
+#include "mu-query.hh"
 #include "index/mu-indexer.hh"
 #include "mu-store.hh"
 #include "mu-msg-part.h"
@@ -59,21 +60,12 @@ struct Server::Private {
                 store_{store},
                 output_{output},
                 command_map_{make_command_map()},
-                query_{make_query(store_)},
+                query_{store_},
-                keep_going_{true} {
+                keep_going_{true} {}
-                if (!query_)
-                        throw Error(Error::Code::Query, "failed to create server");
-        }
-
-        ~Private() {
-                g_clear_pointer(&query_, mu_query_destroy);
-        }
-
         //
         // construction helpers
         //
         CommandMap make_command_map();
-        MuQuery* make_query(Store& store) const;
 
         //
         // acccessors
@@ -81,7 +73,7 @@ struct Server::Private {
         const Store& store() const            { return store_; }
         Indexer& indexer()                    { return store().indexer(); }
         const CommandMap& command_map() const { return command_map_; }
-        MuQuery* query()                      { return query_; }
+        const Query& query() const            { return query_; }
 
         //
         // invoke
@@ -122,24 +114,11 @@ private:
         Store&           store_;
         Server::Output   output_;
         const CommandMap command_map_;
-        MuQuery          *query_{};
+        const Query      query_;
 
         std::atomic<bool> keep_going_{};
 };
 
-MuQuery*
-Server::Private::make_query (Store& store) const
-{
-        GError *gerr{};
-        auto q{mu_query_new (reinterpret_cast<MuStore*>(&store), &gerr)};
-        if (!q) {
-                g_critical("failed to create query: %s",
-                           gerr ? gerr->message : "something went wrong");
-                g_clear_error(&gerr);
-        }
-
-        return q;
-}
 
 CommandMap
 Server::Private::make_command_map ()
@@ -624,22 +603,22 @@ Server::Private::extract_handler (const Parameters& params)
 
 /* get a *list* of all messages with the given message id */
 static std::vector<DocId>
-docids_for_msgid (MuQuery *query, const std::string& msgid, size_t max=100)
+docids_for_msgid (const Query& q, const std::string& msgid, size_t max=100)
 {
         if (msgid.size() > MU_STORE_MAX_TERM_LENGTH - 1) {
                 throw Error(Error::Code::InvalidArgument,
                                   "invalid message-id '%s'", msgid.c_str());
         }
 
-        const auto xprefix{mu_msg_field_xapian_prefix(MU_MSG_FIELD_ID_MSGID)};
+        const auto xprefix{mu_msg_field_shortcut(MU_MSG_FIELD_ID_MSGID)};
         /*XXX this is a bit dodgy */
         auto tmp{g_ascii_strdown(msgid.c_str(), -1)};
-        auto rawq{g_strdup_printf("%c%s", xprefix, tmp)};
+        auto expr{g_strdup_printf("%c:%s", xprefix, tmp)};
         g_free(tmp);
 
         GError *gerr{};
-        auto iter{mu_query_run (query, rawq, MU_MSG_FIELD_ID_NONE, max, MU_QUERY_FLAG_RAW, &gerr)};
+        auto iter{q.run(expr , MU_MSG_FIELD_ID_NONE, Query::Flags::None, max)};
-        g_free (rawq);
+        g_free (expr);
         if (!iter)
                 throw Error(Error::Code::Store, &gerr, "failed to run msgid-query");
         if (mu_msg_iter_is_done (iter))
@@ -680,7 +659,7 @@ path_from_docid (const Store& store, unsigned docid)
 
 
 static std::vector<DocId>
-determine_docids (MuQuery *query, const Parameters& params)
+determine_docids (const Query& q, const Parameters& params)
 {
         auto docid{get_int_or(params, ":docid", 0)};
         const auto msgid{get_string_or(params, ":msgid")};
@@ -692,7 +671,7 @@ determine_docids (MuQuery *query, const Parameters& params)
         if (docid != 0)
                 return { (unsigned)docid };
         else
-                return docids_for_msgid (query, msgid.c_str());
+                return docids_for_msgid (q, msgid.c_str());
 }
 
 
@@ -739,19 +718,18 @@ Server::Private::find_handler (const Parameters& params)
                                         sortfieldstr.c_str()};
         }
 
-        int qflags{MU_QUERY_FLAG_NONE/*UNREADABLE*/};
+        auto qflags{Query::Flags::None};
         if (descending)
-                qflags |= MU_QUERY_FLAG_DESCENDING;
+                qflags |= Query::Flags::Descending;
         if (skip_dups)
-                qflags |= MU_QUERY_FLAG_SKIP_DUPS;
+                qflags |= Query::Flags::SkipDups;
         if (include_related)
-                qflags |= MU_QUERY_FLAG_INCLUDE_RELATED;
+                qflags |= Query::Flags::IncludeRelated;
         if (threads)
-                qflags |= MU_QUERY_FLAG_THREADS;
+                qflags |= Query::Flags::Threading;
 
         GError *gerr{};
-        auto miter{mu_query_run(query(), q.c_str(), sort_field, maxnum,
+        auto miter{query().run(q, sort_field, qflags, maxnum, &gerr)};
-                                (MuQueryFlags)qflags, &gerr)};
         if (!miter)
                 throw Error(Error::Code::Query, &gerr, "failed to run query");
 
@@ -1023,9 +1001,9 @@ Server::Private::ping_handler (const Parameters& params)
         Sexp::List qresults;
         for (auto&& q: queries) {
 
-                const auto count{mu_query_count_run (query(), q.c_str())};
+                const auto count{query().count(q)};
                 const auto unreadq{format("flag:unread AND (%s)", q.c_str())};
-                const auto unread{mu_query_count_run (query(), unreadq.c_str())};
+                const auto unread{query().count(unreadq)};
 
                 Sexp::List lst;
                 lst.add_prop(":query",  Sexp::make_string(q));

lib/mu-store.cc

@@ -20,6 +20,7 @@
 #include "config.h"
 
 #include <chrono>
+#include <memory>
 #include <mutex>
 #include <array>
 #include <cstdlib>
@@ -112,43 +113,42 @@ struct Store::Private {
         enum struct XapianOpts {ReadOnly, Open, CreateOverwrite };
 
         Private (const std::string& path, bool readonly):
-                db_{make_xapian(path, readonly ? XapianOpts::ReadOnly : XapianOpts::Open)},
+                read_only_{readonly},
+                db_{make_xapian_db(path, read_only_ ? XapianOpts::ReadOnly : XapianOpts::Open)},
                 mdata_{make_metadata(path)},
-                contacts_{db()->get_metadata(ContactsKey), mdata_.personal_addresses} {
+                contacts_{db().get_metadata(ContactsKey), mdata_.personal_addresses} {
 
                 if (!readonly)
-                        wdb()->begin_transaction();
+                        writable_db().begin_transaction();
         }
 
         Private (const std::string& path, const std::string& root_maildir,
                  const StringVec& personal_addresses, const Store::Config& conf):
-                db_{make_xapian(path, XapianOpts::CreateOverwrite)},
+                read_only_{false},
+                db_{make_xapian_db(path, XapianOpts::CreateOverwrite)},
                 mdata_{init_metadata(conf, path, root_maildir, personal_addresses)},
                 contacts_{"", mdata_.personal_addresses} {
 
-                wdb()->begin_transaction();
+                writable_db().begin_transaction();
         }
 
         ~Private() try {
-                LOCKED;
                 g_debug("closing store @ %s", mdata_.database_path.c_str());
-                if (wdb()) {
+                if (!read_only_) {
-                        wdb()->set_metadata (ContactsKey, contacts_.serialize());
+                        writable_db().set_metadata (ContactsKey, contacts_.serialize());
                         commit();
                 }
         } MU_XAPIAN_CATCH_BLOCK;
 
-        std::shared_ptr<Xapian::Database> make_xapian (const std::string db_path,
+        std::unique_ptr<Xapian::Database> make_xapian_db (const std::string db_path, XapianOpts opts) try {
-                                                       XapianOpts opts) try {
+
                 switch (opts) {
                 case XapianOpts::ReadOnly:
-                        return std::make_shared<Xapian::Database>(db_path);
+                        return std::make_unique<Xapian::Database>(db_path);
                 case XapianOpts::Open:
-                        return std::make_shared<Xapian::WritableDatabase>(
+                        return std::make_unique<Xapian::WritableDatabase>(db_path, Xapian::DB_OPEN);
-                                db_path, Xapian::DB_OPEN);
                 case XapianOpts::CreateOverwrite:
-                        return std::make_shared<Xapian::WritableDatabase>(
+                        return std::make_unique<Xapian::WritableDatabase>(db_path, Xapian::DB_CREATE_OR_OVERWRITE);
-                                db_path, Xapian::DB_CREATE_OR_OVERWRITE);
                 default:
                         throw std::logic_error ("invalid xapian options");
                 }
@@ -162,22 +162,12 @@ struct Store::Private {
                                 db_path.c_str());
         }
 
-        std::shared_ptr<Xapian::Database> db() const {
+        const Xapian::Database& db() const { return *db_.get(); }
-                if (!db_)
-                        throw Mu::Error(Error::Code::NotFound, "no database found");
-                return db_;
-        }
 
-        std::shared_ptr<Xapian::WritableDatabase> wdb() const {
+        Xapian::WritableDatabase&  writable_db() {
-                return std::dynamic_pointer_cast<Xapian::WritableDatabase>(db_);
+                if (read_only_)
-        }
-
-        std::shared_ptr<Xapian::WritableDatabase> writable_db() const {
-                auto w_db{wdb()};
-                if (!w_db)
                         throw Mu::Error(Error::Code::AccessDenied, "database is read-only");
-                else
+                return dynamic_cast<Xapian::WritableDatabase&>(*db_.get());
-                        return w_db;
         }
 
         void dirty () try {
@@ -188,35 +178,35 @@ struct Store::Private {
         void commit () try {
                 g_debug("committing %zu modification(s)", dirtiness_);
                 dirtiness_      = 0;
-                wdb()->commit_transaction();
+                writable_db().commit_transaction();
-                wdb()->begin_transaction();
+                writable_db().begin_transaction();
         } MU_XAPIAN_CATCH_BLOCK;
 
         void add_synonyms () {
                 mu_flags_foreach ((MuFlagsForeachFunc)add_synonym_for_flag,
-                                  writable_db().get());
+                                  &writable_db());
                 mu_msg_prio_foreach ((MuMsgPrioForeachFunc)add_synonym_for_prio,
-                                     writable_db().get());
+                                     &writable_db());
         }
 
         time_t metadata_time_t (const std::string& key) const {
-                const auto ts = db()->get_metadata(key);
+                const auto ts = db().get_metadata(key);
-                return (time_t)atoll(db()->get_metadata(key).c_str());
+                return (time_t)atoll(db().get_metadata(key).c_str());
         }
 
         Store::Metadata make_metadata(const std::string& db_path) {
                 Store::Metadata mdata;
 
                 mdata.database_path       = db_path;
-                mdata.schema_version      = db()->get_metadata(SchemaVersionKey);
+                mdata.schema_version      = db().get_metadata(SchemaVersionKey);
-                mdata.created             = ::atoll(db()->get_metadata(CreatedKey).c_str());
+                mdata.created             = ::atoll(db().get_metadata(CreatedKey).c_str());
-                mdata.read_only           = !wdb();
+                mdata.read_only           = read_only_;
 
-                mdata.batch_size          = ::atoll(db()->get_metadata(BatchSizeKey).c_str());
+                mdata.batch_size          = ::atoll(db().get_metadata(BatchSizeKey).c_str());
-                mdata.max_message_size    = ::atoll(db()->get_metadata(MaxMessageSizeKey).c_str());
+                mdata.max_message_size    = ::atoll(db().get_metadata(MaxMessageSizeKey).c_str());
 
-                mdata.root_maildir       = db()->get_metadata(RootMaildirKey);
+                mdata.root_maildir       = db().get_metadata(RootMaildirKey);
-                mdata.personal_addresses = Mu::split(db()->get_metadata(PersonalAddressesKey),",");
+                mdata.personal_addresses = Mu::split(db().get_metadata(PersonalAddressesKey),",");
 
                 return mdata;
         }
@@ -225,17 +215,17 @@ struct Store::Private {
                                       const std::string& path, const std::string& root_maildir,
                                       const StringVec& personal_addresses) {
 
-                wdb()->set_metadata(SchemaVersionKey, ExpectedSchemaVersion);
+                writable_db().set_metadata(SchemaVersionKey, ExpectedSchemaVersion);
-                wdb()->set_metadata(CreatedKey, Mu::format("%" PRId64, (int64_t)::time({})));
+                writable_db().set_metadata(CreatedKey, Mu::format("%" PRId64, (int64_t)::time({})));
 
                 const size_t batch_size = conf.batch_size ? conf.batch_size : DefaultBatchSize;
-                wdb()->set_metadata(BatchSizeKey, Mu::format("%zu", batch_size));
+                writable_db().set_metadata(BatchSizeKey, Mu::format("%zu", batch_size));
 
                 const size_t max_msg_size = conf.max_message_size ?
                         conf.max_message_size : DefaultMaxMessageSize;
-                wdb()->set_metadata(MaxMessageSizeKey, Mu::format("%zu", max_msg_size));
+                writable_db().set_metadata(MaxMessageSizeKey, Mu::format("%zu", max_msg_size));
 
-                wdb()->set_metadata(RootMaildirKey, root_maildir);
+                writable_db().set_metadata(RootMaildirKey, root_maildir);
 
                 std::string addrs;
                 for (const auto& addr : personal_addresses) { // _very_ minimal check.
@@ -244,15 +234,17 @@ struct Store::Private {
                                                 "e-mail address '%s' contains comma", addr.c_str());
                         addrs += (addrs.empty() ? "": ",") + addr;
                 }
-                wdb()->set_metadata (PersonalAddressesKey, addrs);
+                writable_db().set_metadata (PersonalAddressesKey, addrs);
 
                 return make_metadata(path);
         }
 
-        std::shared_ptr<Xapian::Database> db_;
+        const bool               read_only_{};
-        const Store::Metadata             mdata_;
+        std::unique_ptr<Xapian::Database> db_;
-        Contacts                          contacts_;
+
-        std::unique_ptr<Indexer>          indexer_;
+        const Store::Metadata    mdata_;
+        Contacts                 contacts_;
+        std::unique_ptr<Indexer> indexer_;
 
         std::atomic<bool>                 in_transaction_{};
         std::mutex                        lock_;
@@ -311,6 +303,20 @@ Store::contacts() const
         return priv_->contacts_;
 }
 
+
+const Xapian::Database&
+Store::database() const
+{
+        return priv_->db();
+
+}
+
+Xapian::WritableDatabase&
+Store::writable_database()
+{
+        return priv_->writable_db();
+}
+
 Indexer&
 Store::indexer()
 {
@@ -328,7 +334,7 @@ std::size_t
 Store::size() const
 {
         LOCKED;
-        return priv_->db()->get_doccount();
+        return priv_->db().get_doccount();
 }
 
 bool
@@ -419,9 +425,7 @@ Store::remove_message (const std::string& path)
 
 	try {
 		const std::string term{(get_uid_term(path.c_str()))};
-                auto wdb{priv()->wdb()};
+                priv()->writable_db().delete_document(term);
-
-		wdb->delete_document (term);
 
 	} MU_XAPIAN_CATCH_BLOCK_RETURN (false);
 
@@ -439,7 +443,7 @@ Store::remove_messages (const std::vector<Store::Id>& ids)
 
 	try {
                 for (auto&& id: ids) {
-                        priv()->wdb()->delete_document(id);
+                        priv()->writable_db().delete_document(id);
                         priv_->dirty();
                 }
 
@@ -451,7 +455,7 @@ Store::dirstamp (const std::string& path) const
 {
         LOCKED;
 
-        const auto ts = priv_->db()->get_metadata(path);
+        const auto ts = priv_->db().get_metadata(path);
         if (ts.empty())
                 return 0;
         else
@@ -466,7 +470,7 @@ Store::set_dirstamp (const std::string& path, time_t tstamp)
         std::array<char, 2*sizeof(tstamp)+1> data{};
         const std::size_t len = g_snprintf (data.data(), data.size(), "%zx", tstamp);
 
-        priv_->writable_db()->set_metadata(path, std::string{data.data(), len});
+        priv_->writable_db().set_metadata(path, std::string{data.data(), len});
         priv_->dirty();
 }
 
@@ -477,7 +481,7 @@ Store::find_message (unsigned docid) const
         LOCKED;
 
 	try {
-		Xapian::Document *doc{new Xapian::Document{priv_->db()->get_document (docid)}};
+		Xapian::Document *doc{new Xapian::Document{priv_->db().get_document (docid)}};
                 GError *gerr{};
                 auto msg{mu_msg_new_from_doc (reinterpret_cast<XapianDocument*>(doc), &gerr)};
                 if (!msg) {
@@ -499,26 +503,26 @@ Store::contains_message (const std::string& path) const
 
 	try {
 		const std::string term (get_uid_term(path.c_str()));
- 		return priv_->db()->term_exists (term);
+ 		return priv_->db().term_exists (term);
 
 	} MU_XAPIAN_CATCH_BLOCK_RETURN(false);
 }
 
 
 std::size_t
-Store::for_each (Store::ForEachFunc func)
+Store::for_each_message_path (Store::ForEachMessageFunc func) const
 {
         LOCKED;
 
         size_t n{};
 
 	try {
-		Xapian::Enquire enq (*priv_->db().get());
+		Xapian::Enquire enq{priv_->db()};
 
 		enq.set_query  (Xapian::Query::MatchAll);
 		enq.set_cutoff (0,0);
 
-		Xapian::MSet matches(enq.get_mset (0, priv_->db()->get_doccount()));
+		Xapian::MSet matches(enq.get_mset (0, priv_->db().get_doccount()));
 
                 for (auto&& it = matches.begin(); it != matches.end(); ++it, ++n)
                         if (!func (*it, it.get_document().get_value(MU_MSG_FIELD_ID_PATH)))
@@ -529,6 +533,53 @@ Store::for_each (Store::ForEachFunc func)
         return n;
 }
 
+static MuMsgFieldId
+field_id (const std::string& field)
+{
+
+        if (field.empty())
+                return MU_MSG_FIELD_ID_NONE;
+
+        MuMsgFieldId id = mu_msg_field_id_from_name (field.c_str(), FALSE);
+        if (id != MU_MSG_FIELD_ID_NONE)
+                return id;
+        else if (field.length() == 1)
+                return mu_msg_field_id_from_shortcut (field[0], FALSE);
+        else
+                return MU_MSG_FIELD_ID_NONE;
+}
+
+
+
+std::size_t
+Store::for_each_term (const std::string& field, Store::ForEachTermFunc func) const
+{
+        LOCKED;
+
+        size_t n{};
+
+	try {
+                const auto id = field_id (field.c_str());
+                if (id == MU_MSG_FIELD_ID_NONE)
+                        return {};
+
+                char pfx[] = {  mu_msg_field_xapian_prefix(id), '\0' };
+
+                std::vector<std::string> terms;
+                for (auto it = priv_->db().allterms_begin(pfx);
+                     it != priv_->db().allterms_end(pfx); ++it) {
+                        if (!func(*it))
+                                break;
+                }
+
+	} MU_XAPIAN_CATCH_BLOCK;
+
+        return n;
+}
+
+
+
+
 void
 Store::commit () try
 {
@@ -643,13 +694,6 @@ mu_store_schema_version (const MuStore *store)
 	return self(store)->metadata().schema_version.c_str();
 }
 
-XapianDatabase*
-mu_store_get_read_only_database (MuStore *store)
-{
-	g_return_val_if_fail (store, NULL);
-	return (XapianDatabase*)self(store)->priv()->db().get();
-}
-
 static void
 add_terms_values_date (Xapian::Document& doc, MuMsg *msg, MuMsgFieldId mfid)
 {
@@ -1085,8 +1129,7 @@ new_doc_from_message (MuStore *store, MuMsg *msg)
 }
 
 static void
-update_threading_info (Xapian::WritableDatabase* db,
+update_threading_info (MuMsg *msg, Xapian::Document& doc)
-		       MuMsg *msg, Xapian::Document& doc)
 {
 	const GSList *refs;
 
@@ -1119,18 +1162,18 @@ add_or_update_msg (MuStore *store, unsigned docid, MuMsg *msg, GError **err)
 		const std::string term (get_uid_term (mu_msg_get_path(msg)));
 
                 auto self = mutable_self(store);
-                auto wdb  = self->priv()->wdb();
+                auto wdb  = self->priv()->writable_db();
 
 		add_term (doc, term);
 
 		// update the threading info if this message has a message id
 		if (mu_msg_get_msgid (msg))
-			update_threading_info (wdb.get(), msg, doc);
+			update_threading_info (msg, doc);
 
 		if (docid == 0)
-			id = wdb->replace_document (term, doc);
+			id = wdb.replace_document (term, doc);
 		else {
-			wdb->replace_document (docid, doc);
+			wdb.replace_document (docid, doc);
 			id = docid;
 		}
 

lib/mu-store.hh

@@ -103,6 +103,23 @@ public:
          */
         const Contacts& contacts() const;
 
+
+        /**
+         * Get the underlying Xapian database for this store.
+         *
+         * @return the database
+         */
+        const Xapian::Database& database() const;
+
+        /**
+         * Get the underlying writable Xapian database for this
+         * store. Throws is this store is not writable.
+         *
+         * @return the writable database
+         */
+        Xapian::WritableDatabase& writable_database();
+
+
         /**
          * Get the Indexer associated with this store. It is an error
          * to call this on a read-only store.
@@ -175,24 +192,44 @@ public:
         bool contains_message (const std::string& path) const;
 
         /**
-         * Prototype for the ForEachFunc
+         * Prototype for the ForEachMessageFunc
          *
          * @param id :t store Id for the message
          * @param path: the absolute path to the message
          *
          * @return true if for_each should continue; false to quit
          */
-        using ForEachFunc = std::function<bool(Id, const std::string&)>;
+        using ForEachMessageFunc = std::function<bool(Id, const std::string&)>;
 
         /**
          * Call @param func for each document in the store. This takes a lock on
          * the store, so the func should _not_ call any other Store:: methods.
          *
-         * @param func a functio
+         * @param func a Callable invoked for each message.
          *
          * @return the number of times func was invoked
          */
-        size_t for_each (ForEachFunc func);
+        size_t for_each_message_path (ForEachMessageFunc func) const;
+
+        /**
+         * Prototype for the ForEachTermFunc
+         *
+         * @param term:
+         *
+         * @return true if for_each should continue; false to quit
+         */
+        using ForEachTermFunc = std::function<bool(const std::string&)>;
+
+        /**
+         * Call @param func for each term for the given field in the store. This
+         * takes a lock on the store, so the func should _not_ call any other
+         * Store:: methods.
+         *
+         * @param func a Callable invoked for each message.
+         *
+         * @return the number of times func was invoked
+         */
+        size_t for_each_term (const std::string& field, ForEachTermFunc func) const;
 
         /**
          * Get the timestamp for some message, or 0 if not found
@@ -303,23 +340,6 @@ MuStore* mu_store_ref (MuStore *store);
  */
 MuStore* mu_store_unref (MuStore *store);
 
-
-/**
- * we need this when using Xapian::(Writable)Database* from C
- */
-typedef gpointer XapianDatabase;
-
-/**
- * get the underlying read-only database object for this store; not that this
- * pointer becomes in valid after mu_store_destroy
- *
- * @param store a valid store
- *
- * @return a Xapian::Database (you'll need to cast in C++), or
- * NULL in case of error.
- */
-XapianDatabase* mu_store_get_read_only_database (MuStore *store);
-
 /**
  * get the version of the xapian database (ie., the version of the
  * 'schema' we are using). If this version != MU_STORE_SCHEMA_VERSION,

lib/mu-tokenizer.hh

@@ -97,7 +97,7 @@ operator<< (std::ostream& os, Token::Type t)
 	case Token::Type::And:      os << "<and>"; break;
 	case Token::Type::Or:       os << "<or>"; break;
 	case Token::Type::Xor:      os << "<xor>"; break;
-
+        case Token::Type::Empty:    os << "<empty>"; break;
 	default:		// can't happen, but pacify compiler
 		throw std::runtime_error ("<<bug>>");
 	}

lib/mu-tree.hh

@@ -24,7 +24,7 @@
 #include <string>
 #include <iostream>
 
-#include <query/mu-data.hh>
+#include <mu-data.hh>
 #include <utils/mu-error.hh>
 
 namespace Mu {

lib/mu-xapian.cc

@@ -32,6 +32,8 @@ xapian_query_op (const Mu::Tree& tree)
 {
 	Xapian::Query::op op;
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored   "-Wswitch-enum"
 	switch (tree.node.type) {
 	case Node::Type::OpNot: // OpNot x ::= <all> AND NOT x
 		  if (tree.children.size() != 1)
@@ -45,7 +47,7 @@ xapian_query_op (const Mu::Tree& tree)
 	case Node::Type::OpAndNot: op = Xapian::Query::OP_AND_NOT; break;
 	default: throw Mu::Error (Error::Code::Internal, "invalid op");	// bug
 	}
-
+#pragma GCC diagnostic pop
 	std::vector<Xapian::Query> childvec;
 	for (const auto& subtree: tree.children)
 		childvec.emplace_back(xapian_query(subtree));
@@ -97,6 +99,8 @@ xapian_query_range (const Mu::Tree& tree)
 Xapian::Query
 Mu::xapian_query (const Mu::Tree& tree)
 {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored   "-Wswitch-enum"
 	switch (tree.node.type) {
 	case Node::Type::Empty:
 		return Xapian::Query();
@@ -113,4 +117,5 @@ Mu::xapian_query (const Mu::Tree& tree)
 	default:
                 throw Mu::Error (Error::Code::Internal, "invalid query");	// bug
 	}
+#pragma GCC diagnostic pop
 }

lib/mu-xapian.hh

@@ -22,7 +22,7 @@
 #define __XAPIAN_HH__
 
 #include <xapian.h>
-#include <query/mu-parser.hh>
+#include <mu-parser.hh>
 
 namespace Mu {
 

lib/query/Makefile.am

@@ -1,99 +0,0 @@
-## Copyright (C) 2017-2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
-##
-## This program is free software; you can redistribute it and/or modify
-## it under the terms of the GNU General Public License as published by
-## the Free Software Foundation; either version 3 of the License, or
-## (at your option) any later version.
-##
-## This program is distributed in the hope that it will be useful,
-## but WITHOUT ANY WARRANTY; without even the implied warranty of
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-## GNU General Public License for more details.
-##
-## You should have received a copy of the GNU General Public License
-## along with this program; if not, write to the Free Software Foundation,
-## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-
-include $(top_srcdir)/gtest.mk
-
-@VALGRIND_CHECK_RULES@
-
-AM_CXXFLAGS=			\
-	-I$(srcdir)/..		\
-	-I$(top_srcdir)/lib	\
-	$(GLIB_CFLAGS)		\
-	$(XAPIAN_CXXFLAGS)	\
-	$(WARN_CXXFLAGS)	\
-	$(ASAN_CXXFLAGS)	\
-	$(CODE_COVERAGE_CFLAGS)	\
-	-Wno-inline		\
-	-Wno-switch-enum
-
-AM_CPPFLAGS=			\
-	$(CODE_COVERAGE_CPPFLAGS)
-
-AM_LDFLAGS=			\
-	$(ASAN_LDFLAGS)		\
-	$(WARN_LDFLAGS)
-
-noinst_PROGRAMS=		\
-	tokenize		\
-	parse
-
-noinst_LTLIBRARIES=		\
-	libmu-query.la
-
-libmu_query_la_SOURCES=   	\
-	mu-data.hh		\
-	mu-parser.cc		\
-	mu-parser.hh		\
-	mu-proc-iface.hh	\
-	mu-tokenizer.cc		\
-	mu-tokenizer.hh		\
-	mu-tree.hh		\
-	mu-xapian.cc		\
-	mu-xapian.hh
-
-libmu_query_la_LIBADD=		\
-	$(WARN_LDFLAGS)		\
-	$(GLIB_LIBS)		\
-	$(XAPIAN_LIBS)		\
-	../utils/libmu-utils.la \
-	$(CODE_COVERAGE_LIBS)
-
-VALGRIND_SUPPRESSIONS_FILES= 	\
-	${top_srcdir}/mu.supp
-
-tokenize_SOURCES=		\
-	tokenize.cc
-
-tokenize_LDADD=			\
-	$(WARN_LDFLAGS)		\
-	libmu-query.la
-
-parse_SOURCES=			\
-	parse.cc
-
-parse_LDADD=			\
-	$(WARN_LDFLAGS)		\
-	libmu-query.la
-
-noinst_PROGRAMS+=$(TEST_PROGS)
-
-TEST_PROGS+= 			\
-	test-tokenizer
-test_tokenizer_SOURCES=		\
-	test-tokenizer.cc
-test_tokenizer_LDADD=		\
-	libmu-query.la
-
-TEST_PROGS+= 			\
-	test-parser
-test_parser_SOURCES=		\
-	test-parser.cc
-test_parser_LDADD=		\
-	libmu-query.la
-
-TESTS=$(TEST_PROGS)
-
-include $(top_srcdir)/aminclude_static.am

lib/query/mu-parser.cc

@@ -1,344 +0,0 @@
-/*
-**  Copyright (C) 2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
-**
-**  This library is free software; you can redistribute it and/or
-**  modify it under the terms of the GNU Lesser General Public License
-**  as published by the Free Software Foundation; either version 2.1
-**  of the License, or (at your option) any later version.
-**
-**  This library is distributed in the hope that it will be useful,
-**  but WITHOUT ANY WARRANTY; without even the implied warranty of
-**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-**  Lesser General Public License for more details.
-**
-**  You should have received a copy of the GNU Lesser General Public
-**  License along with this library; if not, write to the Free
-**  Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA
-**  02110-1301, USA.
-*/
-#include "mu-parser.hh"
-#include "mu-tokenizer.hh"
-#include "utils/mu-utils.hh"
-#include "utils/mu-error.hh"
-
-using namespace Mu;
-
-// 3 precedence levels: units (NOT,()) > factors (OR) > terms (AND)
-
-// query	->      <term-1> | ε
-// <term-1>	->	<factor-1> <term-2> | ε
-// <term-2>	->	OR|XOR <term-1> | ε
-// <factor-1>	->	<unit> <factor-2> | ε
-// <factor-2>	->	[AND]|AND NOT <factor-1> | ε
-// <unit>	->	[NOT] <term-1> | ( <term-1> ) | <data>
-// <data>       ->      <value> | <range> | <regex>
-// <value>      ->      [field:]value
-// <range>      ->      [field:][lower]..[upper]
-// <regex>      ->      [field:]/regex/
-
-
-#define BUG(...) Mu::Error (Error::Code::Internal, format("%u: BUG: ",__LINE__)	\
-				     + format(__VA_ARGS__))
-
-static Token
-look_ahead (const Mu::Tokens& tokens)
-{
-	return tokens.front();
-}
-
-static Mu::Tree
-empty()
-{
-	return {{Node::Type::Empty}};
-}
-
-static Mu::Tree term_1 (Mu::Tokens& tokens, ProcPtr proc, WarningVec& warnings);
-
-
-static Mu::Tree
-value (const ProcIface::FieldInfoVec& fields, const std::string& v,
-       size_t pos, ProcPtr proc, WarningVec& warnings)
-{
-	auto val = utf8_flatten(v);
-
-	if (fields.empty())
-		throw BUG("expected one or more fields");
-
-	if (fields.size() == 1) {
-		const auto item = fields.front();
-		return Tree({Node::Type::Value,
-			     std::make_unique<Value>(
-				     item.field, item.prefix, item.id,
-				     proc->process_value(item.field, val),
-				     item.supports_phrase)});
-	}
-
-	// a 'multi-field' such as "recip:"
-	Tree tree(Node{Node::Type::OpOr});
-	for (const auto& item: fields)
-		tree.add_child (Tree({Node::Type::Value,
-				      std::make_unique<Value>(
-					      item.field, item.prefix, item.id,
-					      proc->process_value(item.field, val),
-					      item.supports_phrase)}));
-	return tree;
-}
-
-static Mu::Tree
-regex (const ProcIface::FieldInfoVec& fields, const std::string& v,
-       size_t pos, ProcPtr proc, WarningVec& warnings)
-{
-	if (v.length() < 2)
-		throw BUG("expected regexp, got '%s'", v.c_str());
-
-	const auto rxstr = utf8_flatten(v.substr(1, v.length()-2));
-
- 	try {
-		Tree tree(Node{Node::Type::OpOr});
-		const auto rx = std::regex (rxstr);
-		for (const auto& field: fields) {
-			const auto terms = proc->process_regex (field.field, rx);
-			for (const auto& term: terms) {
-				tree.add_child (Tree(
-					{Node::Type::Value,
-					 std::make_unique<Value>(field.field, "",
-								 field.id, term)}));
-			}
-		}
-
-		if (tree.children.empty())
-			return empty();
-		else
-			return tree;
-
-	} catch (...) {
-		// fallback
-		warnings.push_back ({pos, "invalid regexp"});
-		return value (fields, v, pos, proc, warnings);
-	}
-}
-
-
-
-static Mu::Tree
-range (const ProcIface::FieldInfoVec& fields, const std::string& lower,
-       const std::string& upper, size_t pos, ProcPtr proc,
-       WarningVec& warnings)
-{
-	if (fields.empty())
-		throw BUG("expected field");
-
-	const auto& field = fields.front();
-	if (!proc->is_range_field(field.field))
-		return value (fields, lower + ".." + upper, pos, proc, warnings);
-
-	auto prange = proc->process_range (field.field, lower, upper);
-	if (prange.lower > prange.upper)
-		prange = proc->process_range (field.field, upper, lower);
-
-	return Tree({Node::Type::Range,
-			     std::make_unique<Range>(field.field, field.prefix, field.id,
-						     prange.lower, prange.upper)});
-}
-
-
-static Mu::Tree
-data (Mu::Tokens& tokens, ProcPtr proc, WarningVec& warnings)
-{
-	const auto token = look_ahead(tokens);
-	if (token.type != Token::Type::Data)
-		warnings.push_back ({token.pos, "expected: value"});
-
-	tokens.pop_front();
-
-	std::string field, val;
-	const auto col = token.str.find (":");
-	if (col != 0 && col != std::string::npos && col != token.str.length()-1) {
-		field = token.str.substr(0, col);
-		val = token.str.substr(col + 1);
-	} else
-		val = token.str;
-
-	auto fields = proc->process_field (field);
-	if (fields.empty()) {// not valid field...
-		warnings.push_back ({token.pos, format ("invalid field '%s'", field.c_str())});
-		fields = proc->process_field ("");
-		// fallback, treat the whole of foo:bar as a value
-		return value (fields, field + ":" + val, token.pos, proc, warnings);
-	}
-
-	// does it look like a regexp?
-	if (val.length() >=2 )
-		if (val[0] == '/' && val[val.length()-1] == '/')
-			return regex (fields, val, token.pos, proc, warnings);
-
-	// does it look like a range?
-	const auto dotdot = val.find("..");
-	if (dotdot != std::string::npos)
-		return range(fields, val.substr(0, dotdot), val.substr(dotdot + 2),
-			     token.pos, proc, warnings);
-	else if (proc->is_range_field(fields.front().field)) {
-		// range field without a range - treat as field:val..val
-		return range (fields, val, val, token.pos, proc, warnings);
-	}
-
-	// if nothing else, it's a value.
-	return value (fields, val, token.pos, proc, warnings);
-}
-
-static Mu::Tree
-unit (Mu::Tokens& tokens, ProcPtr proc, WarningVec& warnings)
-{
-	if (tokens.empty()) {
-		warnings.push_back ({0, "expected: unit"});
-		return empty();
-	}
-
-	const auto token = look_ahead (tokens);
-
-	if (token.type == Token::Type::Not) {
-		tokens.pop_front();
-		Tree tree{{Node::Type::OpNot}};
-		tree.add_child(unit (tokens, proc, warnings));
-		return tree;
-	}
-
-	if (token.type == Token::Type::Open) {
-		tokens.pop_front();
-		auto tree = term_1 (tokens, proc, warnings);
-		if (tokens.empty())
-			warnings.push_back({token.pos, "expected: ')'"});
-		else {
-			const auto token2 = look_ahead(tokens);
-			if (token2.type == Token::Type::Close)
-				tokens.pop_front();
-			else {
-				warnings.push_back(
-				{token2.pos,
-				 std::string("expected: ')' but got ") +
-				 token2.str});
-			}
-
-		}
-		return tree;
-	}
-
-	return data (tokens, proc, warnings);
-}
-
-static Mu::Tree factor_1 (Mu::Tokens& tokens, ProcPtr proc,
-			   WarningVec& warnings);
-
-static Mu::Tree
-factor_2 (Mu::Tokens& tokens, Node::Type& op, ProcPtr proc,
-	  WarningVec& warnings)
-{
-	if (tokens.empty())
-		return empty();
-
-	const auto token = look_ahead(tokens);
-
-	switch (token.type) {
-	case Token::Type::And: {
-		tokens.pop_front();
-		op = Node::Type::OpAnd;
-	} break;
-
-	case Token::Type::Open:
-	case Token::Type::Data:
-	case Token::Type::Not:
-		op = Node::Type::OpAnd; // implicit AND
-		break;
-
-	default:
-		return empty();
-	}
-
-	return factor_1 (tokens, proc, warnings);
-}
-
-static Mu::Tree
-factor_1 (Mu::Tokens& tokens, ProcPtr proc, WarningVec& warnings)
-{
-	Node::Type op { Node::Type::Invalid };
-
-	auto t  = unit (tokens, proc, warnings);
-	auto a2 = factor_2 (tokens, op, proc, warnings);
-
-	if (a2.empty())
-		return t;
-
-	Tree tree {{op}};
-	tree.add_child(std::move(t));
-	tree.add_child(std::move(a2));
-
-	return tree;
-}
-
-
-static Mu::Tree
-term_2 (Mu::Tokens& tokens, Node::Type& op, ProcPtr proc,
-	WarningVec& warnings)
-{
-	if (tokens.empty())
-		return empty();
-
-	const auto token = look_ahead (tokens);
-
-	switch (token.type) {
-	case Token::Type::Or:
-		op = Node::Type::OpOr;
-		break;
-	case Token::Type::Xor:
-		op = Node::Type::OpXor;
-		break;
-	default:
-		if (token.type != Token::Type::Close)
-			warnings.push_back({token.pos, "expected OR|XOR"});
-		return empty();
-	}
-
-	tokens.pop_front();
-
-	return term_1 (tokens, proc, warnings);
-}
-
-static Mu::Tree
-term_1 (Mu::Tokens& tokens, ProcPtr proc, WarningVec& warnings)
-{
-	Node::Type op { Node::Type::Invalid };
-
-	auto t  = factor_1 (tokens, proc, warnings);
-	auto o2 = term_2 (tokens, op, proc, warnings);
-
-	if (o2.empty())
-		return t;
-	else {
-		Tree tree {{op}};
-		tree.add_child(std::move(t));
-		tree.add_child(std::move(o2));
-		return tree;
-	}
-}
-
-static Mu::Tree
-query (Mu::Tokens& tokens, ProcPtr proc, WarningVec& warnings)
-{
-	if (tokens.empty())
-		return empty ();
-	else
-		return term_1 (tokens, proc, warnings);
-}
-
-Mu::Tree
-Mu::parse (const std::string& expr, WarningVec& warnings, ProcPtr proc)
-{
-	try {
-		auto tokens = tokenize (expr);
-		return query (tokens, proc, warnings);
-
-	} catch (const std::runtime_error& ex) {
-		std::cerr << ex.what() << std::endl;
-		return empty();
-	}
-}

lib/query/mu-proc-iface.hh

@@ -1,132 +0,0 @@
-/*
-** Copyright (C) 2017 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
-**
-**  This library is free software; you can redistribute it and/or
-**  modify it under the terms of the GNU Lesser General Public License
-**  as published by the Free Software Foundation; either version 2.1
-**  of the License, or (at your option) any later version.
-**
-**  This library is distributed in the hope that it will be useful,
-**  but WITHOUT ANY WARRANTY; without even the implied warranty of
-**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-**  Lesser General Public License for more details.
-**
-**  You should have received a copy of the GNU Lesser General Public
-**  License along with this library; if not, write to the Free
-**  Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA
-**  02110-1301, USA.
-*/
-#ifndef __PROC_IFACE_HH__
-#define __PROC_IFACE_HH__
-
-#include <string>
-#include <vector>
-#include <tuple>
-#include <regex>
-
-namespace Mu {
-
-struct ProcIface {
-
-	virtual ~ProcIface() = default;
-
-	/**
-	 * Get the "shortcut"/internal fields for the the given fieldstr or empty if there is none
-	 *
-	 * @param fieldstr a fieldstr, e.g "subject" or "s" for the subject field
-	 *
-	 * @return a vector with "exploded" values, with a code and a fullname. E.g. "s" might map
-	 * to [<"S","subject">], while "recip" could map to [<"to", "T">, <"cc", "C">, <"bcc", "B">]
-	 */
-	struct FieldInfo {
-		const std::string	field;
-		const std::string	prefix;
-		bool			supports_phrase;
-		unsigned		id;
-	};
-	using FieldInfoVec = std::vector<FieldInfo>;
-
-	virtual FieldInfoVec process_field (const std::string& field) const = 0;
-
-	/**
-	 * Process a value
-	 *
-	 * @param field a field name
-	 * @param value a value
-	 *
-	 * @return the processed value
-	 */
-	virtual std::string process_value (
-		const std::string& field, const std::string& value) const = 0;
-
-	/**
-	 * Is this a range field?
-	 *
-	 * @param field some field
-	 *
-	 * @return true if it is a range-field; false otherwise.
-	 */
-	virtual bool is_range_field (const std::string& field) const = 0;
-
-
-	/**
-	 * Process a range field
-	 *
-	 * @param fieldstr a fieldstr, e.g "date" or "d" for the date field
-	 * @param lower lower bound or empty
-	 * @param upper upper bound or empty
-	 *
-	 * @return the processed range
-	 */
-	struct Range {
-		std::string lower;
-		std::string upper;
-	};
-	virtual Range process_range (const std::string& field, const std::string& lower,
-				     const std::string& upper) const = 0;
-
-	/**
-	 *
-	 *
-	 * @param field
-	 * @param rx
-	 *
-	 * @return
-	 */
-	virtual std::vector<std::string>
-	process_regex (const std::string& field, const std::regex& rx) const = 0;
-
-}; // ProcIface
-
-
-struct DummyProc: public ProcIface { // For testing
-
-	std::vector<FieldInfo>
-	process_field (const std::string& field) const override {
-		return {{ field, "x", false, 0 }};
-	}
-
-	std::string
-	process_value (const std::string& field, const std::string& value) const override {
-		return value;
-	}
-
-	bool is_range_field (const std::string& field) const override {
-		return field == "range";
-	}
-
-	Range process_range (const std::string& field, const std::string& lower,
-			     const std::string& upper) const override {
-		return { lower, upper };
-	}
-
-	std::vector<std::string>
-	process_regex (const std::string& field, const std::regex& rx) const override {
-		return {};
-	}
-}; //Dummy
-
-
-} // Mu
-
-#endif /* __PROC_IFACE_HH__ */

lib/query/parse.cc

@@ -1,41 +0,0 @@
-/*
-**  Copyright (C) 2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
-**
-**  This library is free software; you can redistribute it and/or
-**  modify it under the terms of the GNU Lesser General Public License
-**  as published by the Free Software Foundation; either version 2.1
-**  of the License, or (at your option) any later version.
-**
-**  This library is distributed in the hope that it will be useful,
-**  but WITHOUT ANY WARRANTY; without even the implied warranty of
-**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-**  Lesser General Public License for more details.
-**
-**  You should have received a copy of the GNU Lesser General Public
-**  License along with this library; if not, write to the Free
-**  Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA
-**  02110-1301, USA.
-*/
-
-#include <string>
-#include <iostream>
-#include "mu-parser.hh"
-
-int
-main (int argc, char *argv[])
-{
-	std::string s;
-
-	for (auto i = 1; i < argc; ++i)
-		s += " " + std::string(argv[i]);
-
-	Mu::WarningVec warnings;
-
-	const auto tree = Mu::parse (s, warnings);
-	for (const auto& w: warnings)
-		std::cerr << "1:" << w.pos << ": " << w.msg << std::endl;
-
-	std::cout << tree << std::endl;
-
-	return 0;
-}

lib/test-indexer.cc

@@ -0,0 +1,70 @@
+/*
+** Copyright (C) 2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify it
+** under the terms of the GNU General Public License as published by the
+** Free Software Foundation; either version 3, or (at your option) any
+** later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+**
+*/
+
+#include <vector>
+#include <glib.h>
+
+#include <iostream>
+#include <sstream>
+#include <unistd.h>
+
+#include "mu-indexer.hh"
+#include "utils/mu-utils.hh"
+#include "test-mu-common.h"
+
+using namespace Mu;
+
+static void
+test_index_maildir ()
+{
+        allow_warnings();
+
+	Store store{test_mu_common_get_random_tmpdir(), std::string{MU_TESTMAILDIR}};
+	Indexer idx{Indexer::Config{}, store};
+
+	g_assert_true (idx.start());
+        while (idx.is_running()) {
+                sleep(1);
+        }
+
+        g_print ("again!\n");
+
+        g_assert_true (idx.start());
+        while (idx.is_running()) {
+                sleep(1);
+        }
+}
+
+int
+main (int argc, char *argv[]) try
+{
+	g_test_init (&argc, &argv, NULL);
+
+	g_test_add_func ("/indexer/index-maildir", test_index_maildir);
+
+	return g_test_run ();
+
+
+} catch (const std::runtime_error& re) {
+	std::cerr << re.what() << "\n";
+	return 1;
+} catch (...) {
+        std::cerr << "caught exception\n";
+        return 1;
+}

lib/test-mu-common.cc

@@ -1,5 +1,5 @@
 /*
-** Copyright (C) 2008-2013 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+** Copyright (C) 2008-2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
 **
 ** This program is free software; you can redistribute it and/or modify it
 ** under the terms of the GNU General Public License as published by the
@@ -31,7 +31,7 @@
 #include <langinfo.h>
 #include <locale.h>
 
-#include "test-mu-common.h"
+#include "test-mu-common.hh"
 
 char*
 test_mu_common_get_random_tmpdir (void)

lib/test-mu-contacts.cc

@@ -21,7 +21,7 @@
 #include "config.h"
 
 #include <glib.h>
-#include "test-mu-common.h"
+#include "test-mu-common.hh"
 #include "mu-contacts.hh"
 
 static void

lib/test-mu-container.cc

@@ -25,7 +25,7 @@
 
 #include <glib.h>
 
-#include "test-mu-common.h"
+#include "test-mu-common.hh"
 #include "mu-container.h"
 
 static gboolean
@@ -76,7 +76,7 @@ main (int argc, char *argv[])
 			 test_mu_container_splice_children_when_parent_has_no_siblings);
 
 	g_log_set_handler (NULL,
-			   G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL| G_LOG_FLAG_RECURSION,
+			   (GLogLevelFlags)(G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL| G_LOG_FLAG_RECURSION),
 			   (GLogFunc)black_hole, NULL);
 
 	return g_test_run ();

lib/test-mu-flags.cc

@@ -24,7 +24,7 @@
 
 #include <glib.h>
 #include "mu-flags.h"
-#include "test-mu-common.h"
+#include "test-mu-common.hh"
 
 
 static void
@@ -41,7 +41,7 @@ test_mu_flag_char (void)
 	g_assert_cmpuint (mu_flag_char (MU_FLAG_ENCRYPTED),	==, 'x');
 	g_assert_cmpuint (mu_flag_char (MU_FLAG_HAS_ATTACH),	==, 'a');
 	g_assert_cmpuint (mu_flag_char (MU_FLAG_UNREAD),	==, 'u');
-	g_assert_cmpuint (mu_flag_char (12345),			==,  0);
+	g_assert_cmpuint (mu_flag_char ((MuFlags)12345),        ==,  0);
 }
 
 
@@ -60,30 +60,30 @@ test_mu_flag_name (void)
 	g_assert_cmpstr (mu_flag_name (MU_FLAG_ENCRYPTED),	==, "encrypted");
 	g_assert_cmpstr (mu_flag_name (MU_FLAG_HAS_ATTACH),	==, "attach");
 	g_assert_cmpstr (mu_flag_name (MU_FLAG_UNREAD),		==, "unread");
-	g_assert_cmpstr (mu_flag_name (12345),			==,  NULL);
+	g_assert_cmpstr (mu_flag_name ((MuFlags)12345),	        ==,  NULL);
 }
 
 static void
 test_mu_flags_to_str_s (void)
 {
-	g_assert_cmpstr (mu_flags_to_str_s(MU_FLAG_PASSED|MU_FLAG_SIGNED,
+	g_assert_cmpstr (mu_flags_to_str_s((MuFlags)(MU_FLAG_PASSED|MU_FLAG_SIGNED),
 					   MU_FLAG_TYPE_ANY),
 			 ==, "Pz");
 	g_assert_cmpstr (mu_flags_to_str_s(MU_FLAG_NEW, MU_FLAG_TYPE_ANY),
 			 ==, "N");
-	g_assert_cmpstr (mu_flags_to_str_s(MU_FLAG_HAS_ATTACH|MU_FLAG_TRASHED,
+	g_assert_cmpstr (mu_flags_to_str_s((MuFlags)(MU_FLAG_HAS_ATTACH|MU_FLAG_TRASHED),
 					   MU_FLAG_TYPE_ANY),
 			 ==, "Ta");
 	g_assert_cmpstr (mu_flags_to_str_s(MU_FLAG_NONE, MU_FLAG_TYPE_ANY),
 			 ==, "");
 
-	g_assert_cmpstr (mu_flags_to_str_s(MU_FLAG_PASSED|MU_FLAG_SIGNED,
+	g_assert_cmpstr (mu_flags_to_str_s((MuFlags)(MU_FLAG_PASSED|MU_FLAG_SIGNED),
 					   MU_FLAG_TYPE_CONTENT),
 			 ==, "z");
 
 	g_assert_cmpstr (mu_flags_to_str_s(MU_FLAG_NEW, MU_FLAG_TYPE_MAILDIR),
 			 ==, "N");
-	g_assert_cmpstr (mu_flags_to_str_s(MU_FLAG_HAS_ATTACH|MU_FLAG_TRASHED,
+	g_assert_cmpstr (mu_flags_to_str_s((MuFlags)(MU_FLAG_HAS_ATTACH|MU_FLAG_TRASHED),
 					   MU_FLAG_TYPE_MAILFILE),
 			 ==, "T");
 
@@ -100,11 +100,11 @@ test_mu_flags_from_str (void)
 	 */
 
 	g_assert_cmpuint (mu_flags_from_str ("RP", MU_FLAG_TYPE_ANY, TRUE), ==,
-			  MU_FLAG_REPLIED | MU_FLAG_PASSED);
+			 (MuFlags)( MU_FLAG_REPLIED | MU_FLAG_PASSED));
 	g_assert_cmpuint (mu_flags_from_str ("Nz", MU_FLAG_TYPE_ANY, TRUE), ==,
 			  MU_FLAG_NEW | MU_FLAG_SIGNED);
 	g_assert_cmpuint (mu_flags_from_str ("axD", MU_FLAG_TYPE_ANY, TRUE), ==,
-			  MU_FLAG_HAS_ATTACH | MU_FLAG_ENCRYPTED | MU_FLAG_DRAFT);
+			 (MuFlags)( MU_FLAG_HAS_ATTACH | MU_FLAG_ENCRYPTED | MU_FLAG_DRAFT));
 
 	g_assert_cmpuint (mu_flags_from_str ("RP", MU_FLAG_TYPE_MAILFILE, TRUE), ==,
 			  MU_FLAG_REPLIED | MU_FLAG_PASSED);
@@ -124,19 +124,19 @@ static void
 test_mu_flags_from_str_delta (void)
 {
 	g_assert_cmpuint (mu_flags_from_str_delta ("+S-R",
-						   MU_FLAG_REPLIED | MU_FLAG_DRAFT,
+						   (MuFlags)(MU_FLAG_REPLIED | MU_FLAG_DRAFT),
 						   MU_FLAG_TYPE_ANY),==,
-			  MU_FLAG_SEEN | MU_FLAG_DRAFT);
+			  (MuFlags)(MU_FLAG_SEEN | MU_FLAG_DRAFT));
 
 	g_assert_cmpuint (mu_flags_from_str_delta ("",
-						   MU_FLAG_REPLIED | MU_FLAG_DRAFT,
+						   (MuFlags)(MU_FLAG_REPLIED | MU_FLAG_DRAFT),
 						   MU_FLAG_TYPE_ANY),==,
-			  MU_FLAG_REPLIED | MU_FLAG_DRAFT);
+			  (MuFlags)(MU_FLAG_REPLIED | MU_FLAG_DRAFT));
 
 	g_assert_cmpuint (mu_flags_from_str_delta ("-N+P+S-D",
-						   MU_FLAG_SIGNED | MU_FLAG_DRAFT,
+						   (MuFlags)(MU_FLAG_SIGNED | MU_FLAG_DRAFT),
 						   MU_FLAG_TYPE_ANY),==,
-			  MU_FLAG_PASSED | MU_FLAG_SEEN | MU_FLAG_SIGNED);
+			  (MuFlags)(MU_FLAG_PASSED | MU_FLAG_SEEN | MU_FLAG_SIGNED));
 }
 
 
@@ -184,7 +184,8 @@ main (int argc, char *argv[])
 			 test_mu_flags_custom_from_str);
 
 	g_log_set_handler (NULL,
-			   G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL| G_LOG_FLAG_RECURSION,
+			   (GLogLevelFlags)(G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL|
+					    G_LOG_FLAG_RECURSION),
 			   (GLogFunc)black_hole, NULL);
 
 	rv = g_test_run ();

lib/test-mu-maildir.cc

@@ -28,10 +28,11 @@
 #include <unistd.h>
 #include <string.h>
 
-#include "test-mu-common.h"
+#include "test-mu-common.hh"
 #include "mu-maildir.h"
 #include "utils/mu-util.h"
 
+
 static void
 test_mu_maildir_mkdir_01 (void)
 {
@@ -400,7 +401,7 @@ test_mu_maildir_get_flags_from_path (void)
 	} paths[] = {
 		{
 			"/home/foo/Maildir/test/cur/123456:2,FSR",
-			MU_FLAG_REPLIED | MU_FLAG_SEEN | MU_FLAG_FLAGGED
+			(MuFlags)(MU_FLAG_REPLIED | MU_FLAG_SEEN | MU_FLAG_FLAGGED)
 		},
 		{
 			"/home/foo/Maildir/test/new/123456",
@@ -413,8 +414,8 @@ test_mu_maildir_get_flags_from_path (void)
 		},
 		{
 			"/home/foo/Maildir/test/cur/123456:2,DTP",
-			MU_FLAG_DRAFT | MU_FLAG_TRASHED |
+			(MuFlags)(MU_FLAG_DRAFT | MU_FLAG_TRASHED |
-			MU_FLAG_PASSED
+			MU_FLAG_PASSED)
 		},
 		{
 			"/home/foo/Maildir/test/cur/123456:2,S",
@@ -434,7 +435,8 @@ test_mu_maildir_get_flags_from_path (void)
 static void
 assert_matches_regexp (const char *str, const char *rx)
 {
-	if (!g_regex_match_simple (rx, str, 0, 0)) {
+	if (!g_regex_match_simple (rx, str, (GRegexCompileFlags)0,
+                                   (GRegexMatchFlags)0)) {
 		if (g_test_verbose ())
 			g_print ("%s does not match %s", str, rx);
 		g_assert (0);
@@ -463,11 +465,11 @@ test_mu_maildir_get_new_path_new (void)
 			"/home/foo/Maildir/test/new/123456"
 		}, {
 			"/home/foo/Maildir/test/new/123456:2,FR",
-			MU_FLAG_SEEN | MU_FLAG_REPLIED,
+			(MuFlags)(MU_FLAG_SEEN | MU_FLAG_REPLIED),
 			"/home/foo/Maildir/test/cur/123456:2,RS"
 		}, {
 			"/home/foo/Maildir/test/new/1313038887_0.697:2,",
-			MU_FLAG_SEEN | MU_FLAG_FLAGGED | MU_FLAG_PASSED,
+			(MuFlags)(MU_FLAG_SEEN | MU_FLAG_FLAGGED | MU_FLAG_PASSED),
 			"/home/foo/Maildir/test/cur/1313038887_0.697:2,FPS"
 		}, {
 			"/home/djcb/Maildir/trash/new/1312920597.2206_16.cthulhu",
@@ -513,11 +515,11 @@ test_mu_maildir_get_new_path_01 (void)
 			"/home/foo/Maildir/test/new/123456"
 		}, {
 			"/home/foo/Maildir/test/new/123456:2,FR",
-			MU_FLAG_SEEN | MU_FLAG_REPLIED,
+			(MuFlags)(MU_FLAG_SEEN | MU_FLAG_REPLIED),
 			"/home/foo/Maildir/test/cur/123456:2,RS"
 		}, {
 			"/home/foo/Maildir/test/new/1313038887_0.697:2,",
-			MU_FLAG_SEEN | MU_FLAG_FLAGGED | MU_FLAG_PASSED,
+			(MuFlags)(MU_FLAG_SEEN | MU_FLAG_FLAGGED | MU_FLAG_PASSED),
 			"/home/foo/Maildir/test/cur/1313038887_0.697:2,FPS"
 		}, {
 			"/home/djcb/Maildir/trash/new/1312920597.2206_16.cthulhu",
@@ -557,12 +559,12 @@ test_mu_maildir_get_new_path_02 (void)
 			"/home/bar/Maildir/coffee/new/123456"
 		}, {
 			"/home/foo/Maildir/test/new/123456",
-			MU_FLAG_SEEN | MU_FLAG_REPLIED,
+			(MuFlags)(MU_FLAG_SEEN | MU_FLAG_REPLIED),
 			"/home/cuux/Maildir/tea",
 			"/home/cuux/Maildir/tea/cur/123456:2,RS"
 		}, {
 			"/home/foo/Maildir/test/new/1313038887_0.697:2,",
-			MU_FLAG_SEEN | MU_FLAG_FLAGGED | MU_FLAG_PASSED,
+			(MuFlags)(MU_FLAG_SEEN | MU_FLAG_FLAGGED | MU_FLAG_PASSED),
 			"/home/boy/Maildir/stuff",
 			"/home/boy/Maildir/stuff/cur/1313038887_0.697:2,FPS"
 		}
@@ -685,8 +687,8 @@ main (int argc, char *argv[])
 			test_mu_maildir_get_maildir_from_path);
 
 	g_log_set_handler (NULL,
-			   G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL|
+			   (GLogLevelFlags)(G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL|
-			   G_LOG_FLAG_RECURSION,
+			   G_LOG_FLAG_RECURSION),
 			   (GLogFunc)black_hole, NULL);
 
 	return g_test_run ();

lib/test-mu-msg-fields.cc

@@ -1,5 +1,5 @@
 /*
-** Copyright (C) 2008-2013 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+** Copyright (C) 2008-2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
 **
 ** This program is free software; you can redistribute it and/or modify it
 ** under the terms of the GNU General Public License as published by the
@@ -28,7 +28,7 @@
 
 #include <locale.h>
 
-#include "test-mu-common.h"
+#include "test-mu-common.hh"
 #include "mu-msg-fields.h"
 
 static void
@@ -126,8 +126,8 @@ main (int argc, char *argv[])
 	 * function simply calls mu_msg_field_str */
 
 	g_log_set_handler (NULL,
-			   G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL |
+			   (GLogLevelFlags)(G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL |
-			   G_LOG_FLAG_RECURSION,
+			   G_LOG_FLAG_RECURSION),
 			   (GLogFunc)black_hole, NULL);
 
 	return g_test_run ();

lib/test-mu-msg.cc

@@ -1,6 +1,5 @@
-/* -*-mode: c; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-*/
 /*
-** Copyright (C) 2008-2013 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+** Copyright (C) 2008-2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
 **
 ** This program is free software; you can redistribute it and/or modify it
 ** under the terms of the GNU General Public License as published by the
@@ -29,7 +28,7 @@
 
 #include <locale.h>
 
-#include "test-mu-common.h"
+#include "test-mu-common.hh"
 #include "mu-msg.h"
 #include "utils/mu-str.h"
 
@@ -250,8 +249,8 @@ test_mu_msg_multimime (void)
 	g_assert_cmpstr (mu_msg_get_body_text(msg, MU_MSG_OPTION_NONE),
 			 ==, "abcdef");
 	g_assert_cmpuint (mu_msg_get_flags(msg),
-			 ==, MU_FLAG_FLAGGED | MU_FLAG_SEEN |
+			 ==, (MuFlags)(MU_FLAG_FLAGGED | MU_FLAG_SEEN |
-			 MU_FLAG_HAS_ATTACH);
+			 MU_FLAG_HAS_ATTACH));
 	mu_msg_unref (msg);
 }
 
@@ -267,8 +266,8 @@ test_mu_msg_flags (void)
 		MuFlags flags;
 	} msgflags [] = {
 		{ MU_TESTMAILDIR4 "/multimime!2,FS",
-		  MU_FLAG_FLAGGED | MU_FLAG_SEEN |
+		  (MuFlags)(MU_FLAG_FLAGGED | MU_FLAG_SEEN |
-		  MU_FLAG_HAS_ATTACH },
+		  MU_FLAG_HAS_ATTACH) },
 		{ MU_TESTMAILDIR4 "/special!2,Sabc",
 		  MU_FLAG_SEEN }
 
@@ -515,7 +514,7 @@ test_mu_str_prio_02 (void)
 {
 	/* this must fail */
 	g_test_log_set_fatal_handler ((GTestLogFatalFunc)ignore_error, NULL);
-	g_assert_cmpstr (mu_msg_prio_name(666), ==, NULL);
+	g_assert_cmpstr (mu_msg_prio_name((MuMsgPrio)666), ==, NULL);
 }
 
 
@@ -588,8 +587,8 @@ main (int argc, char *argv[])
 
 
 	g_log_set_handler (NULL,
-			   G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL|
+			   (GLogLevelFlags)(G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL|
-			   G_LOG_FLAG_RECURSION,
+			   G_LOG_FLAG_RECURSION),
 			   (GLogFunc)black_hole, NULL);
 
 	rv = g_test_run ();

lib/test-mu-store.cc

@@ -26,7 +26,7 @@
 
 #include <locale.h>
 
-#include "test-mu-common.h"
+#include "test-mu-common.hh"
 #include "mu-store.hh"
 
 static std::string MuTestMaildir = Mu::canonicalize_filename(MU_TESTMAILDIR, "/");

lib/test-parser.cc

@@ -37,10 +37,12 @@ using CaseVec = std::vector<Case>;
 static void
 test_cases(const CaseVec& cases)
 {
+        Parser parser;
+
 	for (const auto& casus : cases ) {
 
 		WarningVec warnings;
-		const auto tree = parse (casus.expr, warnings);
+		const auto tree = parser.parse (casus.expr, warnings);
 
 		std::stringstream ss;
 		ss << tree;