message: move to lib/message, update naming

Basically, move/rename lib/mu-message* to lib/mu-*.

Add the beginnings of a Message class.

lib/message/meson.build

@@ -0,0 +1,85 @@
+## Copyright (C) 2021 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+##
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 3 of the License, or
+## (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with this program; if not, write to the Free Software Foundation,
+## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+
+lib_mu_message=static_library(
+  'mu-message',
+  [
+    'mu-message.cc',
+    'mu-message.hh',
+    'mu-contact.hh',
+    'mu-contact.cc',
+    'mu-document.cc',
+    'mu-document.hh',
+    'mu-fields.hh',
+    'mu-fields.cc',
+    'mu-flags.hh',
+    'mu-flags.cc',
+    'mu-priority.hh',
+    'mu-priority.cc'
+  ],
+  dependencies: [
+    glib_dep,
+    gmime_dep,
+    xapian_dep,
+    config_h_dep,
+    lib_mu_utils_dep],
+  install: false)
+
+# some of the libme headers include xapian
+xapian_incs = xapian_dep.get_pkgconfig_variable('includedir')
+lib_mu_message_dep = declare_dependency(
+  link_with: lib_mu_message,
+  include_directories:
+    include_directories(['.', '..', xapian_incs]))
+
+#
+# tests
+#
+
+test('test-contact',
+     executable('test-contact',
+		'mu-contact.cc',
+		install: false,
+		cpp_args: ['-DBUILD_TESTS'],
+		dependencies: [glib_dep, gmime_dep, lib_mu_message_dep]))
+
+test('test-document',
+     executable('test-document',
+		'mu-document.cc',
+		install: false,
+		cpp_args: ['-DBUILD_TESTS'],
+		dependencies: [glib_dep, gmime_dep, lib_mu_message_dep]))
+
+test('test-fields',
+     executable('test-fields',
+		'mu-fields.cc',
+		install: false,
+		cpp_args: ['-DBUILD_TESTS'],
+		dependencies: [glib_dep, gmime_dep, lib_mu_message_dep]))
+
+test('test-flags',
+     executable('test-flags',
+		'mu-flags.cc',
+		install: false,
+		cpp_args: ['-DBUILD_TESTS'],
+		dependencies: [glib_dep, gmime_dep, lib_mu_message_dep]))
+test('test-priority',
+     executable('test-priority',
+		'mu-priority.cc',
+		install: false,
+		cpp_args: ['-DBUILD_TESTS'],
+		dependencies: [glib_dep, gmime_dep, lib_mu_message_dep]))

lib/message/mu-contact.cc

@@ -0,0 +1,243 @@
+/*
+** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify it
+** under the terms of the GNU General Public License as published by the
+** Free Software Foundation; either version 3, or (at your option) any
+** later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+**
+*/
+
+#include "mu-contact.hh"
+#include "mu-message.hh"
+
+#include <gmime/gmime.h>
+#include <glib.h>
+
+using namespace Mu;
+
+std::string
+Contact::display_name() const
+{
+	if (name.empty())
+		return email;
+	else
+		return name + " <" + email + '>';
+}
+
+Mu::Contacts
+Mu::make_contacts(InternetAddressList* addr_lst,
+		  Field::Id field_id, ::time_t message_date)
+{
+	Contacts contacts;
+	size_t num{};
+
+	g_return_val_if_fail(addr_lst, contacts);
+
+	auto lst_len{internet_address_list_length(addr_lst)};
+	contacts.reserve(lst_len);
+	for (auto i = 0; i != lst_len; ++i) {
+
+		auto&& addr{internet_address_list_get_address(addr_lst, i)};
+		const auto name{internet_address_get_name(addr)};
+
+		if (G_UNLIKELY(!INTERNET_ADDRESS_IS_MAILBOX(addr)))
+			continue;
+
+		const auto email{internet_address_mailbox_get_addr (
+				INTERNET_ADDRESS_MAILBOX(addr))};
+		if (G_UNLIKELY(!email))
+			continue;
+
+		contacts.push_back(Contact{email, name ? name : "",
+				field_id, message_date});
+		++num;
+	}
+
+	return contacts;
+}
+
+
+Mu::Contacts
+Mu::make_contacts(const std::string& addrs,
+		  Field::Id field_id,
+		  ::time_t message_date)
+{
+	auto addr_list = internet_address_list_parse(NULL, addrs.c_str());
+	if (!addr_list) {
+		g_warning("no addresses found in '%s'", addrs.c_str());
+		return {};
+	}
+
+	auto contacts{make_contacts(addr_list, field_id, message_date)};
+	g_object_unref(addr_list);
+
+	return contacts;
+}
+
+
+size_t
+Mu::lowercase_hash(const std::string& s)
+{
+	std::size_t djb = 5381; // djb hash
+	for (const auto c : s)
+		djb = ((djb << 5) + djb) +
+			static_cast<size_t>(g_ascii_tolower(c));
+	return djb;
+}
+
+#ifdef BUILD_TESTS
+/*
+ * Tests.
+ *
+ */
+
+#include "utils/mu-utils.hh"
+
+static void
+test_ctor_foo()
+{
+	Contact c{
+		"foo@example.com",
+		"Foo Bar",
+		Field::Id::Bcc,
+		1645214647
+	};
+
+	assert_equal(c.email, "foo@example.com");
+	assert_equal(c.name, "Foo Bar");
+	g_assert_true(c.field_id == Field::Id::Bcc);
+	g_assert_cmpuint(c.message_date,==,1645214647);
+
+	assert_equal(c.display_name(), "Foo Bar <foo@example.com>");
+}
+
+
+static void
+test_ctor_blinky()
+{
+	Contact c{
+		"bar@example.com",
+		"Blinky",
+		1645215014,
+		true, /* personal */
+		13, /*freq*/
+		12345 /* tstamp */
+	};
+
+	assert_equal(c.email, "bar@example.com");
+	assert_equal(c.name, "Blinky");
+	g_assert_true(c.personal);
+	g_assert_cmpuint(c.frequency,==,13);
+	g_assert_cmpuint(c.tstamp,==,12345);
+	g_assert_cmpuint(c.message_date,==,1645215014);
+
+	assert_equal(c.display_name(), "Blinky <bar@example.com>");
+}
+
+static void
+test_ctor_cleanup()
+{
+	Contact c{
+		"bar@example.com",
+		"Bli\nky",
+		1645215014,
+		true, /* personal */
+		13, /*freq*/
+		12345 /* tstamp */
+	};
+
+	assert_equal(c.email, "bar@example.com");
+	assert_equal(c.name, "Bli ky");
+	g_assert_true(c.personal);
+	g_assert_cmpuint(c.frequency,==,13);
+	g_assert_cmpuint(c.tstamp,==,12345);
+	g_assert_cmpuint(c.message_date,==,1645215014);
+
+	assert_equal(c.display_name(), "Bli ky <bar@example.com>");
+}
+
+
+
+static void
+test_make_contacts()
+{
+	const auto str = "Abc <boo@example.com>, "
+		"Def <baa@example.com>, "
+		"Ghi <zzz@example.com>";
+	InternetAddressList *lst{
+		internet_address_list_parse(NULL, str)};
+
+	g_assert_true(lst);
+	const auto addrs{make_contacts(lst, Field::Id::Cc, 54321 )};
+	g_object_unref(lst);
+
+	g_assert_cmpuint(addrs.size(),==,3);
+
+	const auto addrs2{make_contacts(str, Field::Id::To, 12345 )};
+	g_assert_cmpuint(addrs2.size(),==,3);
+
+	assert_equal(addrs2[0].name, "Abc");
+	assert_equal(addrs2[0].email, "boo@example.com");
+	assert_equal(addrs2[1].name, "Def");
+	assert_equal(addrs2[1].email, "baa@example.com");
+	assert_equal(addrs2[2].name, "Ghi");
+	assert_equal(addrs2[2].email, "zzz@example.com");
+}
+
+static void
+test_make_contacts_2()
+{
+	const auto str = "Äbc <boo@example.com>, "
+		"De\nf <baa@example.com>, "
+		"\tGhi <zzz@example.com>";
+
+	const auto addrs2{make_contacts(str, Field::Id::Bcc, 12345 )};
+	g_assert_cmpuint(addrs2.size(),==,3);
+
+	assert_equal(addrs2[0].name, "Äbc");
+	assert_equal(addrs2[0].email, "boo@example.com");
+	assert_equal(addrs2[1].name, "De f");
+	assert_equal(addrs2[1].email, "baa@example.com");
+	assert_equal(addrs2[2].name, "Ghi");
+	assert_equal(addrs2[2].email, "zzz@example.com");
+}
+
+
+static void
+test_make_contacts_err()
+{
+	allow_warnings();
+
+	InternetAddressList *lst{ internet_address_list_parse(NULL, "")};
+	g_assert_false(lst);
+
+	const auto addrs{make_contacts("", Field::Id::To, 77777)};
+	g_assert_true(addrs.empty());
+}
+
+int
+main(int argc, char* argv[])
+{
+	g_test_init(&argc, &argv, NULL);
+	g_mime_init();
+
+	g_test_add_func("/message/contact/ctor-foo", test_ctor_foo);
+	g_test_add_func("/message/contact/ctor-blinky", test_ctor_blinky);
+	g_test_add_func("/message/contact/ctor-cleanup", test_ctor_cleanup);
+	g_test_add_func("/message/contact/make-contacts", test_make_contacts);
+	g_test_add_func("/message/contact/make-contacts-2", test_make_contacts_2);
+	g_test_add_func("/message/contact/make-contacts-err", test_make_contacts_err);
+
+	return g_test_run();
+}
+#endif /*BUILD_TESTS*/

lib/message/mu-contact.hh

@@ -0,0 +1,177 @@
+/*
+** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify it
+** under the terms of the GNU General Public License as published by the
+** Free Software Foundation; either version 3, or (at your option) any
+** later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+**
+*/
+
+#ifndef MU_MESSAGE_CONTACT_HH__
+#define MU_MESSAGE_CONTACT_HH__
+
+#include <functional>
+#include <string>
+#include <vector>
+#include <functional>
+#include <cctype>
+#include <cstring>
+#include <cstdlib>
+#include <ctime>
+
+#include "mu-fields.hh"
+
+struct _InternetAddressList;
+
+namespace Mu {
+
+/**
+ * Get the hash value for a lowercase value of s; useful for email-addresses
+ *
+ * @param s a string
+ *
+ * @return a hash value.
+ */
+size_t lowercase_hash(const std::string& s);
+
+struct Contact {
+	/**
+	 * Construct a new Contact
+	 *
+	 * @param email_ email address
+	 * @param name_ name or empty
+	 * @param field_id_ contact field id, or {}
+	 * @param message_date_ data for the message for this contact
+	 */
+	Contact(const std::string& email_, const std::string& name_ = "",
+		       std::optional<Field::Id> field_id_ = {},
+		       time_t message_date_ = 0)
+	    : email{email_}, name{name_}, field_id{field_id_},
+	      message_date{message_date_}, personal{}, frequency{1}, tstamp{}
+		{ cleanup_name(); }
+
+	/**
+	 * Construct a new Contact
+	 *
+	 * @param email_ email address
+	 * @param name_ name or empty
+	 * @param message_date_ date of message this contact originate from
+	 * @param personal_ is this a personal contact?
+	 * @param freq_ how often was this contact seen?
+	 * @param tstamp_ timestamp for last change
+	 */
+	Contact(const std::string& email_, const std::string& name_,
+		       time_t message_date_, bool personal_, size_t freq_,
+		       int64_t tstamp_)
+	    : email{email_}, name{name_}, field_id{},
+	      message_date{message_date_}, personal{personal_}, frequency{freq_},
+	      tstamp{tstamp_}
+	{ cleanup_name();}
+
+	/**
+	 * Get the "display name" for this contact; basically, if there's a
+	 * non-empty name, it's
+	 *     Jane Doe <email@example.com>
+	 * otherwise it's just the e-mail address.
+	 *
+	 * @return the display name
+	 */
+	std::string display_name() const;
+
+	/**
+	 * Operator==; based on the hash values (ie. lowercase e-mail address)
+	 *
+	 * @param rhs some other Contact
+	 *
+	 * @return true orf false.
+	 */
+	bool operator== (const Contact& rhs) const noexcept {
+		return hash() == rhs.hash();
+	}
+
+	/**
+	 * Get a hash-value for this contact, which gets lazily calculated. This
+	 * is for use with container classes. This uses the _lowercase_ email
+	 * address.
+	 *
+	 * @return the hash
+	 */
+	size_t hash() const {
+		static size_t cached_hash;
+		if (cached_hash == 0) {
+			cached_hash = lowercase_hash(email);
+		}
+		return  cached_hash;
+	}
+
+	/*
+	 * data members
+	 */
+
+	std::string email;                        /**< Email address for this contact.Not empty */
+	std::string name;                         /**< Name for this contact; can be empty. */
+	std::optional<Field::Id> field_id;  /**< Field Id of contact or nullopt */
+	::time_t    message_date;                 /**< date of the message from which the
+						     * contact originates */
+	bool    personal;                         /**<  A personal message? */
+	size_t  frequency;                        /**< Frequency of this contact */
+	int64_t tstamp;                           /**< Timestamp for this contact */
+
+private:
+	void cleanup_name() { // replace control characters by spaces.
+		for (auto& c: name)
+			if (iscntrl(c))
+				c = ' ';
+	}
+};
+
+using Contacts = std::vector<Contact>;
+
+/**
+ * Create a sequence of Contact objects from an InternetAddressList
+ *
+ * @param addr_lst an address list
+ * @param field_id the field_id for message field for these addresses
+ * @param message_date the date of the message from which the InternetAddressList
+ * originates.
+ *
+ * @return a sequence of Contact objects.
+ */
+Contacts
+make_contacts(/*const*/ struct _InternetAddressList* addr_lst,
+	      Field::Id field_id, ::time_t message_date);
+
+/**
+ * Create a sequence of Contact objects from an InternetAddressList
+ *
+ * @param addrs a string with one more valid addresses (as per internet_address_list_parse())
+ * @param field_id the field_id for message field for these addresses
+ * @param message_date the date of the message from which the addresses originate
+ *
+ * @return a sequence of Contact objects.
+ */
+Contacts
+make_contacts(const std::string& addrs,
+	      Field::Id field_id, ::time_t message_date);
+} // namespace Mu
+
+/**
+ * Implement our hash int std::
+ */
+template<> struct std::hash<Mu::Contact> {
+	std::size_t operator()(const Mu::Contact& c) const noexcept {
+		return c.hash();
+	}
+};
+
+#endif /* MU_CONTACT_HH__ */

lib/message/mu-document.cc

@@ -0,0 +1,346 @@
+/*
+** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify it
+** under the terms of the GNU General Public License as published by the
+** Free Software Foundation; either version 3, or (at your option) any
+** later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+**
+*/
+
+#include "mu-document.hh"
+#include "mu-message.hh"
+
+#include <cstdint>
+#include <glib.h>
+#include <numeric>
+#include <algorithm>
+#include <charconv>
+#include <cinttypes>
+
+#include <utils/mu-utils.hh>
+
+
+using namespace Mu;
+
+constexpr char SepaChar1 = 0xfe;
+constexpr char SepaChar2 = 0xff;
+
+static void
+add_index_term(Xapian::Document& doc, const Field& field, const std::string& val)
+{
+	std::string flatval{utf8_flatten(val)};
+	Xapian::TermGenerator termgen;
+	termgen.set_document(doc);
+	termgen.index_text(flatval);
+}
+
+static void
+maybe_add_term(Xapian::Document& doc, const Field& field, const std::string& val)
+{
+	if (field.is_normal_term())
+		doc.add_term(field.xapian_term());
+	else if (field.is_indexable_term()) {
+		add_index_term(doc, field, val);
+	} else if (field.is_boolean_term())
+		doc.add_boolean_term(field.xapian_term(val));
+}
+
+void
+Document::add(Field::Id id, const std::string& val)
+{
+	const auto field{field_from_id(id)};
+
+	if (field.is_value())
+		xdoc_.add_value(field.value_no(), val);
+
+	maybe_add_term(xdoc_, field, val);
+}
+
+void
+Document::add(Field::Id id, const std::vector<std::string>& vals)
+{
+	const auto field{field_from_id(id)};
+
+	if (field.is_value())
+		xdoc_.add_value(field.value_no(), Mu::join(vals, SepaChar1));
+
+	std::for_each(vals.begin(), vals.end(),
+		      [&](const auto& val) { maybe_add_term(xdoc_, field, val); });
+}
+
+
+std::vector<std::string>
+Document::string_vec_value(Field::Id field_id) const noexcept
+{
+	return Mu::split(string_value(field_id), SepaChar1);
+}
+
+void
+Document::add(Field::Id id, const Contacts& contacts)
+{
+	const auto field{field_from_id(id)};
+	std::vector<std::string> cvec;
+
+	const std::string sepa2(1, SepaChar2);
+
+	for (auto&& contact: contacts) {
+		if (!contact.field_id || *contact.field_id != id)
+			continue;
+
+		xdoc_.add_term(contact.email);
+		if (!contact.name.empty())
+			add_index_term(xdoc_, field, contact.name);
+
+		cvec.emplace_back(contact.email + sepa2 + contact.name);
+	}
+
+	if (!cvec.empty())
+		xdoc_.add_value(field.value_no(), join(cvec, SepaChar1));
+}
+
+Contacts
+Document::contacts_value(Field::Id id) const noexcept
+{
+	const auto vals{string_vec_value(id)};
+	Contacts contacts;
+	contacts.reserve(vals.size());
+
+	for (auto&& s: vals) {
+
+		const auto pos = s.find(SepaChar2);
+		if (G_UNLIKELY(pos == std::string::npos)) {
+			g_critical("invalid contact data '%s'", s.c_str());
+			break;
+		}
+
+		contacts.emplace_back(s.substr(0, pos), s.substr(pos + 1), id);
+	}
+
+	return contacts;
+}
+
+static std::string
+integer_to_string(int64_t val)
+{
+	char buf[18];
+	buf[0] = 'f' + ::snprintf(buf + 1, sizeof(buf) - 1, "%" PRIx64, val);
+	return buf;
+}
+
+static int64_t
+string_to_integer(const std::string& str)
+{
+	if (str.empty())
+		return 0;
+
+	int64_t val{};
+	std::from_chars(str.c_str() + 1, str.c_str() + str.size(), val, 16);
+
+	return val;
+}
+
+void
+Document::add(Field::Id id, int64_t val)
+{
+	/*
+	 * Xapian stores everything (incl. numbers) as strings.
+	 *
+	 * we comply, by storing a number a base-16 and prefixing with 'f' +
+	 * length; such that the strings are sorted in the numerical order.
+	 */
+
+	const auto field{field_from_id(id)};
+
+	if (field.is_value())
+		xdoc_.add_value(field.value_no(), integer_to_string(val));
+
+	/* terms are not supported for numerical fields */
+}
+
+int64_t
+Document::integer_value(Field::Id field_id) const noexcept
+{
+	return string_to_integer(string_value(field_id));
+}
+
+void
+Document::add(Priority prio)
+{
+	constexpr auto field{field_from_id(Field::Id::Priority)};
+
+	xdoc_.add_value(field.value_no(), std::string(1, to_char(prio)));
+	xdoc_.add_boolean_term(field.xapian_term(to_char(prio)));
+}
+
+
+Priority
+Document::priority_value() const noexcept
+{
+	const auto val{string_value(Field::Id::Priority)};
+	return priority_from_char(val.empty() ? 'n' : val[0]);
+}
+
+void
+Document::add(Flags flags)
+{
+	constexpr auto field{field_from_id(Field::Id::Flags)};
+
+	xdoc_.add_value(field.value_no(), integer_to_string(static_cast<int64_t>(flags)));
+	flag_infos_for_each([&](auto&& flag_info) {
+		if (any_of(flag_info.flag & flags))
+			xdoc_.add_boolean_term(field.xapian_term(flag_info.shortcut_lower()));
+	});
+}
+
+Flags
+Document::flags_value() const noexcept
+{
+	return static_cast<Flags>(integer_value(Field::Id::Flags));
+}
+
+
+
+#ifdef BUILD_TESTS
+
+#define assert_same_contact(C1,C2) do {				\
+	g_assert_cmpstr(C1.email.c_str(),==,C2.email.c_str());	\
+	g_assert_cmpstr(C2.name.c_str(),==,C2.name.c_str());	\
+	} while (0)
+
+#define assert_same_contacts(CV1,CV2) do {			\
+	g_assert_cmpuint(CV1.size(),==,CV2.size());		\
+	for (auto i = 0U; i != CV1.size(); ++i)			\
+		assert_same_contact(CV1[i], CV2[i]);		\
+	} while(0)
+
+
+
+static const Contacts test_contacts = {{
+		Contact{"john@example.com", "John", Field::Id::Bcc},
+		Contact{"ringo@example.com", "Ringo",  Field::Id::Bcc},
+		Contact{"paul@example.com", "Paul", Field::Id::Cc},
+		Contact{"george@example.com", "George",  Field::Id::Cc},
+		Contact{"james@example.com", "James", Field::Id::From},
+		Contact{"lars@example.com", "Lars",  Field::Id::To},
+		Contact{"kirk@example.com", "Kirk", Field::Id::To},
+		Contact{"jason@example.com", "Jason",  Field::Id::To}
+	}};
+
+static void
+test_bcc()
+{
+	{
+		Document doc;
+		doc.add(Field::Id::Bcc, test_contacts);
+
+		Contacts expected_contacts = {{
+				Contact{"john@example.com", "John", Field::Id::Bcc},
+				Contact{"ringo@example.com", "Ringo",  Field::Id::Bcc},
+			}};
+		const auto actual_contacts = doc.contacts_value(Field::Id::Bcc);
+		assert_same_contacts(expected_contacts, actual_contacts);
+	}
+
+	{
+		Document doc;
+		Contacts contacts = {{
+				Contact{"john@example.com", "John Lennon", Field::Id::Bcc},
+				Contact{"ringo@example.com", "Ringo",  Field::Id::Bcc},
+			}};
+		doc.add(Field::Id::Bcc, contacts);
+		auto db = Xapian::InMemory::open();
+
+		db.add_document(doc.xapian_document());
+
+	}
+
+}
+
+static void
+test_cc()
+{
+	Document doc;
+	doc.add(Field::Id::Cc, test_contacts);
+
+	Contacts expected_contacts = {{
+			Contact{"paul@example.com", "Paul", Field::Id::Cc},
+			Contact{"george@example.com", "George",  Field::Id::Cc}
+		}};
+	const auto actual_contacts = doc.contacts_value(Field::Id::Cc);
+
+	assert_same_contacts(expected_contacts, actual_contacts);
+}
+
+
+static void
+test_from()
+{
+	Document doc;
+	doc.add(Field::Id::From, test_contacts);
+
+	Contacts expected_contacts = {{
+			Contact{"james@example.com", "James", Field::Id::From},
+		}};
+	const auto actual_contacts = doc.contacts_value(Field::Id::From);
+
+	assert_same_contacts(expected_contacts, actual_contacts);
+}
+
+static void
+test_to()
+{
+	Document doc;
+	doc.add(Field::Id::To, test_contacts);
+
+	Contacts expected_contacts = {{
+		Contact{"lars@example.com", "Lars",  Field::Id::To},
+		Contact{"kirk@example.com", "Kirk", Field::Id::To},
+		Contact{"jason@example.com", "Jason",  Field::Id::To}
+		}};
+	const auto actual_contacts = doc.contacts_value(Field::Id::To);
+
+	assert_same_contacts(expected_contacts, actual_contacts);
+}
+
+
+static void
+test_size()
+{
+	{
+		Document doc;
+		doc.add(Field::Id::Size, 12345);
+		g_assert_cmpuint(doc.integer_value(Field::Id::Size),==,12345);
+	}
+
+	{
+		Document doc;
+		g_assert_cmpuint(doc.integer_value(Field::Id::Size),==,0);
+	}
+}
+
+
+int
+main(int argc, char* argv[])
+{
+	g_test_init(&argc, &argv, NULL);
+
+	g_test_add_func("/message/document/bcc", test_bcc);
+	g_test_add_func("/message/document/cc", test_cc);
+	g_test_add_func("/message/document/from", test_from);
+	g_test_add_func("/message/document/to", test_to);
+
+	g_test_add_func("/message/document/size", test_size);
+
+	return g_test_run();
+}
+#endif /*BUILD_TESTS*/

lib/message/mu-document.hh

@@ -0,0 +1,216 @@
+/** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify it
+** under the terms of the GNU General Public License as published by the
+** Free Software Foundation; either version 3, or (at your option) any
+** later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+**
+*/
+
+#ifndef MU_DOCUMENT_HH__
+#define MU_DOCUMENT_HH__
+
+#include <xapian.h>
+#include <utility>
+#include <string>
+#include <vector>
+#include "utils/mu-xapian-utils.hh"
+
+#include "mu-fields.hh"
+#include "mu-priority.hh"
+#include "mu-flags.hh"
+#include "mu-contact.hh"
+
+namespace Mu {
+
+/**
+ * A Document describes the information about a message that is
+ * or can be stored in the database.
+ *
+ */
+class Document {
+public:
+	/**
+	 * Construct a message for a new Xapian Document
+	 *
+	 */
+	Document() {}
+
+	/**
+	 * Construct a message document based on on existing Xapian document.
+	 *
+	 * @param doc
+	 */
+	Document(const Xapian::Document& doc): xdoc_{doc} {}
+
+	/**
+	 * Copy CTOR
+	 */
+	Document(const Document& rhs) { *this = rhs; }
+	/**
+	 * Move CTOR
+	 *
+	 */
+	Document(Document&& rhs) {*this = std::move(rhs); }
+
+	/**
+	 * Get a reference to the underlying Xapian document.
+	 *
+	 */
+	const Xapian::Document& xapian_document() const { return xdoc_; }
+
+	/* Copy assignment operator
+	 *
+	 * @param rhs some message
+	 *
+	 * @return a message ref
+	 */
+	Document& operator=(const Document& rhs) {
+		if (this != &rhs)
+			xdoc_ = rhs.xdoc_;
+		return *this;
+	}
+
+	/**
+	 * Move assignment operator
+	 *
+	 * @param rhs some message
+	 *
+	 * @return a message ref
+	 */
+	Document& operator=(Document&& rhs) {
+		if (this != &rhs)
+			xdoc_ = std::move(rhs.xdoc_);
+		return *this;
+	}
+
+
+	/*
+	 * updating a document with terms & values
+	 */
+
+	/**
+	 * Add a string value to the document
+	 *
+	 * @param field_id field id
+	 * @param val string value
+	 */
+	void	add(Field::Id field_id, const std::string& val);
+
+	/**
+	 * Add a string-vec value to the document
+	 *
+	 * @param field_id field id
+	 * @param val string-vec value
+	 */
+	void	add(Field::Id field_id, const std::vector<std::string>& vals);
+
+
+	/**
+	 * Add message-contacts to the document
+	 *
+	 * @param field_id field id
+	 * @param contacts message contacts
+	 */
+	void    add(Field::Id id, const Contacts& contacts);
+
+	/**
+	 * Add an integer value to the document
+	 *
+	 * @param field_id field id
+	 * @param val integer value
+	 */
+	void	add(Field::Id field_id, int64_t val);
+
+	/**
+	 * Add a message priority to the document
+	 *
+	 * @param prio priority
+	 */
+	void	add(Priority prio);
+
+
+	/**
+	 * Add message flags to the document
+	 *
+	 * @param flags mesage flags.
+	 */
+	void	add(Flags flags);
+
+	/*
+	 * Retrieving values
+	 */
+
+
+	/**
+	 * Get a message-field as a string-value
+	 *
+	 * @param field_id id of the field to get.
+	 *
+	 * @return a string (empty if not found)
+	 */
+	std::string string_value(Field::Id field_id) const noexcept {
+		return xapian_try([&]{
+			return xdoc_.get_value(field_from_id(field_id).value_no());
+		}, std::string{});
+	}
+	/**
+	 * Get a vec of string values.
+	 *
+	 * @param field_id id of the field to get
+	 *
+	 * @return a string list
+	 */
+	std::vector<std::string> string_vec_value(Field::Id field_id) const noexcept;
+
+
+	/**
+	 * Get an integer value
+	 *
+	 * @param field_id id of the field to get
+	 *
+	 * @return an integer or 0 if not found.
+	 */
+	int64_t integer_value(Field::Id field_id) const noexcept;
+
+
+	/**
+	 * Get contacts
+	 *
+	 * @param field_id id of the contacts field to get
+	 *
+	 * @return an integer or 0 if not found.
+	 */
+	Contacts contacts_value(Field::Id id) const noexcept;
+
+	/**
+	 * Get the priority
+	 *
+	 * @return the message priority
+	 */
+	Priority priority_value() const noexcept;
+
+	/**
+	 * Get the message flags
+	 *
+	 *
+	 * @return flags
+	 */
+	Flags    flags_value() const noexcept;
+
+private:
+	Xapian::Document xdoc_;
+};
+
+} // namepace Mu
+
+#endif /* MU_DOCUMENT_HH__ */

lib/message/mu-fields.cc

@@ -0,0 +1,164 @@
+/*
+** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify it
+** under the terms of the GNU General Public License as published by the
+** Free Software Foundation; either version 3, or (at your option) any
+** later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+**
+*/
+
+#include "mu-fields.hh"
+#include "mu-flags.hh"
+
+using namespace Mu;
+
+std::string
+Field::xapian_term(const std::string& s) const
+{
+	return std::string(1U, xapian_prefix()) + s;
+}
+
+std::string
+Field::xapian_term(std::string_view sv) const
+{
+	return std::string(1U, xapian_prefix()) + std::string{sv};
+}
+std::string
+Field::xapian_term(char c) const
+{
+	return std::string(1U, xapian_prefix()) + c;
+}
+
+
+
+/**
+ * compile-time checks
+ */
+constexpr bool
+validate_field_ids()
+{
+	for (auto id = 0U; id != Field::id_size(); ++id) {
+		const auto field_id = static_cast<Field::Id>(id);
+		if (field_from_id(field_id).id != field_id)
+			return false;
+	}
+	return true;
+}
+
+constexpr bool
+validate_field_shortcuts()
+{
+	for (auto id = 0U; id != Field::id_size(); ++id) {
+		const auto field_id = static_cast<Field::Id>(id);
+		const auto shortcut = field_from_id(field_id).shortcut;
+		if (shortcut != 0 &&
+		    (shortcut < 'a' || shortcut > 'z'))
+			return false;
+	}
+	return true;
+}
+
+
+constexpr /*static*/ bool
+validate_field_flags()
+{
+	for (auto&& field: Fields) {
+		/* - A field has at most one of Indexable, HasTerms, IsXapianBoolean and
+		   IsContact. */
+		size_t flagnum{};
+		if (field.is_indexable_term())
+			++flagnum;
+		if (field.is_boolean_term())
+			++flagnum;
+		if (field.is_normal_term())
+			++flagnum;
+		if (field.is_contact())
+			++flagnum;
+
+		if (flagnum > 1) {
+			//g_warning("invalid field %*s", STR_V(field.name));
+			return false;
+		}
+	}
+
+	return true;
+}
+
+
+
+/*
+ * tests... also build as runtime-tests, so we can get coverage info
+ */
+#ifdef BUILD_TESTS
+#define static_assert g_assert_true
+#endif /*BUILD_TESTS*/
+
+
+[[maybe_unused]]
+static void
+test_ids()
+{
+	static_assert(validate_field_ids());
+}
+
+[[maybe_unused]]
+static void
+test_shortcuts()
+{
+	static_assert(validate_field_shortcuts());
+}
+
+[[maybe_unused]]
+static void
+test_prefix()
+{
+	static_assert(field_from_id(Field::Id::Subject).xapian_prefix() == 'S');
+	static_assert(field_from_id(Field::Id::BodyHtml).xapian_prefix() == 0);
+}
+
+[[maybe_unused]]
+static void
+test_field_flags()
+{
+	static_assert(validate_field_flags());
+}
+
+#ifdef BUILD_TESTS
+
+static void
+test_xapian_term()
+{
+	using namespace std::string_literals;
+	using namespace std::literals;
+
+	assert_equal(field_from_id(Field::Id::Subject).xapian_term(""s), "S");
+	assert_equal(field_from_id(Field::Id::Subject).xapian_term("boo"s), "Sboo");
+
+	assert_equal(field_from_id(Field::Id::From).xapian_term('x'), "Fx");
+	assert_equal(field_from_id(Field::Id::To).xapian_term("boo"sv), "Tboo");
+}
+
+int
+main(int argc, char* argv[])
+{
+	g_test_init(&argc, &argv, NULL);
+
+	g_test_add_func("/message/fields/ids", test_ids);
+	g_test_add_func("/message/fields/shortcuts", test_shortcuts);
+	g_test_add_func("/message/fields/prefix", test_prefix);
+	g_test_add_func("/message/fields/xapian-term", test_xapian_term);
+	g_test_add_func("/message/fields/flags", test_field_flags);
+
+	return g_test_run();
+}
+#endif /*BUILD_TESTS*/

lib/message/mu-fields.hh

@@ -0,0 +1,553 @@
+/*
+** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify it
+** under the terms of the GNU General Public License as published by the
+** Free Software Foundation; either version 3, or (at your option) any
+** later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+**
+*/
+
+#ifndef MU_FIELDS_HH__
+#define MU_FIELDS_HH__
+
+#include <cstdint>
+#include <string_view>
+#include <algorithm>
+#include <array>
+#include <optional>
+#include <xapian.h>
+#include <utils/mu-utils.hh>
+
+namespace Mu {
+
+struct Field {
+	/**
+	 * Field Ids.
+	 *
+	 * Note, the Ids are also used as indices in the Fields array,
+	 * so their numerical values must be 0...Count.
+	 *
+	 */
+	enum struct Id {
+		/*
+		 * first all the string-based ones
+		 */
+		Bcc = 0,      /**< Blind Carbon-Copy */
+		BodyHtml,     /**< HTML Body */
+		BodyText,     /**< Text body */
+		Cc,           /**< Carbon-Copy */
+		EmbeddedText, /**< Embedded text in message */
+		File,         /**< Filename */
+		From,         /**< Message sender */
+		Maildir,      /**< Maildir path */
+		Mime,         /**< MIME-Type */
+		MessageId,    /**< Message Id */
+		Path,         /**< File-system Path */
+		Subject,      /**< Message subject */
+		To,           /**< To: recipient */
+		Uid,          /**< Unique id for message (based on path) */
+		/*
+		 * string list items...
+		 */
+		References, /**< All references (incl. Reply-To:) */
+		Tags,       /**< Message Tags */
+		/*
+		 * then the numerical ones
+		 */
+		Date,     /**< Message date */
+		Flags,    /**< Message flags */
+		Priority, /**< Message priority */
+		Size,     /**< Message size (in bytes) */
+
+		/* add new ones here... */
+		MailingList, /**< Mailing list */
+		ThreadId,    /**< Thread Id */
+
+		/*
+		 * <private>
+		 */
+		_count_ /**< Number of FieldIds */
+	};
+
+	/**
+	 * Get the number of Id values.
+	 *
+	 * @return the number.
+	 */
+	static constexpr size_t id_size()
+	{
+		return static_cast<size_t>(Id::_count_);
+	}
+
+	constexpr Xapian::valueno value_no() const {
+		return static_cast<Xapian::valueno>(id);
+	}
+
+	/**
+	 * Field types
+	 *
+	 */
+	enum struct Type {
+		String,     /**< String */
+		StringList, /**< List of strings */
+		ByteSize,   /**< Size in bytes */
+		TimeT,      /**< A time_t value */
+		Integer,    /**< An integer */
+	};
+
+	constexpr bool is_string() const { return type == Type::String; }
+	constexpr bool is_string_list() const { return type == Type::StringList; }
+	constexpr bool is_byte_size() const { return type == Type::ByteSize; }
+	constexpr bool is_time_t() const { return type == Type::TimeT; }
+	constexpr bool is_integer() const { return type == Type::Integer; }
+	constexpr bool is_numerical() const { return is_byte_size() || is_time_t() || is_integer(); }
+
+	/**
+	 * Field flags
+	 * note: the differences for our purposes between a xapian field and a
+	 * term: - there is only a single value for some item in per document
+	 * (msg), ie.  one value containing the list of To: addresses - there
+	 * can be multiple terms, each containing e.g. one of the To:
+	 * addresses - searching uses terms, but to display some field, it
+	 * must be in the value (at least when using MuMsgIter)
+	 *
+	 * Rules (build-time enforced):
+	 * - A field has at most one of Indexable, HasTerms, IsXapianBoolean and IsContact.
+	 */
+
+	enum struct Flag {
+		GMime	       = 1 << 0,
+		/**< Field retrieved through gmime */
+
+		/*
+		 * Different kind of terms; at most one is true,
+		 * and cannot be combined with IsContact. Compile-time enforced.
+		 */
+		NormalTerm	= 1 << 2,
+		/**< Field is a searchable term */
+		BooleanTerm	= 1 << 5,
+		/**< Field is a boolean search-term; wildcards do not work */
+		IndexableTerm = 1 << 1,
+		/**< Field has indexable text as term */
+
+		/*
+		 * Contact flag cannot be combined with any of the term flags.
+		 * This is compile-time enforced.
+		 */
+		Contact      = 1 << 4,
+		/**< field contains one or more e-mail-addresses */
+
+		Value       = 1 << 3,
+		/**< Field value is stored (so the literal value can be retrieved) */
+
+		DoNotCache     = 1 << 6,
+		/**< don't cache this field in * the MuMsg cache */
+		Range	       = 1 << 7
+		/**< whether this is a range field (e.g., date, size)*/
+	};
+
+	constexpr bool any_of(Flag some_flag) const{
+		return (static_cast<int>(some_flag) & static_cast<int>(flags)) != 0;
+	}
+
+	constexpr bool is_gmime() const { return any_of(Flag::GMime); }
+
+	constexpr bool is_indexable_term()	const { return any_of(Flag::IndexableTerm); }
+	constexpr bool is_boolean_term()	const { return any_of(Flag::BooleanTerm); }
+	constexpr bool is_normal_term()		const { return any_of(Flag::NormalTerm); }
+
+	constexpr bool is_searchable()          const { return is_indexable_term() ||
+							       is_boolean_term() ||
+							       is_normal_term(); }
+
+	constexpr bool is_value()		const { return any_of(Flag::Value); }
+
+	constexpr bool is_contact()		const { return any_of(Flag::Contact); }
+	constexpr bool is_range()		const { return any_of(Flag::Range); }
+	constexpr bool do_not_cache()		const { return any_of(Flag::DoNotCache); }
+
+
+
+	/**
+	 * Field members
+	 *
+	 */
+	Id               id;            /**< Id of the message field */
+	Type             type;          /**< Type of the message field */
+	std::string_view name;          /**< Name of the message field */
+	std::string_view description;   /**< Decription of the message field */
+	std::string_view example_query; /**< Example query */
+	char             shortcut;      /**< Shortcut for the message field; a..z */
+	Flag             flags;         /**< Flags */
+
+	/**
+	 * Convenience / helpers
+	 *
+	 */
+
+	constexpr char xapian_prefix() const
+	{ /* xapian uses uppercase shortcuts; toupper is not constexpr */
+		return shortcut == 0 ? 0 : shortcut - ('a' - 'A');
+	}
+
+	std::string xapian_term(const std::string& s="") const;
+	std::string xapian_term(std::string_view sv) const;
+	std::string xapian_term(char c) const;
+};
+
+MU_ENABLE_BITOPS(Field::Flag);
+
+/**
+ * Sequence of _all_ message fields
+ */
+static constexpr std::array<Field, Field::id_size()>
+    Fields = {
+	{
+	    // Bcc
+	    {
+		Field::Id::Bcc,
+		Field::Type::String,
+		"bcc",
+		"Blind carbon-copy recipient",
+		"bcc:foo@example.com",
+		'h',
+		Field::Flag::GMime |
+		Field::Flag::Contact |
+		Field::Flag::Value
+	    },
+	    // HTML Body
+	    {
+		Field::Id::BodyHtml,
+		Field::Type::String,
+		"body",
+		"Message html body",
+		{},
+		{},
+		Field::Flag::GMime |
+		Field::Flag::DoNotCache
+	    },
+	    // Body
+	    {
+		Field::Id::BodyText,
+		Field::Type::String,
+		"body",
+		"Message plain-text body",
+		"body:capybara", // example
+		'b',
+		Field::Flag::GMime |
+		Field::Flag::IndexableTerm |
+		Field::Flag::DoNotCache
+	    },
+	    // Cc
+	    {
+		Field::Id::Cc,
+		Field::Type::String,
+		"cc",
+		"Carbon-copy recipient",
+		"cc:quinn@example.com",
+		'c',
+		Field::Flag::GMime |
+		Field::Flag::Contact |
+		Field::Flag::Value},
+
+	    // Embed
+	    {
+		Field::Id::EmbeddedText,
+		Field::Type::String,
+		"embed",
+		"Embedded text",
+		"embed:war OR embed:peace",
+		'e',
+		Field::Flag::GMime |
+		Field::Flag::IndexableTerm |
+		Field::Flag::DoNotCache},
+	    // File
+	    {
+		Field::Id::File,
+		Field::Type::String,
+		"file",
+		"Attachment file name",
+		"file:/image\\.*.jpg/",
+		'j',
+		Field::Flag::GMime |
+		Field::Flag::NormalTerm |
+		Field::Flag::DoNotCache},
+
+	    // From
+	    {
+		Field::Id::From,
+		Field::Type::String,
+		"from",
+		"Message sender",
+		"from:jimbo",
+		'f',
+		Field::Flag::GMime |
+		Field::Flag::Contact |
+		Field::Flag::Value},
+	    // Maildir
+	    {
+		Field::Id::Maildir,
+		Field::Type::String,
+		"maildir",
+		"Maildir path for message",
+		"maildir:/private/archive",
+		'm',
+		Field::Flag::GMime |
+		Field::Flag::NormalTerm |
+		Field::Flag::Value},
+	    // MIME
+	    {
+		Field::Id::Mime,
+		Field::Type::String,
+		"mime",
+		"Attachment MIME-type",
+		"mime:image/jpeg",
+		'y',
+		Field::Flag::NormalTerm},
+	    // Message-ID
+	    {
+		Field::Id::MessageId,
+		Field::Type::String,
+		"msgid",
+		"Attachment MIME-type",
+		"mime:image/jpeg",
+		'i',
+		Field::Flag::GMime |
+		Field::Flag::NormalTerm |
+		Field::Flag::Value},
+	    // Path
+	    {
+		Field::Id::Path,
+		Field::Type::String,
+		"path",
+		"File system path to message",
+		{},
+		'p',
+		Field::Flag::GMime |
+		Field::Flag::BooleanTerm |
+		Field::Flag::Value},
+
+	    // Subject
+	    {
+		Field::Id::Subject,
+		Field::Type::String,
+		"subject",
+		"Message subject",
+		"subject:wombat",
+		's',
+		Field::Flag::GMime |
+		Field::Flag::Value |
+		Field::Flag::IndexableTerm},
+
+	    // To
+	    {
+		Field::Id::To,
+		Field::Type::String,
+		"to",
+		"Message recipient",
+		"to:flimflam@example.com",
+		't',
+		Field::Flag::GMime |
+		Field::Flag::Contact |
+		Field::Flag::Value
+	    },
+	    // UID (internal)
+	    {
+		Field::Id::Uid,
+		Field::Type::String,
+		"uid",
+		"Message recipient",
+		{},
+		'u',
+		Field::Flag::NormalTerm},
+
+	    // References
+	    {
+		Field::Id::References,
+		Field::Type::StringList,
+		"refs",
+		"Message references to other messages",
+		{},
+		'r',
+		Field::Flag::GMime |
+		Field::Flag::Value
+	    },
+	    // Tags
+	    {
+		Field::Id::Tags,
+		Field::Type::StringList,
+		"tag",
+		"Message tags",
+		"tag:projectx",
+		'x',
+		Field::Flag::GMime |
+		Field::Flag::NormalTerm |
+		Field::Flag::Value
+	    },
+	    // Date
+	    {
+		Field::Id::Date,
+		Field::Type::TimeT,
+		"date",
+		"Message date",
+		"date:20220101..20220505",
+		'd',
+		Field::Flag::GMime |
+		Field::Flag::Value |
+		Field::Flag::Range
+	    },
+	    // Flags
+	    {
+		Field::Id::Flags,
+		Field::Type::Integer,
+		"flag",
+		"Message properties",
+		"flag:unread",
+		'g',
+		Field::Flag::GMime |
+		Field::Flag::NormalTerm |
+		Field::Flag::Value
+	    },
+	    // Priority
+	    {
+		Field::Id::Priority,
+		Field::Type::Integer,
+		"prio",
+		"Priority",
+		"prio:high",
+		'p',
+		Field::Flag::GMime |
+		Field::Flag::NormalTerm |
+		Field::Flag::Value
+	    },
+	    // Size
+	    {
+		Field::Id::Size,
+		Field::Type::ByteSize,
+		"size",
+		"Message size in bytes",
+		"size:1M..5M",
+		'z',
+		Field::Flag::GMime |
+		Field::Flag::Value |
+		Field::Flag::Range
+	    },
+	    // Mailing List
+	    {
+		Field::Id::MailingList,
+		Field::Type::String,
+		"list",
+		"Mailing list (List-Id:)",
+		"list:mu-discuss.googlegroups.com",
+		'v',
+		Field::Flag::GMime |
+		Field::Flag::NormalTerm |
+		Field::Flag::Value
+	    },
+	    // ThreadId
+	    {
+		Field::Id::ThreadId,
+		Field::Type::String,
+		"thread",
+		"Thread a message belongs to",
+		{},
+		'w',
+		Field::Flag::NormalTerm
+	    },
+	}};
+
+/*
+ * Convenience
+ */
+
+/**
+ * Get the message field for the given Id.
+ *
+ * @param id of the message field
+ *
+ * @return ref of the message field.
+ */
+constexpr const Field&
+field_from_id(Field::Id id)
+{
+	return Fields.at(static_cast<size_t>(id));
+}
+
+/**
+ * Invoke func for each message-field
+ *
+ * @param func some callable
+ */
+template <typename Func>
+void field_for_each(Func&& func) {
+	for (const auto& field: Fields)
+		func(field);
+}
+
+/**
+ * Find a message field that satisfies some predicate
+ *
+ * @param pred the predicate (a callable)
+ *
+ * @return a message-field id, or nullopt if not found.
+ */
+template <typename Pred>
+std::optional<Field> field_find_if(Pred&& pred) {
+	for (auto&& field: Fields)
+		if (pred(field))
+			return field;
+	return std::nullopt;
+}
+
+/**
+ * Get the the message-field id for the given name or shortcut
+ *
+ * @param name_or_shortcut
+ *
+ * @return the message-field-id or nullopt.
+ */
+static inline
+std::optional<Field> field_from_shortcut(char shortcut) {
+	return field_find_if([&](auto&& field){
+		return field.shortcut == shortcut;
+	});
+}
+static inline
+std::optional<Field> field_from_name(const std::string& name) {
+	if (name.length() == 1)
+		return field_from_shortcut(name[0]);
+	else
+		return field_find_if([&](auto&& field){
+			return field.name == name;
+	});
+}
+
+/**
+ * Get the Field::Id for some number, or nullopt if it does not match
+ *
+ * @param id an id number
+ *
+ * @return Field::Id  or nullopt
+ */
+static inline
+std::optional<Field> field_from_number(size_t id)
+{
+	if (id >= static_cast<size_t>(Field::Id::_count_))
+		return std::nullopt;
+	else
+		return field_from_id(static_cast<Field::Id>(id));
+}
+
+
+} // namespace Mu
+#endif /* MU_FIELDS_HH__ */

lib/message/mu-flags.cc

@@ -0,0 +1,170 @@
+/*
+** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify it
+** under the terms of the GNU General Public License as published by the
+** Free Software Foundation; either version 3, or (at your option) any
+** later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+**
+*/
+
+/*
+ * implementation is almost completely in the header; here we just add some
+ * compile-time tests.
+ */
+
+#include "mu-flags.hh"
+
+using namespace Mu;
+
+std::string
+Mu::flags_to_string(Flags flags)
+{
+	std::string str;
+
+	for (auto&& info: AllMessageFlagInfos)
+		if (any_of(info.flag & flags))
+			str+=info.shortcut;
+
+	return str;
+}
+
+
+/*
+ * flags & flag-info
+ */
+constexpr bool
+validate_message_info_flags()
+{
+	for (auto id = 0U; id != AllMessageFlagInfos.size(); ++id) {
+		const auto flag = static_cast<Flags>(1 << id);
+		if (flag != AllMessageFlagInfos[id].flag)
+			return false;
+	}
+	return true;
+}
+
+
+/*
+ * tests... also build as runtime-tests, so we can get coverage info
+ */
+#ifdef BUILD_TESTS
+#define static_assert g_assert_true
+#endif /*BUILD_TESTS*/
+
+[[maybe_unused]] static void
+test_basic()
+{
+	static_assert(AllMessageFlagInfos.size() ==
+		      __builtin_ctz(static_cast<unsigned>(Flags::_final_)));
+	static_assert(validate_message_info_flags());
+
+	static_assert(!!flag_info(Flags::Encrypted));
+	static_assert(!flag_info(Flags::None));
+	static_assert(!flag_info(static_cast<Flags>(0)));
+	static_assert(!flag_info(static_cast<Flags>(1<<AllMessageFlagInfos.size())));
+}
+
+/*
+ * flag_info
+ */
+[[maybe_unused]] static void
+test_flag_info()
+{
+	static_assert(flag_info('D')->flag == Flags::Draft);
+	static_assert(flag_info('l')->flag == Flags::MailingList);
+	static_assert(!flag_info('q'));
+
+	static_assert(flag_info("trashed")->flag == Flags::Trashed);
+	static_assert(flag_info("attach")->flag == Flags::HasAttachment);
+	static_assert(!flag_info("fnorb"));
+
+
+	static_assert(flag_info('D')->shortcut_lower() == 'd');
+	static_assert(flag_info('u')->shortcut_lower() == 'u');
+}
+
+/*
+ * flags_from_expr
+ */
+[[maybe_unused]] static void
+test_flags_from_expr()
+{
+	static_assert(flags_from_absolute_expr("SRP").value() ==
+		      (Flags::Seen | Flags::Replied | Flags::Passed));
+	static_assert(flags_from_absolute_expr("Faul").value() ==
+		      (Flags::Flagged | Flags::Unread |
+		       Flags::HasAttachment | Flags::MailingList));
+
+	static_assert(!flags_from_absolute_expr("DRT?"));
+	static_assert(flags_from_absolute_expr("DRT?", true/*ignore invalid*/).value() ==
+		      (Flags::Draft | Flags::Replied |
+		       Flags::Trashed));
+	static_assert(flags_from_absolute_expr("DFPNxulabcdef", true/*ignore invalid*/).value() ==
+		      (Flags::Draft|Flags::Flagged|Flags::Passed|
+		       Flags::New | Flags::Encrypted |
+		       Flags::Unread | Flags::MailingList |
+		       Flags::HasAttachment));
+}
+
+
+/*
+ * flags_from_delta_expr
+ */
+[[maybe_unused]] static void
+test_flags_from_delta_expr()
+{
+	static_assert(flags_from_delta_expr(
+			      "+S-u-N", Flags::New|Flags::Unread).value() ==
+		      Flags::Seen);
+	static_assert(flags_from_delta_expr("+R+P-F", Flags::Seen).value() ==
+		      (Flags::Seen|Flags::Passed|Flags::Replied));
+	/* '-B' is invalid */
+	static_assert(!flags_from_delta_expr("+R+P-B", Flags::Seen));
+	/* '-B' is invalid, but ignore invalid */
+	static_assert(flags_from_delta_expr("+R+P-B", Flags::Seen, true) ==
+		      (Flags::Replied|Flags::Passed|Flags::Seen));
+	static_assert(flags_from_delta_expr("+F+T-S", Flags::None, true).value() ==
+		      (Flags::Flagged|Flags::Trashed));
+}
+
+/*
+ * flags_filter
+ */
+[[maybe_unused]] static void
+test_flags_filter()
+{
+	static_assert(flags_filter(flags_from_absolute_expr(
+						   "DFPNxulabcdef", true/*ignore invalid*/).value(),
+					   MessageFlagCategory::Mailfile) ==
+		      (Flags::Draft|Flags::Flagged|Flags::Passed));
+}
+
+
+#ifdef BUILD_TESTS
+int
+main(int argc, char* argv[])
+{
+	g_test_init(&argc, &argv, NULL);
+
+	g_test_add_func("/message/flags/basic", test_basic);
+	g_test_add_func("/message/flags/flag-info", test_flag_info);
+	g_test_add_func("/message/flags/flags-from-absolute-expr",
+			test_flags_from_expr);
+	g_test_add_func("/message/flags/flags-from-delta-expr",
+			test_flags_from_delta_expr);
+	g_test_add_func("/message/flags/flags-filter",
+			test_flags_filter);
+
+	return g_test_run();
+}
+#endif /*BUILD_TESTS*/

lib/message/mu-flags.hh

@@ -0,0 +1,321 @@
+/*
+** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify it
+** under the terms of the GNU General Public License as published by the
+** Free Software Foundation; either version 3, or (at your option) any
+** later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+**
+*/
+
+#ifndef MU_FLAGS_HH__
+#define MU_FLAGS_HH__
+
+#include <algorithm>
+#include <optional>
+#include <string_view>
+#include <array>
+#include "utils/mu-utils.hh"
+
+namespace Mu {
+
+enum struct Flags {
+	None = 0, /**< No flags */
+	/**
+	 * next 6 are seen in the file-info part of maildir message file
+	 * names, ie., in a name like "1234345346:2,<fileinfo>",
+	 * <fileinfo> consists of zero or more of the following
+	 * characters (in ascii order)
+	 */
+	Draft   = 1 << 0, /**< A draft message */
+	Flagged = 1 << 1, /**< A flagged message */
+	Passed  = 1 << 2, /**< A passed (forwarded) message */
+	Replied = 1 << 3, /**< A replied message */
+	Seen    = 1 << 4, /**< A seen (read) message */
+	Trashed = 1 << 5, /**< A trashed message */
+
+	/**
+	 * decides on cur/ or new/ in the maildir
+	 */
+	New = 1 << 6, /**< A new message */
+
+	/**
+	 * content flags -- not visible in the filename, but used for
+	 * searching
+	 */
+	Signed        = 1 << 7, /**< Cryptographically signed */
+	Encrypted     = 1 << 8, /**< Encrypted */
+	HasAttachment = 1 << 9, /**< Has an attachment */
+
+	Unread = 1 << 10, /**< Unread; pseudo-flag, only for queries, so we can
+			   * search for flag:unread, which is equivalent to
+			   * 'flag:new OR NOT flag:seen' */
+	/**
+	 * other content flags
+	 */
+	MailingList = 1 << 11, /**< A mailing-list message */
+
+	/*
+	 * <private>
+	 */
+	_final_ = 1 << 12
+};
+MU_ENABLE_BITOPS(Flags);
+
+/**
+ * Message flags category
+ *
+ */
+enum struct MessageFlagCategory {
+	None,     /**< Nothing */
+	Mailfile, /**< Flag for a message file */
+	Maildir,  /**< Flag for message file's location */
+	Content,  /**< Message content flag */
+	Pseudo    /**< Pseudo flag */
+};
+
+/**
+ * Info about invidual message flags
+ *
+ */
+struct MessageFlagInfo {
+	Flags        flag;     /**< The message flag */
+	char                shortcut; /**< Shortcut character */
+	std::string_view    name;     /**< Name of the flag */
+	MessageFlagCategory category; /**< Flag category */
+
+	/**
+	 * Get the lower-case version of shortcut
+	 *
+	 * @return lower-case shortcut
+	 */
+	constexpr char shortcut_lower() const {
+		return shortcut >= 'A' && shortcut <= 'Z' ?
+			shortcut +  ('a' - 'A') : shortcut;
+	}
+};
+
+/**
+ * Array of all flag information.
+ */
+constexpr std::array<MessageFlagInfo, 12> AllMessageFlagInfos = {{
+    MessageFlagInfo{Flags::Draft,         'D', "draft",	 MessageFlagCategory::Mailfile},
+    MessageFlagInfo{Flags::Flagged,	 'F', "flagged", MessageFlagCategory::Mailfile},
+    MessageFlagInfo{Flags::Passed,	 'P', "passed",	 MessageFlagCategory::Mailfile},
+    MessageFlagInfo{Flags::Replied,	 'R', "replied", MessageFlagCategory::Mailfile},
+    MessageFlagInfo{Flags::Seen,		 'S', "seen",	 MessageFlagCategory::Mailfile},
+    MessageFlagInfo{Flags::Trashed,	 'T', "trashed", MessageFlagCategory::Mailfile},
+
+    MessageFlagInfo{Flags::New,		 'N', "new",	 MessageFlagCategory::Maildir},
+
+    MessageFlagInfo{Flags::Signed,	 'z', "signed",	 MessageFlagCategory::Content},
+    MessageFlagInfo{Flags::Encrypted,	 'x', "encrypted",
+								 MessageFlagCategory::Content},
+    MessageFlagInfo{Flags::HasAttachment, 'a', "attach",
+								 MessageFlagCategory::Content},
+
+    MessageFlagInfo{Flags::Unread,	 'u', "unread",	 MessageFlagCategory::Pseudo},
+
+    MessageFlagInfo{Flags::MailingList,	 'l', "list",	 MessageFlagCategory::Content},
+}};
+
+
+/**
+ * Invoke some callable Func for each flag info
+ *
+ * @param func some callable
+ */
+template<typename Func>
+constexpr void flag_infos_for_each(Func&& func)
+{
+	for (auto&& info: AllMessageFlagInfos)
+		func(info);
+}
+
+/**
+ * Get flag info for some flag
+ *
+ * @param flag a singular flag
+ *
+ * @return the MessageFlagInfo, or std::nullopt in case of error.
+ */
+constexpr const std::optional<MessageFlagInfo>
+flag_info(Flags flag)
+{
+	constexpr auto upper = static_cast<unsigned>(Flags::_final_);
+	const auto     val   = static_cast<unsigned>(flag);
+
+	if (__builtin_popcount(val) != 1 || val >= upper)
+		return std::nullopt;
+
+	return AllMessageFlagInfos[static_cast<unsigned>(__builtin_ctz(val))];
+}
+
+/**
+ * Get flag info for some flag
+ *
+ * @param shortcut shortcut character
+ *
+ * @return the MessageFlagInfo
+ */
+constexpr const std::optional<MessageFlagInfo>
+flag_info(char shortcut)
+{
+	for (auto&& info : AllMessageFlagInfos)
+		if (info.shortcut == shortcut)
+			return info;
+
+	return std::nullopt;
+}
+
+/**
+ * Get flag info for some flag
+ *
+ * @param name of the message-flag.
+ *
+ * @return the MessageFlagInfo
+ */
+constexpr const std::optional<MessageFlagInfo>
+flag_info(std::string_view name)
+{
+	for (auto&& info : AllMessageFlagInfos)
+		if (info.name == name)
+			return info;
+
+	return std::nullopt;
+}
+
+/**
+ * There are two string-based expression types for flags:
+ * 1) 'absolute': replace the existing flags
+ * 2_ 'delta'  : flags as a delta of existing flags.
+ */
+
+/**
+ * Get the (OR'ed) flags corresponding to an expression.
+ *
+ * @param expr the expression (a sequence of flag shortcut characters)
+ * @param ignore_invalid if @true, ignore invalid flags, otherwise return
+ * nullopt if an invalid flag is encountered
+ *
+ * @return the (OR'ed) flags or Flags::None
+ */
+constexpr std::optional<Flags>
+flags_from_absolute_expr(std::string_view expr, bool ignore_invalid = false)
+{
+	Flags flags{Flags::None};
+
+	for (auto&& kar : expr) {
+		if (const auto& info{flag_info(kar)}; !info) {
+			if (!ignore_invalid)
+				return std::nullopt;
+		} else
+			flags |= info->flag;
+	}
+
+	return flags;
+}
+
+/**
+ * Calculate flags from existing flags and a delta expression
+ *
+ * Update @p flags with the flags in @p expr, where @p exprt consists of the the
+ * normal flag shortcut characters, prefixed with either '+' or '-', which means
+ * resp. "add this flag" or "remove this flag".
+ *
+ * So, e.g. "-N+S" would unset the NEW flag and set the SEEN flag, without
+ * affecting other flags.
+ *
+ * @param expr delta expression
+ * @param flags existing flags
+ * @param ignore_invalid if @true, ignore invalid flags, otherwise return
+ * nullopt if an invalid flag is encountered
+ *
+ * @return new flags, or nullopt in case of error
+ */
+constexpr std::optional<Flags>
+flags_from_delta_expr(std::string_view expr, Flags flags,
+			      bool ignore_invalid = false)
+{
+	if (expr.size() % 2 != 0)
+		return std::nullopt;
+
+	for (auto u = 0U; u != expr.size(); u += 2) {
+		if (const auto& info{flag_info(expr[u + 1])}; !info) {
+			if (!ignore_invalid)
+				return std::nullopt;
+		} else {
+			switch (expr[u]) {
+			case '+': flags |= info->flag; break;
+			case '-': flags &= ~info->flag; break;
+			default:
+				if (!ignore_invalid)
+					return std::nullopt;
+				break;
+			}
+		}
+	}
+
+	return flags;
+}
+
+/**
+ * Calculate the flags from either 'absolute' or 'delta' expressions
+ *
+ * @param expr a flag expression, either 'delta' or 'absolute'
+ * @param flags optional: existing flags or none. Required for delta.
+ *
+ * @return either messages flags or std::nullopt in case of error.
+ */
+constexpr std::optional<Flags>
+flags_from_expr(std::string_view            expr,
+			std::optional<Flags> flags = std::nullopt)
+{
+	if (expr.empty())
+		return std::nullopt;
+
+	if (expr[0] == '+' || expr[0] == '-')
+		return flags_from_delta_expr(
+		    expr, flags.value_or(Flags::None), true);
+	else
+		return flags_from_absolute_expr(expr, true);
+}
+
+/**
+ * Filter out flags which are not in the given category
+ *
+ * @param flags flags
+ * @param cat category
+ *
+ * @return filter flags
+ */
+constexpr Flags
+flags_filter(Flags flags, MessageFlagCategory cat)
+{
+	for (auto&& info : AllMessageFlagInfos)
+		if (info.category != cat)
+			flags &= ~info.flag;
+	return flags;
+}
+
+/**
+ * Get a string representation of flags
+ *
+ * @param flags flags
+ *
+ * @return string as a sequence of message-flag shortcuts
+ */
+std::string flags_to_string(Flags flags);
+
+} // namespace Mu
+
+#endif /* MU_FLAGS_HH__ */

lib/message/mu-message.cc

@@ -0,0 +1,425 @@
+/*
+** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify it
+** under the terms of the GNU General Public License as published by the
+** Free Software Foundation; either version 3, or (at your option) any
+** later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+**
+*/
+
+
+#include "mu-message.hh"
+#include "mu-maildir.hh"
+
+#include <utils/mu-util.h>
+#include <utils/mu-utils.hh>
+#include <utils/mu-error.hh>
+
+#include <atomic>
+#include <mutex>
+#include <cstdlib>
+
+#include <glib.h>
+#include <gmime/gmime.h>
+
+#include "gmime/gmime-message.h"
+
+using namespace Mu;
+
+/* note, we do the gmime initialization here rather than in mu-runtime, because this way
+ * we don't need mu-runtime for simple cases -- such as our unit tests. Also note that we
+ * need gmime init even for the doc backend, as we use the address parsing functions also
+ * there. */
+static bool
+gmime_maybe_init(void)
+{
+	static std::atomic_bool gmime_initialized = false;
+
+	if (gmime_initialized)
+		return true;
+
+	static std::mutex lock;
+	g_debug("initializing gmime %u.%u.%u",
+		gmime_major_version,
+		gmime_minor_version,
+		gmime_micro_version);
+
+	g_mime_init();
+	std::atexit([] {
+		g_debug("shutting down gmime");
+		g_mime_shutdown();
+		gmime_initialized = false;
+	});
+
+	return true;
+}
+
+static GMimeMessage*
+make_mime_message(const std::string& path, GError** err)
+{
+	GMimeStream *stream{g_mime_stream_file_open(path.c_str(), "r", err)};
+	if (!stream)
+		return {};
+
+	GMimeParser *parser{g_mime_parser_new_with_stream(stream)};
+	g_object_unref(stream);
+	if (!parser) {
+		g_set_error(err,MU_ERROR_DOMAIN, MU_ERROR_GMIME,
+			    "cannot create mime parser for %s", path.c_str());
+		return {};
+	}
+
+	GMimeMessage *mime_msg{g_mime_parser_construct_message(parser, NULL)};
+	g_object_unref(parser);
+	if (!mime_msg) {
+		g_set_error(err, MU_ERROR_DOMAIN, MU_ERROR_GMIME,
+			    "message seems invalid, ignoring (%s)", path.c_str());
+		return {};
+	}
+
+	return mime_msg;
+}
+
+static void fill_document(Document& doc, GMimeMessage *mime_msg);
+
+
+Message::Message(const std::string& path, const std::string& mdir)
+{
+	gmime_maybe_init();
+
+	/*
+	 * sanity checks.
+	 */
+
+	if (!g_path_is_absolute(path.c_str()))
+		throw Error(Error::Code::File, "path '%s' is not absolute", path.c_str());
+
+	if (::access(path.c_str(), R_OK) != 0)
+		throw Error(Error::Code::File, "file @ '%s' is not readable", path.c_str());
+
+	struct stat statbuf{};
+	if (::stat(path.c_str(), &statbuf) < 0)
+		throw Error(Error::Code::File, "cannot stat %s: %s", path.c_str(),
+			    g_strerror(errno));
+
+	if (!S_ISREG(statbuf.st_mode))
+		throw Error(Error::Code::File, "not a regular file: %s", path);
+
+	/*
+	 * let's get the mime message
+	 */
+	GError *err{};
+	mime_msg_ = make_mime_message(path, &err);
+	if (!mime_msg_)
+		throw Error(Error::Code::File, &err, "invalid message");
+
+	doc_.add(Field::Id::Path,
+		 Mu::from_gchars(g_canonicalize_filename(path.c_str(), NULL)));
+	doc_.add(Field::Id::Maildir, mdir);
+	doc_.add(Field::Id::Size, static_cast<int64_t>(statbuf.st_size));
+
+	// rest of the fields
+	//fill_fields(doc_, mime_msg_);
+}
+
+Message::~Message()
+{
+	g_clear_object(&mime_msg_);
+}
+
+
+Message&
+Message::operator=(const Message& rhs) {
+
+	if (this != &rhs) {
+		doc_ = rhs.doc_;
+		g_clear_object(&mime_msg_);
+		if (rhs.mime_msg_)
+			mime_msg_ = g_object_ref(rhs.mime_msg_);
+	}
+
+	return *this;
+}
+
+
+Message&
+Message::operator=(Message&& rhs)
+{
+	if (this != &rhs) {
+		doc_ = std::move(rhs.doc_);
+		rhs.doc_ = {};
+
+		g_clear_object(&mime_msg_);
+		mime_msg_ = rhs.mime_msg_;
+		rhs.mime_msg_ = {};
+	}
+
+	return *this;
+}
+
+
+static Priority
+parse_prio_str(const char* priostr)
+{
+	int i;
+	struct {
+		const char*     _str;
+		Priority _prio;
+	} str_prio[] = {{"high", Priority::High},
+			{"1", Priority::High},
+			{"2", Priority::High},
+
+			{"normal", Priority::Normal},
+			{"3", Priority::Normal},
+
+			{"low", Priority::Low},
+			{"list", Priority::Low},
+			{"bulk", Priority::Low},
+			{"4", Priority::Low},
+			{"5", Priority::Low}};
+
+	for (i = 0; i != G_N_ELEMENTS(str_prio); ++i)
+		if (g_ascii_strcasecmp(priostr, str_prio[i]._str) == 0)
+			return str_prio[i]._prio;
+
+	/* e.g., last-fm uses 'fm-user'... as precedence */
+	return Priority::Normal;
+}
+
+static Priority
+get_priority(GMimeMessage *mime_msg)
+{
+	auto obj{GMIME_OBJECT(mime_msg)};
+	auto priostr = g_mime_object_get_header(obj, "Precedence");
+	if (!priostr)
+		priostr = g_mime_object_get_header(obj, "X-Priority");
+	if (!priostr)
+		priostr = g_mime_object_get_header(obj, "Importance");
+	if (!priostr)
+		return Priority::Normal;
+	else
+		return parse_prio_str(priostr);
+}
+
+
+static gboolean
+looks_like_attachment(GMimeObject* part)
+{
+	GMimeContentDisposition* disp;
+	GMimeContentType*        ctype;
+	const char*              dispstr;
+	guint                    u;
+	const struct {
+		const char* type;
+		const char* sub_type;
+	} att_types[] = {{"image", "*"},
+			 {"audio", "*"},
+			 {"application", "*"},
+			 {"application", "x-patch"}};
+
+	disp = g_mime_object_get_content_disposition(part);
+
+	if (!GMIME_IS_CONTENT_DISPOSITION(disp))
+		return FALSE;
+
+	dispstr = g_mime_content_disposition_get_disposition(disp);
+
+	if (g_ascii_strcasecmp(dispstr, "attachment") == 0)
+		return TRUE;
+
+	/* we also consider patches, images, audio, and non-pgp-signature
+	 * application attachments to be attachments... */
+	ctype = g_mime_object_get_content_type(part);
+
+	if (g_mime_content_type_is_type(ctype, "*", "pgp-signature"))
+		return FALSE; /* don't consider as a signature */
+
+	if (g_mime_content_type_is_type(ctype, "text", "*")) {
+		if (g_mime_content_type_is_type(ctype, "*", "plain") ||
+		    g_mime_content_type_is_type(ctype, "*", "html"))
+			return FALSE;
+		else
+			return TRUE;
+	}
+
+	for (u = 0; u != G_N_ELEMENTS(att_types); ++u)
+		if (g_mime_content_type_is_type(ctype, att_types[u].type, att_types[u].sub_type))
+			return TRUE;
+
+	return FALSE;
+}
+
+static void
+msg_cflags_cb(GMimeObject* parent, GMimeObject* part, Flags* flags)
+{
+	if (GMIME_IS_MULTIPART_SIGNED(part))
+		*flags |= Flags::Signed;
+
+	/* FIXME: An encrypted part might be signed at the same time.
+	 *        In that case the signed flag is lost. */
+	if (GMIME_IS_MULTIPART_ENCRYPTED(part))
+		*flags |= Flags::Encrypted;
+
+	/* smime */
+	if (GMIME_IS_APPLICATION_PKCS7_MIME(part)) {
+		GMimeApplicationPkcs7Mime *pkcs7;
+		pkcs7 = GMIME_APPLICATION_PKCS7_MIME(part);
+		if (pkcs7) {
+			switch(pkcs7->smime_type) {
+			case GMIME_SECURE_MIME_TYPE_ENVELOPED_DATA:
+				*flags |= Flags::Encrypted;
+				break;
+			case GMIME_SECURE_MIME_TYPE_SIGNED_DATA:
+				*flags |= Flags::Signed;
+				break;
+			default:
+				break;
+			}
+		}
+	}
+
+	if (any_of(*flags & Flags::HasAttachment))
+		return;
+
+	if (!GMIME_IS_PART(part))
+		return;
+
+	if (looks_like_attachment(part))
+		*flags |= Flags::HasAttachment;
+}
+
+static Flags
+get_content_flags(GMimeMessage *mime_msg)
+{
+	Flags flags{Flags::None};
+
+	/* toplevel */
+	msg_cflags_cb(NULL, GMIME_OBJECT(mime_msg), &flags);
+	/* parts */
+	// mu_mime_message_foreach(mime_msg,
+	//			FALSE, /* never decrypt for this */
+	//			(GMimeObjectForeachFunc)msg_cflags_cb,
+	//			&flags);
+
+
+	// char *ml{get_mailing_list(self)};
+	// if (ml) {
+	//	flags |= Flags::MailingList;
+	//	g_free(ml);
+	// }
+
+	return flags;
+}
+
+static Flags
+get_flags(GMimeMessage *mime_msg, const std::string& path)
+{
+	auto flags{mu_maildir_flags_from_path(path)
+		.value_or(Flags::None)};
+	flags |= get_content_flags(mime_msg);
+
+	/* pseudo-flag --> unread means either NEW or NOT SEEN, just
+	 * for searching convenience */
+	if (any_of(flags & Flags::New) ||
+	    none_of(flags & Flags::Seen))
+		flags |= Flags::Unread;
+
+	return flags;
+}
+
+static void
+fill_document(Document& doc, GMimeMessage *mime_msg)
+{
+
+	//const auto contacts{mu_msg_get_contacts(msg)};
+	const auto path{doc.string_value(Field::Id::Path)};
+
+	// auto add_str=[&](Document& doc, Field::Id field_id, const char *str) {
+	//	if (str)
+	//		doc.add(field_id, std::string(str));
+	// };
+
+	field_for_each([&](auto&& field) {
+
+		if (!field.is_indexable_term() && !field.is_normal_term() && !field.is_value())
+			return;
+		// else if (field.is_contact())
+		//	doc.add(field.id, contacts);
+		else if (field.id == Field::Id::Priority)
+			doc.add(get_priority(mime_msg));
+		// else if (field.id == Field::Id::Flags)
+		//	doc.add(get_flags(mime_
+		else if (field.id == Field::Id::ThreadId) {
+			// refs contains a list of parent messages, with the
+			// oldest one first until the last one, which is the
+			// direct parent of the current message. of course, it
+			// may be empty.
+			//
+			// NOTE: there may be cases where the list is truncated;
+			// we happily ignore that case.
+			// const auto refs{mu_msg_get_references(msg)};
+			// const auto thread_id{refs ? (const char*)refs->data : mu_msg_get_msgid(msg)};
+			// doc.add(Field::Id::ThreadId, std::string(thread_id));
+		}
+		// else if (field.id == Field::Id::BodyText)
+		//	add_str(doc, field.id, mu_msg_get_body_text(msg, MU_MSG_OPTION_NONE));
+		// else if (field.id == Field::Id::BodyHtml)
+		//	add_str(doc, field.id, mu_msg_get_body_html(msg, MU_MSG_OPTION_NONE));
+		// else if (field.id == Field::Id::EmbeddedText || field.id == Field::Id::File) {
+		//	/* handle with MIME */
+		// } else if (field.id == Field::Id::Mime)
+		//	mu_msg_part_foreach(msg, MU_MSG_OPTION_RECURSE_RFC822,
+		//			    (MuMsgPartForeachFunc)each_part, &doc);
+		// else if (field.is_numerical())
+		//	doc.add(field.id, mu_msg_get_field_numeric(msg, field.id));
+		// else if (field.is_string())
+		//	add_str(doc, field.id, mu_msg_get_field_string(msg, field.id));
+		// else if (field.is_string_list()) {
+		//	std::vector<std::string> vec;
+		//	auto vals{mu_msg_get_field_string_list(msg, field.id)};
+		//	while (vals) {
+		//		vec.emplace_back ((const char*)vals->data);
+		//		vals = g_slist_next((GList*)vals);
+		//	}
+		//	doc.add(field.id, vec);
+		else {
+			g_warning("unhandled field %*s", STR_V(field.name));
+		}
+	});
+
+	//contacts_cache_.add(std::move(contacts));
+}
+
+
+std::string
+Message::header(const std::string& header_field) const
+{
+	if (!mime_msg_)
+		return {};
+
+	const char *hdr = g_mime_object_get_header(GMIME_OBJECT(mime_msg_),
+						   header_field.c_str());
+	if (!hdr)
+		return {};
+
+	if (!g_utf8_validate(hdr, -1, {})) {
+		char *hdr_u{g_strdup(hdr)};
+		for (auto c = hdr_u; c && *c; ++c) {
+			if ((!isprint(*c) && !isspace (*c)) || !isascii(*c))
+				*c = '.';
+		}
+		return from_gchars(std::move(hdr_u));
+	}
+
+	return hdr;
+}

lib/message/mu-message.hh

@@ -0,0 +1,244 @@
+/*
+** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify it
+** under the terms of the GNU General Public License as published by the
+** Free Software Foundation; either version 3, or (at your option) any
+** later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+**
+*/
+
+#ifndef MU_MESSAGE_HH__
+#define MU_MESSAGE_HH__
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mu-contact.hh"
+#include "mu-priority.hh"
+#include "mu-flags.hh"
+#include "mu-fields.hh"
+#include "mu-document.hh"
+
+struct _GMimeMessage;
+
+namespace Mu {
+
+class Message {
+public:
+	/**
+	 * Construct a message based on a path
+	 *
+	 * @param path path to message
+	 * @param mdir the maildir for this message; ie, if the path is
+	 * ~/Maildir/foo/bar/cur/msg, the maildir would be foo/bar; you can
+	 * pass NULL for this parameter, in which case some maildir-specific
+	 * information is not available.
+	 */
+	Message(const std::string& path, const std::string& mdir);
+
+	/**
+	 * Construct a message based on a Message::Document
+	 *
+	 * @param doc
+	 */
+	Message(Document& doc): doc_{doc} {}
+
+	/**
+	 * Copy CTOR
+	 *
+	 * @param rhs a Message
+	 */
+	Message(const Message& rhs) {
+		*this = rhs;
+	}
+
+	/**
+	 * Move CTOR
+	 *
+	 * @param rhs a Message
+	 */
+
+	Message(Message&& rhs) {
+		*this = std::move(rhs);
+	}
+
+	/**
+	 * DTOR
+	 *
+	 */
+	~Message();
+/**
+	 * Copy assignment operator
+	 *
+	 * @param rhs some message
+	 *
+	 * @return a message ref
+	 */
+	Message& operator=(const Message& rhs);
+
+	/**
+	 * Move assignment operator
+	 *
+	 * @param rhs some message
+	 *
+	 * @return a message ref
+	 */
+	Message& operator=(Message&& rhs);
+
+	/**
+	 * Get the document.
+	 *
+	 *
+	 * @return document
+	 */
+	const Document& document() const { return doc_; }
+
+	/**
+	 * Get the file system path of this message
+	 *
+	 * @return the path of this Message or NULL in case of error.
+	 * the returned string should *not* be modified or freed.
+	 */
+	std::string path() const { return doc_.string_value(Field::Id::Path); }
+
+	/**
+	 * Get the sender (From:) of this message
+	 *
+	 * @return the sender(s) of this Message
+	 */
+	Contacts from() const { return doc_.contacts_value(Field::Id::From); }
+
+	/**
+	 * Get the recipient(s) (To:) for this message
+	 *
+	 * @return recipients
+	 */
+	Contacts to() const { return doc_.contacts_value(Field::Id::To); }
+
+	/**
+	 * Get the recipient(s) (Cc:) for this message
+	 *
+	 * @return recipients
+	 */
+	Contacts cc() const { return doc_.contacts_value(Field::Id::Cc); }
+
+
+	/**
+	 * Get the recipient(s) (Bcc:) for this message
+	 *
+	 * @return recipients
+	 */
+	Contacts bcc() const { return doc_.contacts_value(Field::Id::Bcc); }
+
+	/**
+	 * Get the maildir this message lives in; ie, if the path is
+	 * ~/Maildir/foo/bar/cur/msg, the maildir would be foo/bar
+	 *
+	 * @return the maildir requested or empty */
+	std::string maildir() const  { return doc_.string_value(Field::Id::Maildir); }
+
+	/**
+	 * Get the subject of this message
+	 *
+	 * @return the subject of this Message
+	 */
+	std::string subject() const  { return doc_.string_value(Field::Id::Subject); }
+
+	/**
+	 * Get the Message-Id of this message
+	 *
+	 * @return the Message-Id of this message (without the enclosing <>), or
+	 * a fake message-id for messages that don't have them
+	 */
+	std::string message_id() const { return doc_.string_value(Field::Id::MessageId);}
+
+	/**
+	 * get the mailing list for a message, i.e. the mailing-list
+	 * identifier in the List-Id header.
+	 *
+	 * @return the mailing list id for this message (without the enclosing <>)
+	 * or NULL in case of error or if there is none.
+	 */
+	std::string mailing_list() const { return doc_.string_value(Field::Id::MailingList);}
+
+	/**
+	 * get the message date/time (the Date: field) as time_t, using UTC
+	 *
+	 * @return message date/time or 0 in case of error or if there
+	 * is no such header.
+	 */
+	time_t date() const { return static_cast<time_t>(doc_.integer_value(Field::Id::Date)); }
+
+	/**
+	 * get the flags for this message
+	 *
+	 * @return the file/content flags
+	 */
+	Flags flags() const { return doc_.flags_value(); }
+
+	/**
+	 * get the message priority for this message. The X-Priority, X-MSMailPriority,
+	 * Importance and Precedence header are checked, in that order. if no known or
+	 * explicit priority is set, Priority::Id::Normal is assumed
+	 *
+	 * @return the message priority
+	 */
+	Priority priority() const { return doc_.priority_value(); }
+
+	/**
+	 * get the file size in bytes of this message
+	 *
+	 * @return the filesize
+	 */
+	size_t size() const { return static_cast<size_t>(doc_.integer_value(Field::Id::Size)); }
+
+	/**
+	 * get the list of references (consisting of both the References and
+	 * In-Reply-To fields), with the oldest first and the direct parent as
+	 * the last one. Note, any reference (message-id) will appear at most
+	 * once, duplicates are filtered out.
+	 *
+	 * @return a vec with the references for this msg.
+	 */
+	std::vector<std::string> references() const {
+		return doc_.string_vec_value(Field::Id::References);
+	}
+
+	/**
+	 * get the list of tags (ie., X-Label)
+	 *
+	 * @param msg a valid MuMsg
+	 *
+	 * @return a list with the tags for this msg. Don't modify/free
+	 */
+	std::vector<std::string> tags() const {
+		return doc_.string_vec_value(Field::Id::References);
+	}
+
+	/**
+	 * Get some message-header
+	 *
+	 * @param header_field name of the header
+	 *
+	 * @return the value
+	 */
+	std::string header(const std::string& header_field) const;
+
+private:
+	Document doc_;
+	mutable struct _GMimeMessage *mime_msg_{};
+
+}; // Message
+} // Mu
+#endif /* MU_MESSAGE_HH__ */

lib/message/mu-priority.cc

@@ -0,0 +1,76 @@
+/*
+** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify it
+** under the terms of the GNU General Public License as published by the
+** Free Software Foundation; either version 3, or (at your option) any
+** later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+**
+*/
+
+#include "mu-priority.hh"
+
+using namespace Mu;
+
+std::string
+Mu::to_string(Priority prio)
+{
+	return std::string{priority_name(prio)};
+}
+
+/*
+ * tests... also build as runtime-tests, so we can get coverage info
+ */
+#ifdef BUILD_TESTS
+#include <glib.h>
+#define static_assert g_assert_true
+#endif /*BUILD_TESTS*/
+
+[[maybe_unused]] static void
+test_priority_to_char()
+{
+	static_assert(to_char(Priority::Low) == 'l');
+	static_assert(to_char(Priority::Normal) == 'n');
+	static_assert(to_char(Priority::High) == 'h');
+}
+
+[[maybe_unused]] static void
+test_priority_from_char()
+{
+	static_assert(priority_from_char('l') == Priority::Low);
+	static_assert(priority_from_char('n') == Priority::Normal);
+	static_assert(priority_from_char('h') == Priority::High);
+	static_assert(priority_from_char('x') == Priority::Normal);
+}
+
+[[maybe_unused]] static void
+test_priority_name()
+{
+	static_assert(priority_name(Priority::Low) == "low");
+	static_assert(priority_name(Priority::Normal) == "normal");
+	static_assert(priority_name(Priority::High) == "high");
+}
+
+
+#ifdef BUILD_TESTS
+int
+main(int argc, char* argv[])
+{
+	g_test_init(&argc, &argv, NULL);
+
+	g_test_add_func("/message/priority/to-char", test_priority_to_char);
+	g_test_add_func("/message/priority/from-char", test_priority_from_char);
+	g_test_add_func("/message/priority/name", test_priority_name);
+
+	return g_test_run();
+}
+#endif /*BUILD_TESTS*/

lib/message/mu-priority.hh

@@ -0,0 +1,129 @@
+/*
+** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
+**
+** This program is free software; you can redistribute it and/or modify it
+** under the terms of the GNU General Public License as published by the
+** Free Software Foundation; either version 3, or (at your option) any
+** later version.
+**
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with this program; if not, write to the Free Software Foundation,
+** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+**
+*/
+
+#ifndef MU_PRIORITY_HH__
+#define MU_PRIORITY_HH__
+
+#include <array>
+#include <string>
+#include <string_view>
+#include "mu-fields.hh"
+
+namespace Mu {
+/**
+ * Message priorities
+ *
+ */
+
+/**
+ * The priority ids
+ *
+ */
+enum struct Priority : char {
+	Low    = 'l', /**< Low priority */
+	Normal = 'n', /**< Normal priority */
+	High   = 'h', /**< High priority */
+};
+
+/**
+ * Sequence of all message priorities.
+ */
+static constexpr std::array<Priority, 3> AllMessagePriorities = {
+    Priority::Low, Priority::Normal, Priority::High};
+
+/**
+ * Get the char for some priority
+ *
+ * @param id an id
+ *
+ * @return the char
+ */
+constexpr char
+to_char(Priority prio)
+{
+	return static_cast<char>(prio);
+}
+
+/**
+ * Get the priority for some character; unknown onws
+ * become Normal.
+ *
+ * @param c some character
+ */
+constexpr Priority
+priority_from_char(char c)
+{
+	switch (c) {
+	case 'l':
+		return Priority::Low;
+	case 'h':
+		return Priority::High;
+	case 'n':
+	default:
+		return Priority::Normal;
+	}
+}
+
+/**
+ * Get the name for a given priority
+ *
+ * @return the name
+ */
+constexpr std::string_view
+priority_name(Priority prio)
+{
+	switch (prio) {
+	case Priority::Low:
+		return "low";
+	case Priority::High:
+		return "high";
+	case Priority::Normal:
+	default:
+		return "normal";
+	}
+}
+
+/**
+ * Get the name for a given priority (backward compatibility)
+ *
+ * @return the name
+ */
+constexpr const char*
+priority_name_c_str(Priority prio)
+{
+	switch (prio) {
+	case Priority::Low: return "low";
+	case Priority::High: return "high";
+	case Priority::Normal:
+	default: return "normal";
+	}
+}
+
+/**
+ * Get a the message priority as a string
+ *
+ * @param prio priority
+ *
+ * @return a string
+ */
+std::string to_string(Priority prio);
+
+} // namespace Mu
+
+#endif /*MU_PRIORITY_HH_*/