Merge branch 'wip/djcb/xapian-single-thread'

This makes mu (/mu4e) use only single-threaded access to Xapian((*),
to avoid the problems with #2601 that some people are seeing.

In the mu4e UI, you'll see an '-st' suffix to the version, and
occasionally (hopefully not too often!) you get a warning from mu4e when
trying to talk to mu4e while indexing is underway,

  "Cannot handle command while indexing, please retry later."

which means just what is says.

(*) unless you pass `-Dxapian-single-threaded=false` to meson.

IDEAS.org

@@ -51,6 +51,9 @@ future.
   this support, so it becomes more widely useful.
   https://github.com/djcb/mu/issues/1982
 
+- Display the messages from old-to-new (still get the newest though)
+  https://github.com/djcb/mu/issues/2759
+
 * Done
 
 - Support mu4e-mark-handle-when also for when leaving emacs

lib/mu-indexer.cc

@@ -105,6 +105,7 @@ struct Indexer::Private {
 	bool handler(const std::string& fullpath, struct stat* statbuf, Scanner::HandleType htype);
 
 	void maybe_start_worker();
+
 	void item_worker();
 	void scan_worker();
 
@@ -135,6 +136,8 @@ struct Indexer::Private {
 		Type type;
 	};
 
+	void handle_item(WorkItem&& item);
+
 	AsyncQueue<WorkItem> todos_;
 
 	Progress   progress_{};
@@ -193,7 +196,11 @@ Indexer::Private::handler(const std::string& fullpath, struct stat* statbuf,
 		return true;
 	}
 	case Scanner::HandleType::LeaveDir: {
+#ifdef XAPIAN_SINGLE_THREADED
+		handle_item({fullpath, WorkItem::Type::Dir});
+#else
 		todos_.push({fullpath, WorkItem::Type::Dir});
+#endif /*XAPIAN_SINGLE_THREADED*/
 		return true;
 	}
 
@@ -210,9 +217,13 @@ Indexer::Private::handler(const std::string& fullpath, struct stat* statbuf,
 		if (statbuf->st_ctime <= dirstamp_ && store_.contains_message(fullpath))
 			return false;
 
+#ifdef XAPIAN_SINGLE_THREADED
+		handle_item({fullpath, WorkItem::Type::File});
+#else
 		// push the remaining messages to our "todo" queue for
 		// (re)parsing and adding/updating to the database.
 		todos_.push({fullpath, WorkItem::Type::File});
+#endif
 		return true;
 	}
 	default:
@@ -260,6 +271,30 @@ Indexer::Private::add_message(const std::string& path)
 	return true;
 }
 
+
+void
+Indexer::Private::handle_item(WorkItem&& item)
+{
+	try {
+		switch (item.type) {
+		case WorkItem::Type::File: {
+			if (G_LIKELY(add_message(item.full_path)))
+				++progress_.updated;
+		} break;
+		case WorkItem::Type::Dir:
+			store_.set_dirstamp(item.full_path, ::time(NULL));
+			break;
+		default:
+			g_warn_if_reached();
+			break;
+		}
+	} catch (const Mu::Error& er) {
+		mu_warning("error adding message @ {}: {}", item.full_path, er.what());
+	}
+}
+
+
+
 void
 Indexer::Private::item_worker()
 {
@@ -270,22 +305,8 @@ Indexer::Private::item_worker()
 	while (state_ == IndexState::Scanning) {
 		if (!todos_.pop(item, 250ms))
 			continue;
-		try {
+
-			switch (item.type) {
+		handle_item(std::move(item));
-			case WorkItem::Type::File: {
-				if (G_LIKELY(add_message(item.full_path)))
-					++progress_.updated;
-			} break;
-			case WorkItem::Type::Dir:
-				store_.set_dirstamp(item.full_path, ::time(NULL));
-				break;
-			default:
-				g_warn_if_reached();
-				break;
-			}
-		} catch (const Mu::Error& er) {
-			mu_warning("error adding message @ {}: {}", item.full_path, er.what());
-		}
 
 		maybe_start_worker();
 		std::this_thread::yield();

lib/mu-server.cc

@@ -149,6 +149,7 @@ struct Server::Private {
 	Store&            store() { return store_; }
 	const Store&      store() const { return store_; }
 	Indexer&          indexer() { return store().indexer(); }
+	void              do_index(const Indexer::Config& conf);
 	//CommandMap&       command_map() const { return command_map_; }
 
 	//
@@ -761,6 +762,20 @@ get_stats(const Indexer::Progress& stats, const std::string& state)
 	return sexp;
 }
 
+void
+Server::Private::do_index(const Indexer::Config& conf)
+{
+	StopWatch sw{"indexing"};
+	indexer().start(conf);
+	while (indexer().is_running()) {
+		std::this_thread::sleep_for(std::chrono::milliseconds(2000));
+		output_sexp(get_stats(indexer().progress(), "running"),
+			    Server::OutputFlags::Flush);
+	}
+	output_sexp(get_stats(indexer().progress(), "complete"),
+		    Server::OutputFlags::Flush);
+}
+
 void
 Server::Private::index_handler(const Command& cmd)
 {
@@ -770,22 +785,23 @@ Server::Private::index_handler(const Command& cmd)
 	// ignore .noupdate with an empty store.
 	conf.ignore_noupdate = store().empty();
 
+#ifdef XAPIAN_SINGLE_THREADED
+	// nothing to do
+	if (indexer().is_running()) {
+		throw Error{Error::Code::Xapian, "indexer is already running"};
+	}
+	do_index(conf);
+#else
 	indexer().stop();
 	if (index_thread_.joinable())
 		index_thread_.join();
 
 	// start a background track.
 	index_thread_ = std::thread([this, conf = std::move(conf)] {
-		StopWatch sw{"indexing"};
+		do_index(conf);
-		indexer().start(conf);
-		while (indexer().is_running()) {
-			std::this_thread::sleep_for(std::chrono::milliseconds(2000));
-			output_sexp(get_stats(indexer().progress(), "running"),
-				    Server::OutputFlags::Flush);
-		}
-		output_sexp(get_stats(indexer().progress(), "complete"),
-			    Server::OutputFlags::Flush);
 	});
+#endif /*XAPIAN_SINGLE_THREADED */
+
 }
 
 void
@@ -959,6 +975,9 @@ Server::Private::ping_handler(const Command& cmd)
 				       ":personal-addresses", std::move(addrs),
 				       ":database-path", store().path(),
 				       ":root-maildir", store().root_maildir(),
+#ifdef XAPIAN_SINGLE_THREADED
+				       ":xapian-single-threaded", Sexp::t_sym,
+#endif /*XAPIAN_SINGLE_THREADED*/
 				       ":doccount", storecount)));
 }
 

meson.build

@@ -133,7 +133,13 @@ add_project_arguments(['-DHAVE_CONFIG_H'], language: 'cpp')
 config_h_dep=declare_dependency(
   include_directories: include_directories(['.']))
 
-
+#
+# single-threaded Xapian access?
+#
+if get_option('xapian-single-threaded')
+  config_h_data.set('XAPIAN_SINGLE_THREADED', true)
+  message('use Xapian only in a single thread')
+endif
 #
 # d_type, d_ino are not available universally, so let's check
 # (we use them for optimizations in mu-scanner
@@ -322,6 +328,8 @@ if gmime_dep.version() == '3.2.13'
   warning('See: https://github.com/jstedfast/gmime/issues/133')
 endif
 
+
+
 # Local Variables:
 # indent-tabs-mode: nil
 # End:

meson_options.txt

@@ -14,36 +14,56 @@
 ## along with this program; if not, write to the Free Software Foundation,
 ## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 
-option('tests',
+
+option('cld2',
        type : 'feature',
        value: 'auto',
-       description: 'build unit tests')
+       description: 'Add support for language-detection through cld2')
+
+#
+# emacs
+#
+
+option('emacs',
+       type: 'string',
+       value: 'emacs',
+       description: 'name/path of the emacs executable (for byte-compilation)')
+
+option('lispdir',
+       type: 'string',
+       description: 'path under which to install emacs-lisp files')
+
+
+#
+# guile
+#
 
 option('guile',
        type : 'feature',
        value: 'auto',
        description: 'build the guile scripting support (requires guile-3.x)')
 
-option('cld2',
-       type : 'feature',
-       value: 'auto',
-       description: 'Compact Language Detector2')
-
 # by default, this uses guile_dep.get_variable(pkgconfig: 'extensiondir')
 option('guile-extension-dir',
        type: 'string',
        description: 'custom install path for the guile extension module')
 
+
+#
+# misc
+#
+
+option('tests',
+       type : 'feature',
+       value: 'auto',
+       description: 'build unit tests')
+
+option('xapian-single-threaded',
+       type : 'boolean',
+       value: true,
+       description: 'only use Xapian from a single thread')
+
 option('readline',
        type: 'feature',
        value: 'auto',
        description: 'enable readline support for the mu4e repl')
-
-option('emacs',
-       type: 'string',
-       value: 'emacs',
-       description: 'name/path of the emacs executable')
-
-option('lispdir',
-       type: 'string',
-       description: 'path under which to install emacs-lisp files')

mu4e/meson.build

@@ -16,14 +16,20 @@
 
 
 # generate some build data for use in mu4e
+version_extra=''
+if get_option('xapian-single-threaded')
+  version_extra='-st'
+endif
+
 mu4e_meta = configure_file(
   input: 'mu4e-config.el.in',
   output: 'mu4e-config.el',
   install: true,
   install_dir: mu4e_lispdir,
   configuration: {
-      'VERSION'         : meson.project_version(),
+      'VERSION'           : meson.project_version(),
-      'MU_DOC_DIR'      : join_paths(datadir, 'doc', 'mu'),
+      'MU_VERSION_EXTRA'  : version_extra,
+      'MU_DOC_DIR'        : join_paths(datadir, 'doc', 'mu'),
     })
 
 mu4e_pkg_desc = configure_file(

mu4e/mu4e-main.el

@@ -1,6 +1,6 @@
 ;;; mu4e-main.el --- The Main interface for mu4e -*- lexical-binding: t -*-
 
-;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema
+;; Copyright (C) 2011-2024 Dirk-Jan C. Binnema
 
 ;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
 ;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl>
@@ -299,7 +299,9 @@ Otherwise, do nothing."
          "* "
          (propertize "mu4e" 'face 'mu4e-header-key-face)
          (propertize " - mu for emacs version " 'face 'mu4e-title-face)
-         (propertize  mu4e-mu-version 'face 'mu4e-header-key-face)
+         (propertize  (concat mu4e-mu-version
+                              (if (mu4e--server-xapian-single-threaded-p) "-st" ""))
+                      'face 'mu4e-header-key-face)
          "\n\n"
          (propertize "  Basics\n\n" 'face 'mu4e-title-face)
          (mu4e--main-action

mu4e/mu4e-server.el

@@ -29,7 +29,7 @@
 
 ;;; Configuration
 (defcustom mu4e-mu-home nil
-  "Location of an alternate mu home dir.
+  "Location of an alternate mu home directory.
 If not set, use the defaults, based on the XDG Base Directory
 Specification.
 
@@ -188,6 +188,11 @@ for bookmarks and maildirs.")
   "Get the latest server query items."
   mu4e--server-query-items)
 
+;; temporary
+(defun mu4e--server-xapian-single-threaded-p()
+  "Are we using Xapian in single-threaded mode?"
+  (plist-get mu4e--server-props :xapian-single-threaded))
+
 
 ;;; Handling raw server data
 
@@ -210,6 +215,9 @@ for bookmarks and maildirs.")
           mu4e--server-cookie-post)
   "Regular expression matching the length cookie.
 Match 1 will be the length (in hex).")
+
+(defvar mu4e--server-indexing nil "Currently indexing?")
+
 
 (defun mu4e-running-p ()
   "Whether mu4e is running.
@@ -249,15 +257,18 @@ removed."
 (defun mu4e--server-plist-get (plist key)
   "Like `plist-get' but load data from file if it is a string.
 
-I.e. (mu4e--server-plist-get (:foo bar) :foo)
+PLIST is a property-list, and KEY is the the key to search for.
+
+
+E.g., (mu4e--server-plist-get (:foo bar) :foo)
   => bar
 but
      (mu4e--server-plist-get (:foo \"/tmp/data.eld\") :foo)
   => evaluates the contents of /tmp/data.eld
    (and deletes the file afterward).
 
-This for the few sexps we get from the mu server that support this
+This for the few sexps we get from the mu server that support
-(headers, contacts, maildirs)."
+ this -- headers, contacts, maildirs."
   ;; XXX: perhaps re-use the same buffer?
   (let ((val (plist-get plist key)))
     (if (stringp val)
@@ -383,6 +394,11 @@ The server output is as follows:
 
          ;; get some info
          ((plist-get sexp :info)
+          ;; when indexing is finished, remove the block
+          (when (and (eq (plist-get sexp :info) 'index)
+                     (eq (plist-get sexp :status) 'complete))
+            (setq mu4e--server-indexing nil))
+
           (funcall mu4e-info-func sexp))
 
          ;; get some data
@@ -423,6 +439,7 @@ As per issue #2198."
                 ,(when mu4e-mu-home (format "--muhome=%s" mu4e-mu-home)))))
 
 (defun mu4e--version-check ()
+  "Verify that the versions for mu4e and mu are the same."
   ;; sanity-check 1
   (let ((default-directory temporary-file-directory)) ;;ensure it's local.
     (unless (and mu4e-mu-binary (file-executable-p mu4e-mu-binary))
@@ -486,6 +503,7 @@ You cannot run the repl when mu4e is running (or vice-versa)."
          (proc (and (buffer-live-p buf) (get-buffer-process buf))))
     (when proc
       (mu4e-message "shutting down")
+      (setq mu4e--server-indexing nil)
       (set-process-filter mu4e--server-process nil)
       (set-process-sentinel mu4e--server-process nil)
       (let ((delete-exited-processes t))
@@ -517,16 +535,20 @@ You cannot run the repl when mu4e is running (or vice-versa)."
        ((eq code 0)
         (message nil)) ;; don't do anything
        ((eq code 11)
-        (error "schema mismatch; please re-init mu from command-line"))
+        (error "Schema mismatch; please re-init mu from command-line"))
        ((eq code 19)
-        (error "mu database is locked by another process"))
+        (error "Mu database is locked by another process"))
-       (t (error "mu server process ended with exit code %d" code))))
+       (t (error "Mu server process ended with exit code %d" code))))
      (t
-      (error "something bad happened to the mu server process")))))
+      (error "Something bad happened to the mu server process")))))
 
 (defun mu4e--server-call-mu (form)
   "Call the mu server with some command FORM."
-  (unless (mu4e-running-p) (mu4e--server-start))
+  (unless (mu4e-running-p)
+    (mu4e--server-start))
+  ;; in single-threaded mode, mu can't accept our command right now.
+  (when (and (mu4e--server-xapian-single-threaded-p) mu4e--server-indexing)
+    (mu4e-warn "Cannot handle command while indexing, please retry later."))
   (let* ((print-length nil) (print-level nil)
          (cmd (format "%S" form)))
     (mu4e-log 'to-server "%s" cmd)
@@ -591,7 +613,7 @@ or an error."
 (defun mu4e--server-index (&optional cleanup lazy-check)
   "Index messages.
 If CLEANUP is non-nil, remove messages which are in the database
-but no longer in the filesystem. If LAZY-CHECK is non-nil, only
+but no longer in the file system. If LAZY-CHECK is non-nil, only
 consider messages for which the time stamp (ctime) of the
 directory they reside in has not changed since the previous
 indexing run. This is much faster than the non-lazy check, but
@@ -600,10 +622,11 @@ added or removed), since merely editing a message does not update
 the directory time stamp."
   (mu4e--server-call-mu
    `(index :cleanup ,(and cleanup t)
-           :lazy-check ,(and lazy-check t))))
+           :lazy-check ,(and lazy-check t)))
+  (setq mu4e--server-indexing t)) ;; remember we're indexing.
 
 (defun mu4e--server-mkdir (path &optional update)
-  "Create a new maildir-directory at filesystem PATH.
+  "Create a new maildir-directory at file system PATH.
 When UPDATE is non-nil, send a update when completed.
 PATH must be below the root-maildir."
   ;; handle maildir cache
@@ -674,7 +697,7 @@ read/unread status are returned in the pong-response."
   (mu4e--server-call-mu `(queries :queries ,queries)))
 
 (defun mu4e--server-remove (docid-or-path)
-  "Remove message with either DOCID or PATH.
+  "Remove message with either DOCID-OR-PATH.
 The results are reported through either (:update ... )
 or (:error) sexps."
   (if (stringp docid-or-path)