clang-format: update c/cc coding style
Update all cc code using .clang-format; please do so as well for future PRs etc.; emacs has a handy 'clang-format' mode to make this automatic. For comparing old changes with git blame, we can disregard this one using --ignore-rev (see https://www.moxio.com/blog/43/ignoring-bulk-change-commits-with-git-blame )
This commit is contained in:
Dirk-Jan C. Binnema
@ -41,333 +41,321 @@ using namespace std::chrono_literals;
|
||||
using namespace Mu;
|
||||
|
||||
struct IndexState {
|
||||
enum State { Idle, Scanning, Cleaning };
|
||||
static const char* name(State s) {
|
||||
switch(s) {
|
||||
case Idle: return "idle";
|
||||
case Scanning: return "scanning";
|
||||
case Cleaning: return "cleaning";
|
||||
default: return "<error>";
|
||||
}
|
||||
}
|
||||
enum State { Idle, Scanning, Cleaning };
|
||||
static const char* name(State s)
|
||||
{
|
||||
switch (s) {
|
||||
case Idle: return "idle";
|
||||
case Scanning: return "scanning";
|
||||
case Cleaning: return "cleaning";
|
||||
default: return "<error>";
|
||||
}
|
||||
}
|
||||
|
||||
bool operator==(State rhs) const { return state_ == rhs; }
|
||||
bool operator==(State rhs) const { return state_ == rhs; }
|
||||
|
||||
void change_to(State new_state) {
|
||||
g_debug ("changing indexer state %s->%s",
|
||||
name((State)state_), name((State)new_state));
|
||||
state_ = new_state;
|
||||
}
|
||||
void change_to(State new_state)
|
||||
{
|
||||
g_debug("changing indexer state %s->%s",
|
||||
name((State)state_),
|
||||
name((State)new_state));
|
||||
state_ = new_state;
|
||||
}
|
||||
|
||||
private:
|
||||
State state_{Idle};
|
||||
private:
|
||||
State state_{Idle};
|
||||
};
|
||||
|
||||
struct Indexer::Private {
|
||||
Private (Mu::Store& store):
|
||||
store_{store},
|
||||
scanner_{store_.metadata().root_maildir,
|
||||
[this](auto&& path, auto&& statbuf, auto&& info){
|
||||
return handler(path, statbuf, info);
|
||||
}},
|
||||
max_message_size_{store_.metadata().max_message_size} {
|
||||
Private(Mu::Store& store)
|
||||
: store_{store}, scanner_{store_.metadata().root_maildir,
|
||||
[this](auto&& path, auto&& statbuf, auto&& info) {
|
||||
return handler(path, statbuf, info);
|
||||
}},
|
||||
max_message_size_{store_.metadata().max_message_size}
|
||||
{
|
||||
g_message("created indexer for %s -> %s",
|
||||
store.metadata().root_maildir.c_str(),
|
||||
store.metadata().database_path.c_str());
|
||||
}
|
||||
|
||||
g_message ("created indexer for %s -> %s",
|
||||
store.metadata().root_maildir.c_str(),
|
||||
store.metadata().database_path.c_str());
|
||||
}
|
||||
~Private() { stop(); }
|
||||
|
||||
~Private() { stop(); }
|
||||
bool dir_predicate(const std::string& path, const struct dirent* dirent) const;
|
||||
bool handler(const std::string& fullpath, struct stat* statbuf, Scanner::HandleType htype);
|
||||
|
||||
bool dir_predicate (const std::string& path, const struct dirent* dirent) const;
|
||||
bool handler (const std::string& fullpath, struct stat *statbuf,
|
||||
Scanner::HandleType htype);
|
||||
void maybe_start_worker();
|
||||
void worker();
|
||||
|
||||
void maybe_start_worker();
|
||||
void worker();
|
||||
bool cleanup();
|
||||
|
||||
bool cleanup();
|
||||
bool start(const Indexer::Config& conf);
|
||||
bool stop();
|
||||
|
||||
bool start(const Indexer::Config& conf);
|
||||
bool stop();
|
||||
Indexer::Config conf_;
|
||||
Store& store_;
|
||||
Scanner scanner_;
|
||||
const size_t max_message_size_;
|
||||
|
||||
Indexer::Config conf_;
|
||||
Store& store_;
|
||||
Scanner scanner_;
|
||||
const size_t max_message_size_;
|
||||
time_t dirstamp_{};
|
||||
std::size_t max_workers_;
|
||||
std::vector<std::thread> workers_;
|
||||
std::thread scanner_worker_;
|
||||
|
||||
time_t dirstamp_{};
|
||||
std::size_t max_workers_;
|
||||
std::vector<std::thread> workers_;
|
||||
std::thread scanner_worker_;
|
||||
AsyncQueue<std::string> fq_;
|
||||
|
||||
AsyncQueue<std::string> fq_;
|
||||
Progress progress_;
|
||||
IndexState state_;
|
||||
|
||||
Progress progress_;
|
||||
IndexState state_;
|
||||
|
||||
std::mutex lock_, wlock_;
|
||||
std::mutex lock_, wlock_;
|
||||
};
|
||||
|
||||
|
||||
bool
|
||||
Indexer::Private::handler (const std::string& fullpath, struct stat *statbuf,
|
||||
Scanner::HandleType htype)
|
||||
Indexer::Private::handler(const std::string& fullpath,
|
||||
struct stat* statbuf,
|
||||
Scanner::HandleType htype)
|
||||
{
|
||||
switch (htype) {
|
||||
case Scanner::HandleType::EnterDir:
|
||||
case Scanner::HandleType::EnterNewCur: {
|
||||
// in lazy-mode, we ignore this dir if its dirstamp suggest it
|
||||
// is up-to-date (this is _not_ always true; hence we call it
|
||||
// lazy-mode); only for actual message dirs, since the dir
|
||||
// tstamps may not bubble up.
|
||||
dirstamp_ = store_.dirstamp(fullpath);
|
||||
if (conf_.lazy_check &&
|
||||
dirstamp_ >= statbuf->st_mtime &&
|
||||
htype == Scanner::HandleType::EnterNewCur) {
|
||||
g_debug("skip %s (seems up-to-date)", fullpath.c_str());
|
||||
return false;
|
||||
}
|
||||
switch (htype) {
|
||||
case Scanner::HandleType::EnterDir:
|
||||
case Scanner::HandleType::EnterNewCur: {
|
||||
// in lazy-mode, we ignore this dir if its dirstamp suggest it
|
||||
// is up-to-date (this is _not_ always true; hence we call it
|
||||
// lazy-mode); only for actual message dirs, since the dir
|
||||
// tstamps may not bubble up.
|
||||
dirstamp_ = store_.dirstamp(fullpath);
|
||||
if (conf_.lazy_check && dirstamp_ >= statbuf->st_mtime &&
|
||||
htype == Scanner::HandleType::EnterNewCur) {
|
||||
g_debug("skip %s (seems up-to-date)", fullpath.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
// don't index dirs with '.noindex'
|
||||
auto noindex = ::access((fullpath + "/.noindex").c_str(), F_OK) == 0;
|
||||
if (noindex) {
|
||||
g_debug ("skip %s (has .noindex)", fullpath.c_str());
|
||||
return false; // don't descend into this dir.
|
||||
}
|
||||
// don't index dirs with '.noindex'
|
||||
auto noindex = ::access((fullpath + "/.noindex").c_str(), F_OK) == 0;
|
||||
if (noindex) {
|
||||
g_debug("skip %s (has .noindex)", fullpath.c_str());
|
||||
return false; // don't descend into this dir.
|
||||
}
|
||||
|
||||
// don't index dirs with '.noupdate', unless we do a full
|
||||
// (re)index.
|
||||
if (!conf_.ignore_noupdate) {
|
||||
auto noupdate = ::access((fullpath + "/.noupdate").c_str(), F_OK) == 0;
|
||||
if (noupdate) {
|
||||
g_debug ("skip %s (has .noupdate)", fullpath.c_str());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// don't index dirs with '.noupdate', unless we do a full
|
||||
// (re)index.
|
||||
if (!conf_.ignore_noupdate) {
|
||||
auto noupdate = ::access((fullpath + "/.noupdate").c_str(), F_OK) == 0;
|
||||
if (noupdate) {
|
||||
g_debug("skip %s (has .noupdate)", fullpath.c_str());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
g_debug ("process %s", fullpath.c_str());
|
||||
return true;
|
||||
g_debug("process %s", fullpath.c_str());
|
||||
return true;
|
||||
}
|
||||
case Scanner::HandleType::LeaveDir: {
|
||||
store_.set_dirstamp(fullpath, ::time(NULL));
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
case Scanner::HandleType::LeaveDir: {
|
||||
store_.set_dirstamp(fullpath, ::time(NULL));
|
||||
return true;
|
||||
}
|
||||
case Scanner::HandleType::File: {
|
||||
if ((size_t)statbuf->st_size > max_message_size_) {
|
||||
g_debug("skip %s (too big: %" G_GINT64_FORMAT " bytes)",
|
||||
fullpath.c_str(),
|
||||
(gint64)statbuf->st_size);
|
||||
return false;
|
||||
}
|
||||
|
||||
case Scanner::HandleType::File: {
|
||||
// if the message is not in the db yet, or not up-to-date, queue
|
||||
// it for updating/inserting.
|
||||
if (statbuf->st_mtime <= dirstamp_ && store_.contains_message(fullpath)) {
|
||||
// g_debug ("skip %s: already up-to-date");
|
||||
return false;
|
||||
}
|
||||
|
||||
if ((size_t)statbuf->st_size > max_message_size_) {
|
||||
g_debug ("skip %s (too big: %" G_GINT64_FORMAT " bytes)",
|
||||
fullpath.c_str(), (gint64)statbuf->st_size);
|
||||
return false;
|
||||
}
|
||||
|
||||
// if the message is not in the db yet, or not up-to-date, queue
|
||||
// it for updating/inserting.
|
||||
if (statbuf->st_mtime <= dirstamp_ &&
|
||||
store_.contains_message (fullpath)) {
|
||||
//g_debug ("skip %s: already up-to-date");
|
||||
return false;
|
||||
}
|
||||
|
||||
fq_.push(std::string{fullpath});
|
||||
return true;
|
||||
}
|
||||
default:
|
||||
g_return_val_if_reached (false);
|
||||
return false;
|
||||
}
|
||||
fq_.push(std::string{fullpath});
|
||||
return true;
|
||||
}
|
||||
default: g_return_val_if_reached(false); return false;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
Indexer::Private::maybe_start_worker()
|
||||
{
|
||||
std::lock_guard<std::mutex> wlock{wlock_};
|
||||
std::lock_guard<std::mutex> wlock{wlock_};
|
||||
|
||||
if (fq_.size() > workers_.size() && workers_.size() < max_workers_)
|
||||
workers_.emplace_back(std::thread([this]{worker();}));
|
||||
if (fq_.size() > workers_.size() && workers_.size() < max_workers_)
|
||||
workers_.emplace_back(std::thread([this] { worker(); }));
|
||||
}
|
||||
|
||||
void
|
||||
Indexer::Private::worker()
|
||||
{
|
||||
std::string item;
|
||||
std::string item;
|
||||
|
||||
g_debug ("started worker");
|
||||
g_debug("started worker");
|
||||
|
||||
while (state_ == IndexState::Scanning || !fq_.empty()) {
|
||||
while (state_ == IndexState::Scanning || !fq_.empty()) {
|
||||
if (!fq_.pop(item, 250ms))
|
||||
continue;
|
||||
|
||||
if (!fq_.pop (item, 250ms))
|
||||
continue;
|
||||
// g_debug ("popped (n=%zu) path %s", fq_.size(), item.c_str());
|
||||
++progress_.processed;
|
||||
|
||||
//g_debug ("popped (n=%zu) path %s", fq_.size(), item.c_str());
|
||||
++progress_.processed;
|
||||
try {
|
||||
store_.add_message(item);
|
||||
++progress_.updated;
|
||||
|
||||
try {
|
||||
store_.add_message(item);
|
||||
++progress_.updated;
|
||||
} catch (const Mu::Error& er) {
|
||||
g_warning("error adding message @ %s: %s", item.c_str(), er.what());
|
||||
}
|
||||
|
||||
} catch (const Mu::Error& er) {
|
||||
g_warning ("error adding message @ %s: %s",
|
||||
item.c_str(), er.what());
|
||||
}
|
||||
|
||||
maybe_start_worker();
|
||||
}
|
||||
maybe_start_worker();
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
Indexer::Private::cleanup()
|
||||
{
|
||||
g_debug ("starting cleanup");
|
||||
g_debug("starting cleanup");
|
||||
|
||||
size_t n{};
|
||||
std::vector<Store::Id> orphans; // store messages without files.
|
||||
store_.for_each_message_path([&](Store::Id id, const std::string &path) {
|
||||
size_t n{};
|
||||
std::vector<Store::Id> orphans; // store messages without files.
|
||||
store_.for_each_message_path([&](Store::Id id, const std::string& path) {
|
||||
++n;
|
||||
if (::access(path.c_str(), R_OK) != 0) {
|
||||
g_debug("cannot read %s (id=%u); queueing for removal from store",
|
||||
path.c_str(),
|
||||
id);
|
||||
orphans.emplace_back(id);
|
||||
}
|
||||
|
||||
++n;
|
||||
if (::access(path.c_str(), R_OK) != 0) {
|
||||
g_debug ("cannot read %s (id=%u); queueing for removal from store",
|
||||
path.c_str(), id);
|
||||
orphans.emplace_back(id);
|
||||
}
|
||||
return state_ == IndexState::Cleaning;
|
||||
});
|
||||
|
||||
return state_ == IndexState::Cleaning;
|
||||
});
|
||||
g_debug("remove %zu message(s) from store", orphans.size());
|
||||
store_.remove_messages(orphans);
|
||||
progress_.removed += orphans.size();
|
||||
|
||||
g_debug("remove %zu message(s) from store", orphans.size());
|
||||
store_.remove_messages (orphans);
|
||||
progress_.removed += orphans.size();
|
||||
|
||||
return true;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
bool
|
||||
Indexer::Private::start(const Indexer::Config& conf)
|
||||
{
|
||||
stop();
|
||||
stop();
|
||||
|
||||
conf_ = conf;
|
||||
if (conf_.max_threads == 0)
|
||||
max_workers_ = std::thread::hardware_concurrency();
|
||||
else
|
||||
max_workers_ = conf.max_threads;
|
||||
conf_ = conf;
|
||||
if (conf_.max_threads == 0)
|
||||
max_workers_ = std::thread::hardware_concurrency();
|
||||
else
|
||||
max_workers_ = conf.max_threads;
|
||||
|
||||
g_debug ("starting indexer with <= %zu worker thread(s)", max_workers_);
|
||||
g_debug ("indexing: %s; clean-up: %s",
|
||||
conf_.scan ? "yes" : "no",
|
||||
conf_.cleanup ? "yes" : "no");
|
||||
g_debug("starting indexer with <= %zu worker thread(s)", max_workers_);
|
||||
g_debug("indexing: %s; clean-up: %s",
|
||||
conf_.scan ? "yes" : "no",
|
||||
conf_.cleanup ? "yes" : "no");
|
||||
|
||||
workers_.emplace_back(std::thread([this]{worker();}));
|
||||
workers_.emplace_back(std::thread([this] { worker(); }));
|
||||
|
||||
state_.change_to(IndexState::Scanning);
|
||||
scanner_worker_ = std::thread([this]{
|
||||
progress_ = {};
|
||||
state_.change_to(IndexState::Scanning);
|
||||
scanner_worker_ = std::thread([this] {
|
||||
progress_ = {};
|
||||
|
||||
if (conf_.scan) {
|
||||
g_debug("starting scanner");
|
||||
if (!scanner_.start()) { // blocks.
|
||||
g_warning ("failed to start scanner");
|
||||
goto leave;
|
||||
}
|
||||
g_debug ("scanner finished with %zu file(s) in queue",
|
||||
fq_.size());
|
||||
}
|
||||
if (conf_.scan) {
|
||||
g_debug("starting scanner");
|
||||
if (!scanner_.start()) { // blocks.
|
||||
g_warning("failed to start scanner");
|
||||
goto leave;
|
||||
}
|
||||
g_debug("scanner finished with %zu file(s) in queue", fq_.size());
|
||||
}
|
||||
|
||||
// now there may still be messages in the work queue...
|
||||
// finish those; this is a bit ugly; perhaps we should
|
||||
// handle SIGTERM etc.
|
||||
while (!fq_.empty())
|
||||
std::this_thread::sleep_for(100ms);
|
||||
|
||||
// now there may still be messages in the work queue...
|
||||
// finish those; this is a bit ugly; perhaps we should
|
||||
// handle SIGTERM etc.
|
||||
while (!fq_.empty())
|
||||
std::this_thread::sleep_for(100ms);
|
||||
if (conf_.cleanup) {
|
||||
g_debug("starting cleanup");
|
||||
state_.change_to(IndexState::Cleaning);
|
||||
cleanup();
|
||||
g_debug("cleanup finished");
|
||||
}
|
||||
|
||||
if (conf_.cleanup) {
|
||||
g_debug ("starting cleanup");
|
||||
state_.change_to(IndexState::Cleaning);
|
||||
cleanup();
|
||||
g_debug ("cleanup finished");
|
||||
}
|
||||
store_.commit();
|
||||
leave:
|
||||
state_.change_to(IndexState::Idle);
|
||||
});
|
||||
|
||||
store_.commit();
|
||||
leave:
|
||||
state_.change_to(IndexState::Idle);
|
||||
});
|
||||
g_debug("started indexer");
|
||||
|
||||
g_debug ("started indexer");
|
||||
|
||||
return true;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool
|
||||
Indexer::Private::stop()
|
||||
{
|
||||
scanner_.stop();
|
||||
state_.change_to(IndexState::Idle);
|
||||
scanner_.stop();
|
||||
state_.change_to(IndexState::Idle);
|
||||
|
||||
const auto w_n = workers_.size();
|
||||
const auto w_n = workers_.size();
|
||||
|
||||
fq_.clear();
|
||||
if (scanner_worker_.joinable())
|
||||
scanner_worker_.join();
|
||||
fq_.clear();
|
||||
if (scanner_worker_.joinable())
|
||||
scanner_worker_.join();
|
||||
|
||||
for (auto&& w: workers_)
|
||||
if (w.joinable())
|
||||
w.join();
|
||||
workers_.clear();
|
||||
for (auto&& w : workers_)
|
||||
if (w.joinable())
|
||||
w.join();
|
||||
workers_.clear();
|
||||
|
||||
if (w_n > 0)
|
||||
g_debug ("stopped indexer (joined %zu worker(s))", w_n);
|
||||
if (w_n > 0)
|
||||
g_debug("stopped indexer (joined %zu worker(s))", w_n);
|
||||
|
||||
return true;
|
||||
return true;
|
||||
}
|
||||
|
||||
Indexer::Indexer (Store& store):
|
||||
priv_{std::make_unique<Private>(store)}
|
||||
{}
|
||||
Indexer::Indexer(Store& store) : priv_{std::make_unique<Private>(store)} {}
|
||||
|
||||
Indexer::~Indexer() = default;
|
||||
|
||||
bool
|
||||
Indexer::start(const Indexer::Config& conf)
|
||||
{
|
||||
const auto mdir{priv_->store_.metadata().root_maildir};
|
||||
if (G_UNLIKELY(access (mdir.c_str(), R_OK) != 0)) {
|
||||
g_critical("'%s' is not readable: %s", mdir.c_str(), g_strerror (errno));
|
||||
return false;
|
||||
}
|
||||
const auto mdir{priv_->store_.metadata().root_maildir};
|
||||
if (G_UNLIKELY(access(mdir.c_str(), R_OK) != 0)) {
|
||||
g_critical("'%s' is not readable: %s", mdir.c_str(), g_strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
std::lock_guard<std::mutex> l(priv_->lock_);
|
||||
if (is_running())
|
||||
return true;
|
||||
std::lock_guard<std::mutex> l(priv_->lock_);
|
||||
if (is_running())
|
||||
return true;
|
||||
|
||||
return priv_->start(conf);
|
||||
return priv_->start(conf);
|
||||
}
|
||||
|
||||
bool
|
||||
Indexer::stop()
|
||||
{
|
||||
std::lock_guard<std::mutex> l(priv_->lock_);
|
||||
std::lock_guard<std::mutex> l(priv_->lock_);
|
||||
|
||||
if (!is_running())
|
||||
return true;
|
||||
if (!is_running())
|
||||
return true;
|
||||
|
||||
g_debug ("stopping indexer");
|
||||
return priv_->stop();
|
||||
g_debug("stopping indexer");
|
||||
return priv_->stop();
|
||||
}
|
||||
|
||||
bool
|
||||
Indexer::is_running() const
|
||||
{
|
||||
return !(priv_->state_ == IndexState::Idle) || !priv_->fq_.empty();
|
||||
return !(priv_->state_ == IndexState::Idle) || !priv_->fq_.empty();
|
||||
}
|
||||
|
||||
Indexer::Progress
|
||||
Indexer::progress() const
|
||||
{
|
||||
priv_->progress_.running =
|
||||
priv_->state_ == IndexState::Idle ? false : true;
|
||||
priv_->progress_.running = priv_->state_ == IndexState::Idle ? false : true;
|
||||
|
||||
return priv_->progress_;
|
||||
return priv_->progress_;
|
||||
}
|
||||
|
||||
@ -29,86 +29,82 @@ class Store;
|
||||
|
||||
/// An object abstracting the index process.
|
||||
class Indexer {
|
||||
public:
|
||||
/**
|
||||
* Construct an indexer object
|
||||
*
|
||||
* @param store the message store to use
|
||||
*/
|
||||
Indexer (Store& store);
|
||||
public:
|
||||
/**
|
||||
* Construct an indexer object
|
||||
*
|
||||
* @param store the message store to use
|
||||
*/
|
||||
Indexer(Store& store);
|
||||
|
||||
/**
|
||||
* DTOR
|
||||
*/
|
||||
~Indexer();
|
||||
/**
|
||||
* DTOR
|
||||
*/
|
||||
~Indexer();
|
||||
|
||||
/// A configuration object for the indexer
|
||||
struct Config {
|
||||
bool scan{true};
|
||||
/**< scan for new messages */
|
||||
bool cleanup{true};
|
||||
/**< clean messages no longer in the file system */
|
||||
size_t max_threads{};
|
||||
/**< maximum # of threads to use */
|
||||
bool ignore_noupdate{};
|
||||
/**< ignore .noupdate files */
|
||||
bool lazy_check{};
|
||||
/**< whether to skip directories that don't have a changed
|
||||
* mtime */
|
||||
};
|
||||
/// A configuration object for the indexer
|
||||
struct Config {
|
||||
bool scan{true};
|
||||
/**< scan for new messages */
|
||||
bool cleanup{true};
|
||||
/**< clean messages no longer in the file system */
|
||||
size_t max_threads{};
|
||||
/**< maximum # of threads to use */
|
||||
bool ignore_noupdate{};
|
||||
/**< ignore .noupdate files */
|
||||
bool lazy_check{};
|
||||
/**< whether to skip directories that don't have a changed
|
||||
* mtime */
|
||||
};
|
||||
|
||||
/**
|
||||
* Start indexing. If already underway, do nothing.
|
||||
*
|
||||
* @param conf a configuration object
|
||||
*
|
||||
* @return true if starting worked or an indexing process was already
|
||||
* underway; false otherwise.
|
||||
*
|
||||
*/
|
||||
bool start(const Config& conf);
|
||||
|
||||
/**
|
||||
* Start indexing. If already underway, do nothing.
|
||||
*
|
||||
* @param conf a configuration object
|
||||
*
|
||||
* @return true if starting worked or an indexing process was already
|
||||
* underway; false otherwise.
|
||||
*
|
||||
*/
|
||||
bool start(const Config& conf);
|
||||
/**
|
||||
* Stop indexing. If not indexing, do nothing.
|
||||
*
|
||||
*
|
||||
* @return true if we stopped indexing, or indexing was not underway.
|
||||
* False otherwise.
|
||||
*/
|
||||
bool stop();
|
||||
|
||||
/**
|
||||
* Stop indexing. If not indexing, do nothing.
|
||||
*
|
||||
*
|
||||
* @return true if we stopped indexing, or indexing was not underway.
|
||||
* False otherwise.
|
||||
*/
|
||||
bool stop();
|
||||
/**
|
||||
* Is an indexing process running?
|
||||
*
|
||||
* @return true or false.
|
||||
*/
|
||||
bool is_running() const;
|
||||
|
||||
/**
|
||||
* Is an indexing process running?
|
||||
*
|
||||
* @return true or false.
|
||||
*/
|
||||
bool is_running() const;
|
||||
// Object describing current progress
|
||||
struct Progress {
|
||||
bool running{}; /**< Is an index operation in progress? */
|
||||
size_t processed{}; /**< Number of messages processed */
|
||||
size_t updated{}; /**< Number of messages added/updated to store */
|
||||
size_t removed{}; /**< Number of message removed from store */
|
||||
};
|
||||
|
||||
/**
|
||||
* Get an object describing the current progress. The progress object
|
||||
* describes the most recent indexing job, and is reset up a fresh
|
||||
* start().
|
||||
*
|
||||
* @return a progress object.
|
||||
*/
|
||||
Progress progress() const;
|
||||
|
||||
// Object describing current progress
|
||||
struct Progress {
|
||||
bool running{}; /**< Is an index operation in progress? */
|
||||
size_t processed{}; /**< Number of messages processed */
|
||||
size_t updated{}; /**< Number of messages added/updated to store */
|
||||
size_t removed{}; /**< Number of message removed from store */
|
||||
};
|
||||
|
||||
/**
|
||||
* Get an object describing the current progress. The progress object
|
||||
* describes the most recent indexing job, and is reset up a fresh
|
||||
* start().
|
||||
*
|
||||
* @return a progress object.
|
||||
*/
|
||||
Progress progress() const;
|
||||
|
||||
private:
|
||||
struct Private;
|
||||
std::unique_ptr<Private> priv_;
|
||||
private:
|
||||
struct Private;
|
||||
std::unique_ptr<Private> priv_;
|
||||
};
|
||||
|
||||
|
||||
|
||||
} // namespace Mu
|
||||
#endif /* MU_INDEXER_HH__ */
|
||||
|
||||
@ -38,207 +38,202 @@
|
||||
using namespace Mu;
|
||||
|
||||
struct Scanner::Private {
|
||||
Private (const std::string& root_dir,
|
||||
Scanner::Handler handler):
|
||||
root_dir_{root_dir}, handler_{handler} {
|
||||
if (!handler_)
|
||||
throw Mu::Error{Error::Code::Internal, "missing handler"};
|
||||
}
|
||||
~Private() {
|
||||
stop();
|
||||
}
|
||||
Private(const std::string& root_dir, Scanner::Handler handler)
|
||||
: root_dir_{root_dir}, handler_{handler}
|
||||
{
|
||||
if (!handler_)
|
||||
throw Mu::Error{Error::Code::Internal, "missing handler"};
|
||||
}
|
||||
~Private() { stop(); }
|
||||
|
||||
bool start();
|
||||
bool stop();
|
||||
bool process_dentry (const std::string& path, struct dirent *dentry, bool is_maildir);
|
||||
bool process_dir (const std::string& path, bool is_maildir);
|
||||
bool start();
|
||||
bool stop();
|
||||
bool process_dentry(const std::string& path, struct dirent* dentry, bool is_maildir);
|
||||
bool process_dir(const std::string& path, bool is_maildir);
|
||||
|
||||
const std::string root_dir_;
|
||||
const Scanner::Handler handler_;
|
||||
std::atomic<bool> running_{};
|
||||
std::mutex lock_;
|
||||
const std::string root_dir_;
|
||||
const Scanner::Handler handler_;
|
||||
std::atomic<bool> running_{};
|
||||
std::mutex lock_;
|
||||
};
|
||||
|
||||
|
||||
static bool
|
||||
is_special_dir (const struct dirent *dentry)
|
||||
is_special_dir(const struct dirent* dentry)
|
||||
{
|
||||
const auto d_name{dentry->d_name};
|
||||
return d_name[0] == '\0' ||
|
||||
(d_name[1] == '\0' && d_name[0] == '.') ||
|
||||
(d_name[2] == '\0' && d_name[0] == '.' && d_name[1] == '.');
|
||||
const auto d_name{dentry->d_name};
|
||||
return d_name[0] == '\0' || (d_name[1] == '\0' && d_name[0] == '.') ||
|
||||
(d_name[2] == '\0' && d_name[0] == '.' && d_name[1] == '.');
|
||||
}
|
||||
|
||||
static bool
|
||||
is_new_cur (const char *dirname)
|
||||
is_new_cur(const char* dirname)
|
||||
{
|
||||
if (dirname[0] == 'c' && dirname[1] == 'u' && dirname[2] == 'r' && dirname[3] == '\0')
|
||||
return true;
|
||||
if (dirname[0] == 'c' && dirname[1] == 'u' && dirname[2] == 'r' && dirname[3] == '\0')
|
||||
return true;
|
||||
|
||||
if (dirname[0] == 'n' && dirname[1] == 'e' && dirname[2] == 'w' && dirname[3] == '\0')
|
||||
return true;
|
||||
if (dirname[0] == 'n' && dirname[1] == 'e' && dirname[2] == 'w' && dirname[3] == '\0')
|
||||
return true;
|
||||
|
||||
return false;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool
|
||||
Scanner::Private::process_dentry (const std::string& path, struct dirent *dentry,
|
||||
bool is_maildir)
|
||||
Scanner::Private::process_dentry(const std::string& path, struct dirent* dentry, bool is_maildir)
|
||||
{
|
||||
if (is_special_dir (dentry))
|
||||
return true; // ignore.
|
||||
if (is_special_dir(dentry))
|
||||
return true; // ignore.
|
||||
|
||||
const auto fullpath{path + "/" + dentry->d_name};
|
||||
struct stat statbuf;
|
||||
if (::stat(fullpath.c_str(), &statbuf) != 0) {
|
||||
g_warning ("failed to stat %s: %s", fullpath.c_str(), g_strerror(errno));
|
||||
return false;
|
||||
}
|
||||
const auto fullpath{path + "/" + dentry->d_name};
|
||||
struct stat statbuf;
|
||||
if (::stat(fullpath.c_str(), &statbuf) != 0) {
|
||||
g_warning("failed to stat %s: %s", fullpath.c_str(), g_strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
if (S_ISDIR(statbuf.st_mode)) {
|
||||
const auto new_cur = is_new_cur(dentry->d_name);
|
||||
const auto htype = new_cur ?
|
||||
Scanner::HandleType::EnterNewCur :
|
||||
Scanner::HandleType::EnterDir;
|
||||
const auto res = handler_(fullpath, &statbuf, htype);
|
||||
if (!res)
|
||||
return true; // skip
|
||||
if (S_ISDIR(statbuf.st_mode)) {
|
||||
const auto new_cur = is_new_cur(dentry->d_name);
|
||||
const auto htype = new_cur ? Scanner::HandleType::EnterNewCur
|
||||
: Scanner::HandleType::EnterDir;
|
||||
const auto res = handler_(fullpath, &statbuf, htype);
|
||||
if (!res)
|
||||
return true; // skip
|
||||
|
||||
process_dir (fullpath, new_cur);
|
||||
process_dir(fullpath, new_cur);
|
||||
|
||||
return handler_(fullpath, &statbuf, Scanner::HandleType::LeaveDir);
|
||||
return handler_(fullpath, &statbuf, Scanner::HandleType::LeaveDir);
|
||||
|
||||
} else if (S_ISREG(statbuf.st_mode) && is_maildir)
|
||||
return handler_(fullpath, &statbuf, Scanner::HandleType::File);
|
||||
} else if (S_ISREG(statbuf.st_mode) && is_maildir)
|
||||
return handler_(fullpath, &statbuf, Scanner::HandleType::File);
|
||||
|
||||
g_debug ("skip %s (neither maildir-file nor directory)", fullpath.c_str());
|
||||
return true;
|
||||
g_debug("skip %s (neither maildir-file nor directory)", fullpath.c_str());
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
bool
|
||||
Scanner::Private::process_dir (const std::string& path, bool is_maildir)
|
||||
Scanner::Private::process_dir(const std::string& path, bool is_maildir)
|
||||
{
|
||||
const auto dir = opendir (path.c_str());
|
||||
if (G_UNLIKELY(!dir)) {
|
||||
g_warning("failed to scan dir %s: %s", path.c_str(), g_strerror(errno));
|
||||
return false;
|
||||
}
|
||||
const auto dir = opendir(path.c_str());
|
||||
if (G_UNLIKELY(!dir)) {
|
||||
g_warning("failed to scan dir %s: %s", path.c_str(), g_strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
// TODO: sort dentries by inode order, which makes things faster for extfs.
|
||||
// see mu-maildir.c
|
||||
// TODO: sort dentries by inode order, which makes things faster for extfs.
|
||||
// see mu-maildir.c
|
||||
|
||||
while (running_) {
|
||||
errno = 0;
|
||||
const auto dentry{readdir(dir)};
|
||||
while (running_) {
|
||||
errno = 0;
|
||||
const auto dentry{readdir(dir)};
|
||||
|
||||
if (G_LIKELY(dentry)) {
|
||||
process_dentry (path, dentry, is_maildir);
|
||||
continue;
|
||||
}
|
||||
if (G_LIKELY(dentry)) {
|
||||
process_dentry(path, dentry, is_maildir);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (errno != 0) {
|
||||
g_warning("failed to read %s: %s", path.c_str(), g_strerror(errno));
|
||||
continue;
|
||||
}
|
||||
if (errno != 0) {
|
||||
g_warning("failed to read %s: %s", path.c_str(), g_strerror(errno));
|
||||
continue;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
closedir (dir);
|
||||
break;
|
||||
}
|
||||
closedir(dir);
|
||||
|
||||
return true;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool
|
||||
Scanner::Private::start()
|
||||
{
|
||||
const auto& path{root_dir_};
|
||||
if (G_UNLIKELY(path.length() > PATH_MAX)) {
|
||||
g_warning("path too long");
|
||||
return false;
|
||||
}
|
||||
const auto& path{root_dir_};
|
||||
if (G_UNLIKELY(path.length() > PATH_MAX)) {
|
||||
g_warning("path too long");
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto mode{F_OK | R_OK};
|
||||
if (G_UNLIKELY(access (path.c_str(), mode) != 0)) {
|
||||
g_warning("'%s' is not readable: %s", path.c_str(), g_strerror (errno));
|
||||
return false;
|
||||
}
|
||||
const auto mode{F_OK | R_OK};
|
||||
if (G_UNLIKELY(access(path.c_str(), mode) != 0)) {
|
||||
g_warning("'%s' is not readable: %s", path.c_str(), g_strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
struct stat statbuf{};
|
||||
if (G_UNLIKELY(stat (path.c_str(), &statbuf) != 0)) {
|
||||
g_warning("'%s' is not stat'able: %s", path.c_str(), g_strerror (errno));
|
||||
return false;
|
||||
}
|
||||
struct stat statbuf {
|
||||
};
|
||||
if (G_UNLIKELY(stat(path.c_str(), &statbuf) != 0)) {
|
||||
g_warning("'%s' is not stat'able: %s", path.c_str(), g_strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
if (G_UNLIKELY(!S_ISDIR (statbuf.st_mode))) {
|
||||
g_warning("'%s' is not a directory", path.c_str());
|
||||
return false;
|
||||
}
|
||||
if (G_UNLIKELY(!S_ISDIR(statbuf.st_mode))) {
|
||||
g_warning("'%s' is not a directory", path.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
running_ = true;
|
||||
g_debug ("starting scan @ %s", root_dir_.c_str());
|
||||
running_ = true;
|
||||
g_debug("starting scan @ %s", root_dir_.c_str());
|
||||
|
||||
auto basename{g_path_get_basename(root_dir_.c_str())};
|
||||
const auto is_maildir = (g_strcmp0(basename, "cur") == 0 ||
|
||||
g_strcmp0(basename,"new") == 0);
|
||||
g_free(basename);
|
||||
auto basename{g_path_get_basename(root_dir_.c_str())};
|
||||
const auto is_maildir =
|
||||
(g_strcmp0(basename, "cur") == 0 || g_strcmp0(basename, "new") == 0);
|
||||
g_free(basename);
|
||||
|
||||
const auto start{std::chrono::steady_clock::now()};
|
||||
process_dir(root_dir_, is_maildir);
|
||||
const auto elapsed = std::chrono::steady_clock::now() - start;
|
||||
g_debug ("finished scan of %s in %" G_GINT64_FORMAT " ms", root_dir_.c_str(),
|
||||
to_ms(elapsed));
|
||||
running_ = false;
|
||||
const auto start{std::chrono::steady_clock::now()};
|
||||
process_dir(root_dir_, is_maildir);
|
||||
const auto elapsed = std::chrono::steady_clock::now() - start;
|
||||
g_debug("finished scan of %s in %" G_GINT64_FORMAT " ms",
|
||||
root_dir_.c_str(),
|
||||
to_ms(elapsed));
|
||||
running_ = false;
|
||||
|
||||
return true;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool
|
||||
Scanner::Private::stop()
|
||||
{
|
||||
if (!running_)
|
||||
return true; // nothing to do
|
||||
if (!running_)
|
||||
return true; // nothing to do
|
||||
|
||||
g_debug ("stopping scan");
|
||||
running_ = false;
|
||||
g_debug("stopping scan");
|
||||
running_ = false;
|
||||
|
||||
return true;
|
||||
return true;
|
||||
}
|
||||
|
||||
Scanner::Scanner (const std::string& root_dir,
|
||||
Scanner::Handler handler):
|
||||
priv_{std::make_unique<Private>(root_dir, handler)}
|
||||
{}
|
||||
Scanner::Scanner(const std::string& root_dir, Scanner::Handler handler)
|
||||
: priv_{std::make_unique<Private>(root_dir, handler)}
|
||||
{
|
||||
}
|
||||
|
||||
Scanner::~Scanner() = default;
|
||||
|
||||
bool
|
||||
Scanner::start()
|
||||
{
|
||||
{
|
||||
std::lock_guard<std::mutex> l(priv_->lock_);
|
||||
if (priv_->running_)
|
||||
return true; //nothing to do
|
||||
{
|
||||
std::lock_guard<std::mutex> l(priv_->lock_);
|
||||
if (priv_->running_)
|
||||
return true; // nothing to do
|
||||
|
||||
priv_->running_ = true;
|
||||
}
|
||||
priv_->running_ = true;
|
||||
}
|
||||
|
||||
const auto res = priv_->start();
|
||||
priv_->running_ = false;
|
||||
const auto res = priv_->start();
|
||||
priv_->running_ = false;
|
||||
|
||||
return res;
|
||||
return res;
|
||||
}
|
||||
|
||||
bool
|
||||
Scanner::stop()
|
||||
{
|
||||
std::lock_guard<std::mutex> l(priv_->lock_);
|
||||
std::lock_guard<std::mutex> l(priv_->lock_);
|
||||
|
||||
return priv_->stop();
|
||||
return priv_->stop();
|
||||
}
|
||||
|
||||
bool
|
||||
Scanner::is_running() const
|
||||
{
|
||||
return priv_->running_;
|
||||
return priv_->running_;
|
||||
}
|
||||
|
||||
@ -41,59 +41,58 @@ namespace Mu {
|
||||
/// - directories '.' and '..'
|
||||
///
|
||||
class Scanner {
|
||||
public:
|
||||
enum struct HandleType {
|
||||
File,
|
||||
EnterNewCur, /* cur/ or new/ */
|
||||
EnterDir, /* some other directory */
|
||||
LeaveDir
|
||||
};
|
||||
public:
|
||||
enum struct HandleType {
|
||||
File,
|
||||
EnterNewCur, /* cur/ or new/ */
|
||||
EnterDir, /* some other directory */
|
||||
LeaveDir
|
||||
};
|
||||
|
||||
/// Prototype for a handler function
|
||||
using Handler = std::function<bool(const std::string& fullpath,
|
||||
struct stat* statbuf,
|
||||
HandleType htype)>;
|
||||
/**
|
||||
* Construct a scanner object for scanning a directory, recursively.
|
||||
*
|
||||
* If handler is a directory
|
||||
*
|
||||
*
|
||||
* @param root_dir root dir to start scanning
|
||||
* @param handler handler function for some direntry
|
||||
*/
|
||||
Scanner (const std::string& root_dir, Handler handler);
|
||||
/// Prototype for a handler function
|
||||
using Handler = std::function<
|
||||
bool(const std::string& fullpath, struct stat* statbuf, HandleType htype)>;
|
||||
/**
|
||||
* Construct a scanner object for scanning a directory, recursively.
|
||||
*
|
||||
* If handler is a directory
|
||||
*
|
||||
*
|
||||
* @param root_dir root dir to start scanning
|
||||
* @param handler handler function for some direntry
|
||||
*/
|
||||
Scanner(const std::string& root_dir, Handler handler);
|
||||
|
||||
/**
|
||||
* DTOR
|
||||
*/
|
||||
~Scanner();
|
||||
/**
|
||||
* DTOR
|
||||
*/
|
||||
~Scanner();
|
||||
|
||||
/**
|
||||
* Start the scan; this is a blocking call than runs until
|
||||
* finished or (from another thread) stop() is called.
|
||||
*
|
||||
* @return true if starting worked; false otherwise
|
||||
*/
|
||||
bool start();
|
||||
/**
|
||||
* Start the scan; this is a blocking call than runs until
|
||||
* finished or (from another thread) stop() is called.
|
||||
*
|
||||
* @return true if starting worked; false otherwise
|
||||
*/
|
||||
bool start();
|
||||
|
||||
/**
|
||||
* Stop the scan
|
||||
*
|
||||
* @return true if stopping worked; false otherwi%sse
|
||||
*/
|
||||
bool stop();
|
||||
/**
|
||||
* Stop the scan
|
||||
*
|
||||
* @return true if stopping worked; false otherwi%sse
|
||||
*/
|
||||
bool stop();
|
||||
|
||||
/**
|
||||
* Is a scan currently running?
|
||||
*
|
||||
* @return true or false
|
||||
*/
|
||||
bool is_running() const;
|
||||
/**
|
||||
* Is a scan currently running?
|
||||
*
|
||||
* @return true or false
|
||||
*/
|
||||
bool is_running() const;
|
||||
|
||||
private:
|
||||
struct Private;
|
||||
std::unique_ptr<Private> priv_;
|
||||
private:
|
||||
struct Private;
|
||||
std::unique_ptr<Private> priv_;
|
||||
};
|
||||
|
||||
} // namespace Mu
|
||||
|
||||
@ -28,39 +28,36 @@
|
||||
|
||||
using namespace Mu;
|
||||
|
||||
|
||||
static void
|
||||
test_scan_maildir ()
|
||||
test_scan_maildir()
|
||||
{
|
||||
allow_warnings();
|
||||
|
||||
Scanner scanner{"/home/djcb/Maildir",
|
||||
[](const dirent* dentry)->bool {
|
||||
g_print ("%02x %s\n", dentry->d_type, dentry->d_name);
|
||||
return true;
|
||||
},
|
||||
[](const std::string& fullpath, const struct stat* statbuf,
|
||||
auto&& info)->bool {
|
||||
g_print ("%s %zu\n", fullpath.c_str(), statbuf->st_size);
|
||||
return true;
|
||||
}
|
||||
};
|
||||
g_assert_true (scanner.start());
|
||||
Scanner scanner{
|
||||
"/home/djcb/Maildir",
|
||||
[](const dirent* dentry) -> bool {
|
||||
g_print("%02x %s\n", dentry->d_type, dentry->d_name);
|
||||
return true;
|
||||
},
|
||||
[](const std::string& fullpath, const struct stat* statbuf, auto&& info) -> bool {
|
||||
g_print("%s %zu\n", fullpath.c_str(), statbuf->st_size);
|
||||
return true;
|
||||
}};
|
||||
g_assert_true(scanner.start());
|
||||
|
||||
while (scanner.is_running()) {
|
||||
sleep(1);
|
||||
}
|
||||
while (scanner.is_running()) {
|
||||
sleep(1);
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
main (int argc, char *argv[]) try
|
||||
{
|
||||
g_test_init (&argc, &argv, NULL);
|
||||
main(int argc, char* argv[])
|
||||
try {
|
||||
g_test_init(&argc, &argv, NULL);
|
||||
|
||||
g_test_add_func ("/utils/scanner/scan-maildir", test_scan_maildir);
|
||||
|
||||
return g_test_run ();
|
||||
g_test_add_func("/utils/scanner/scan-maildir", test_scan_maildir);
|
||||
|
||||
return g_test_run();
|
||||
|
||||
} catch (const std::runtime_error& re) {
|
||||
std::cerr << re.what() << "\n";
|
||||
|
||||
Reference in New Issue
Block a user