diff --git a/lib/utils/mu-html-to-text.cc b/lib/utils/mu-html-to-text.cc
index 51106905..0512c7bb 100644
--- a/lib/utils/mu-html-to-text.cc
+++ b/lib/utils/mu-html-to-text.cc
@@ -1,5 +1,5 @@
/*
-** Copyright (C) 2023 Dirk-Jan C. Binnema
+** Copyright (C) 2026 Dirk-Jan C. Binnema
**
** This program is free software; you can redistribute it and/or modify it
** under the terms of the GNU General Public License as published by the
@@ -19,12 +19,12 @@
#include "mu-utils.hh"
#include "mu-option.hh"
-#include "mu-regex.hh"
#include
#include
#include
#include
+#include
using namespace Mu;
@@ -35,7 +35,7 @@ starts_with(std::string_view haystack, std::string_view needle)
if (needle.size() > haystack.size())
return false;
- for (auto&& c = 0U; c != needle.size(); ++c)
+ for (size_t c{}; c != needle.size(); ++c)
if (to_ascii_lower(haystack[c]) != to_ascii_lower(needle[c]))
return false;
@@ -64,7 +64,9 @@ public:
*
* @param html some html to parse
*/
- Context(const std::string& html): html_{html}, pos_{} {}
+ Context(const std::string& html): html_{html}, pos_{} {
+ raw_scraped_.reserve(html.size()/2);
+ }
/**
* Are we done with the html blob, i.e, has it been fully scraped?
@@ -188,7 +190,7 @@ private:
*
* @return cleaned up string.
*/
- std::string cleanup(const std::string unclean) const {
+ std::string cleanup(const std::string& unclean) const {
// reduce whitespace and avoid too long lines;
// makes it easier to debug.
bool was_wspace{};
@@ -196,8 +198,7 @@ private:
std::string clean;
clean.reserve(unclean.size()/2);
for(auto&& c: unclean) {
- auto wspace = c == ' ' || c == '\t' || c == '\n';
- if (wspace) {
+ if (is_ascii_space(c)) {
was_wspace = true;
continue;
}
@@ -233,12 +234,16 @@ format_as(const Context& ctx)
}
+// skip until (and over) the closing quote; pos must be just after the
+// opening quote.
static void
skip_quoted(Context& ctx, std::string_view quote)
{
while(!ctx.done()) {
- if (ctx.looking_at(quote)) // closing quote
+ if (ctx.looking_at(quote)) { // closing quote
+ ctx.advance();
return;
+ }
ctx.advance();
}
}
@@ -250,13 +255,16 @@ skip_script_style(Context& ctx, std::string_view tag)
{
// world!)",
"hello, world!"},
+ // tags are case-insensitive
+ { "AB",
+ "A B"},
+ // '//'-comment ending in the same line as the end-tag
+ { "AB",
+ "A B"},
+ // end-tag inside a string
+ { "A\"; var t = 1;B",
+ "A B"},
+ { "A'; }B",
+ "A B"},
+ // '//' in css is not a comment
+ { "AB",
+ "A B"},
};
for (auto&& test: tests)
assert_equal(html_to_text(test.first), test.second);
}
+static void
+test_entities() // entities
+{
+ static std::vector>
+ tests = {
+ // bare '&' is not an entity; keep the text
+ { "Tom & Jerry; forever", "Tom & Jerry; forever"},
+ { "AT&T <3", "AT&T <3"},
+ // accents are dropped
+ { "página", "pagina"},
+ // numeric entities, decimal and hex
+ { "don't don’t", "don't don’t"},
+ { "café", "café"},
+ // invalid numeric entities become a space
+ { "ABC", "A B C"},
+ };
+
+ for (auto&& test: tests)
+ assert_equal(html_to_text(test.first), test.second);
+}
+
+static void
+test_skipped() // skipped elements
+{
+ static std::vector>
+ tests = {
+ { "Title"
+ ""
+ "Hello",
+ "Hello"},
+ // missing : don't swallow the body
+ { ""
+ "Hello",
+ "Hello"},
+ };
+
+ for (auto&& test: tests)
+ assert_equal(html_to_text(test.first), test.second);
+}
+
+
int
main(int argc, char* argv[])
{
mu_test_init(&argc, &argv);
- g_test_add_func("/html-to-text/test-1", test_1);
- g_test_add_func("/html-to-text/test-2", test_2);
- g_test_add_func("/html-to-text/test-3", test_3);
+ g_test_add_func("/html-to-text/test-basics", test_basics);
+ g_test_add_func("/html-to-text/test-quoted", test_quoted);
+ g_test_add_func("/html-to-text/test-script-style", test_script_style);
+ g_test_add_func("/html-to-text/test-entities", test_entities);
+ g_test_add_func("/html-to-text/test-skipped", test_skipped);
return g_test_run();
}