charset.c: add CHARSET_NO_CANONIFY flag to charset_extract

brong · Oct 25, 2016 · 0c30c3d · 0c30c3d
1 parent a4991f7
commit 0c30c3d
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 1 deletion.
diff --git a/cunit/charset.testc b/cunit/charset.testc
@@ -910,6 +910,12 @@ static void test_extract(void)
 
     /* &nonesuch; is most definitely not defined */
     TESTCASE("A&nonesuch;B", "us-ascii", ENCODING_NONE, "HTML", "A"UTF8_REPLACEMENT"B");
+
+    /* HTML: skip html but don't canonify */
+    flags |= CHARSET_NO_CANONIFY;
+    TESTCASE("<b>Photo</b> <em>booth</em>",
+        "us-ascii", ENCODING_NONE, "HTML",
+        "Photo booth");
 }
 #undef TESTCASE
 

diff --git a/lib/charset.c b/lib/charset.c
@@ -2429,7 +2429,9 @@ EXPORTED int charset_extract(void (*cb)(const struct buf *, void *),
     utf8 = charset_lookupname("utf-8");
     tobuffer = buffer_init();
     input = convert_init(utf8, 0/*to_uni*/, 0, tobuffer);
-    input = canon_init(flags, input);
+    if (!(flags & CHARSET_NO_CANONIFY)) {
+        input = canon_init(flags, input);
+    }
 
     if (!strcmpsafe(subtype, "HTML")) {
         if ((flags & CHARSET_SKIPHTML)) {

diff --git a/lib/charset.h b/lib/charset.h
@@ -54,6 +54,7 @@
 #define CHARSET_SNIPPET (1<<4)
 #define CHARSET_UNFOLD_SKIPWS (1<<5)
 #define CHARSET_MIME_UTF8 (1<<6)
+#define CHARSET_NO_CANONIFY (1<<7)
 
 #define CHARSET_UNKNOWN_CHARSET (NULL)