From e27b57dc3a5e7977ce12d6aebc34d178da98988b Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Mon, 14 Dec 2015 18:36:20 +0900
Subject: [PATCH 01/27] Conform ampersand-error reporting to HTML spec

---
 .../validator/htmlparser/impl/Tokenizer.java  | 54 ++++++++++---------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/src/nu/validator/htmlparser/impl/Tokenizer.java b/src/nu/validator/htmlparser/impl/Tokenizer.java
index 13fd56b1..84c59e01 100755
--- a/src/nu/validator/htmlparser/impl/Tokenizer.java
+++ b/src/nu/validator/htmlparser/impl/Tokenizer.java
@@ -221,6 +221,8 @@ public class Tokenizer implements Locator, Locator2 {
 
     public static final int PROCESSING_INSTRUCTION_QUESTION_MARK = 74;
 
+    public static final int AMBIGUOUS_AMPERSAND = 75;
+
     /**
      * Magic value for UTF-16 operations.
      */
@@ -3054,6 +3056,7 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                         case '<':
                         case '&':
                         case '\u0000':
+                        case ';':
                             emitOrAppendCharRefBuf(returnState);
                             if ((returnState & DATA_AND_RCDATA_MASK) == 0) {
                                 cstart = pos;
@@ -3082,17 +3085,12 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 firstCharKey = c - 'A';
                             } else {
                                 // No match
-                                /*
-                                 * If no match can be made, then this is a parse
-                                 * error.
-                                 */
-                                errNoNamedCharacterMatch();
                                 emitOrAppendCharRefBuf(returnState);
                                 if ((returnState & DATA_AND_RCDATA_MASK) == 0) {
                                     cstart = pos;
                                 }
                                 reconsume = true;
-                                state = transition(state, returnState, reconsume, pos);
+                                state = transition(state, Tokenizer.AMBIGUOUS_AMPERSAND, reconsume, pos);
                                 continue stateloop;
                             }
                             // Didn't fail yet
@@ -3153,17 +3151,12 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                             }
                         }
                         if (hilo == 0) {
-                            /*
-                             * If no match can be made, then this is a parse
-                             * error.
-                             */
-                            errNoNamedCharacterMatch();
                             emitOrAppendCharRefBuf(returnState);
                             if ((returnState & DATA_AND_RCDATA_MASK) == 0) {
                                 cstart = pos;
                             }
                             reconsume = true;
-                            state = transition(state, returnState, reconsume, pos);
+                            state = transition(state, Tokenizer.AMBIGUOUS_AMPERSAND, reconsume, pos);
                             continue stateloop;
                         }
                         // Didn't fail yet
@@ -3246,16 +3239,12 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
 
                     if (candidate == -1) {
                         // reconsume deals with CR, LF or nul
-                        /*
-                         * If no match can be made, then this is a parse error.
-                         */
-                        errNoNamedCharacterMatch();
                         emitOrAppendCharRefBuf(returnState);
                         if ((returnState & DATA_AND_RCDATA_MASK) == 0) {
                             cstart = pos;
                         }
                         reconsume = true;
-                        state = transition(state, returnState, reconsume, pos);
+                        state = transition(state, Tokenizer.AMBIGUOUS_AMPERSAND, reconsume, pos);
                         continue stateloop;
                     } else {
                         // c can't be CR, LF or nul if we got here
@@ -3293,10 +3282,9 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                      * after the U+0026 AMPERSAND (&) must be
                                      * unconsumed, and nothing is returned.
                                      */
-                                    errNoNamedCharacterMatch();
                                     appendCharRefBufToStrBuf();
                                     reconsume = true;
-                                    state = transition(state, returnState, reconsume, pos);
+                                    state = transition(state, Tokenizer.AMBIGUOUS_AMPERSAND, reconsume, pos);
                                     continue stateloop;
                                 }
                             }
@@ -3359,6 +3347,28 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                          * I'm ∉ I tell you.
                          */
                     }
+                    // XXX reorder point
+                case AMBIGUOUS_AMPERSAND:
+                    ampersandloop: for (;;) {
+                        if (reconsume) {
+                            if (++pos == endPos) {
+                                break stateloop;
+                            }
+                            pos--;
+                            c = checkChar(buf, pos);
+                        }
+                        if (c == ';') {
+                            errNoNamedCharacterMatch();
+                        } else if ((c >= '0' && c <= '9')
+                                || (c >= 'A' && c <= 'Z')
+                                || (c >= 'a' && c <= 'z')) {
+                            appendStrBuf(c);
+                            pos++;
+                            continue;
+                        }
+                        state = transition(state, returnState, reconsume, pos);
+                        continue stateloop;
+                    }
                 case CONSUME_NCR:
                     if (++pos == endPos) {
                         break stateloop;
@@ -6449,7 +6459,6 @@ public void eof() throws SAXException {
                     state = returnState;
                     continue;
                 case CHARACTER_REFERENCE_HILO_LOOKUP:
-                    errNoNamedCharacterMatch();
                     emitOrAppendCharRefBuf(returnState);
                     state = returnState;
                     continue;
@@ -6503,10 +6512,6 @@ public void eof() throws SAXException {
                     }
 
                     if (candidate == -1) {
-                        /*
-                         * If no match can be made, then this is a parse error.
-                         */
-                        errNoNamedCharacterMatch();
                         emitOrAppendCharRefBuf(returnState);
                         state = returnState;
                         continue eofloop;
@@ -6544,7 +6549,6 @@ public void eof() throws SAXException {
                                      * after the U+0026 AMPERSAND (&) must be
                                      * unconsumed, and nothing is returned.
                                      */
-                                    errNoNamedCharacterMatch();
                                     appendCharRefBufToStrBuf();
                                     state = returnState;
                                     continue eofloop;

From 82ef614c00e0872ebf122af489a451fb9b327c2c Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Fri, 10 Jun 2016 21:39:28 +0900
Subject: [PATCH 02/27] Make consecutive hyphens in comments a non-error

Also allow `<!-->` at (IE conditional) comment end

See https://github.com/whatwg/html/pull/1356
See https://github.com/whatwg/html/pull/1456
---
 .../impl/ErrorReportingTokenizer.java         |  10 +-
 .../validator/htmlparser/impl/Tokenizer.java  | 199 ++++++++++++++++--
 2 files changed, 190 insertions(+), 19 deletions(-)

diff --git a/src/nu/validator/htmlparser/impl/ErrorReportingTokenizer.java b/src/nu/validator/htmlparser/impl/ErrorReportingTokenizer.java
index 19fbe7a6..77dc9090 100644
--- a/src/nu/validator/htmlparser/impl/ErrorReportingTokenizer.java
+++ b/src/nu/validator/htmlparser/impl/ErrorReportingTokenizer.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2009-2013 Mozilla Foundation
+ * Copyright (c) 2009-2017 Mozilla Foundation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -395,8 +395,8 @@ private boolean isAstralPrivateUse(int c) {
         err("Nameless doctype.");
     }
 
-    @Override protected void errConsecutiveHyphens() throws SAXException {
-        err("Consecutive hyphens did not terminate a comment. \u201C--\u201D is not permitted inside a comment, but e.g. \u201C- -\u201D is.");
+    @Override protected void errNestedComment() throws SAXException {
+        err("Saw \u201C<!--\u201D within a comment. Probable cause: Nested comment (not allowed).");
     }
 
     @Override protected void errPrematureEndOfComment() throws SAXException {
@@ -678,10 +678,6 @@ private boolean isAstralPrivateUse(int c) {
         err("Missing space before doctype name.");
     }
 
-    @Override protected void errHyphenHyphenBang() throws SAXException {
-        err("\u201C--!\u201D found in comment.");
-    }
-
     @Override protected void errNcrControlChar() throws SAXException {
         err("Character reference expands to a control character ("
                 + toUPlusString((char) value) + ").");
diff --git a/src/nu/validator/htmlparser/impl/Tokenizer.java b/src/nu/validator/htmlparser/impl/Tokenizer.java
index 84c59e01..43b4de9d 100755
--- a/src/nu/validator/htmlparser/impl/Tokenizer.java
+++ b/src/nu/validator/htmlparser/impl/Tokenizer.java
@@ -223,6 +223,14 @@ public class Tokenizer implements Locator, Locator2 {
 
     public static final int AMBIGUOUS_AMPERSAND = 75;
 
+    public static final int COMMENT_LESSTHAN = 76;
+
+    public static final int COMMENT_LESSTHAN_BANG = 77;
+
+    public static final int COMMENT_LESSTHAN_BANG_DASH = 78;
+
+    public static final int COMMENT_LESSTHAN_BANG_DASH_DASH = 79;
+
     /**
      * Magic value for UTF-16 operations.
      */
@@ -1031,9 +1039,8 @@ private void maybeAppendSpaceToBogusComment() throws SAXException {
 
     // ]NOCPP]
 
-    @Inline private void adjustDoubleHyphenAndAppendToStrBufAndErr(char c)
+    @Inline private void adjustDoubleHyphenAndAppendToStrBufAndErr(char c, boolean reportedConsecutiveHyphens)
             throws SAXException {
-        errConsecutiveHyphens();
         // [NOCPP[
         switch (commentPolicy) {
             case ALTER_INFOSET:
@@ -1044,7 +1051,9 @@ private void maybeAppendSpaceToBogusComment() throws SAXException {
                 appendStrBuf('-');
                 // CPPONLY: MOZ_FALLTHROUGH;
             case ALLOW:
-                warn("The document is not mappable to XML 1.0 due to two consecutive hyphens in a comment.");
+                if (!reportedConsecutiveHyphens) {
+                    warn("The document is not mappable to XML 1.0 due to two consecutive hyphens in a comment.");
+                }
                 // ]NOCPP]
                 appendStrBuf(c);
                 // [NOCPP[
@@ -1466,6 +1475,7 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
     @SuppressWarnings("unused") private int stateLoop(int state, char c,
             int pos, @NoLength char[] buf, boolean reconsume, int returnState,
             int endPos) throws SAXException {
+        boolean reportedConsecutiveHyphens = false;
         /*
          * Idioms used in this code:
          *
@@ -2542,6 +2552,7 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                     }
                     // CPPONLY: MOZ_FALLTHROUGH;
                 case COMMENT_START:
+                    reportedConsecutiveHyphens = false;
                     commentstartloop: for (;;) {
                         if (++pos == endPos) {
                             break stateloop;
@@ -2574,6 +2585,10 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                  */
                                 state = transition(state, Tokenizer.DATA, reconsume, pos);
                                 continue stateloop;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
                             case '\r':
                                 appendStrBufCarriageReturn();
                                 state = transition(state, Tokenizer.COMMENT, reconsume, pos);
@@ -2619,6 +2634,10 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 state = transition(state, Tokenizer.COMMENT_END_DASH, reconsume, pos);
                                 break commentloop;
                             // continue stateloop;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
                             case '\r':
                                 appendStrBufCarriageReturn();
                                 break stateloop;
@@ -2661,6 +2680,10 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 state = transition(state, Tokenizer.COMMENT_END, reconsume, pos);
                                 break commentenddashloop;
                             // continue stateloop;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
                             case '\r':
                                 appendStrBufCarriageReturn();
                                 state = transition(state, Tokenizer.COMMENT, reconsume, pos);
@@ -2715,11 +2738,16 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                  * Append a U+002D HYPHEN-MINUS (-) character to
                                  * the comment token's data.
                                  */
-                                adjustDoubleHyphenAndAppendToStrBufAndErr(c);
+                                adjustDoubleHyphenAndAppendToStrBufAndErr(c, reportedConsecutiveHyphens);
+                                reportedConsecutiveHyphens = true;
                                 /*
                                  * Stay in the comment end state.
                                  */
                                 continue;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
                             case '\r':
                                 adjustDoubleHyphenAndAppendToStrBufCarriageReturn();
                                 state = transition(state, Tokenizer.COMMENT, reconsume, pos);
@@ -2729,7 +2757,6 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 state = transition(state, Tokenizer.COMMENT, reconsume, pos);
                                 continue stateloop;
                             case '!':
-                                errHyphenHyphenBang();
                                 appendStrBuf(c);
                                 state = transition(state, Tokenizer.COMMENT_END_BANG, reconsume, pos);
                                 continue stateloop;
@@ -2742,7 +2769,8 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                  * and the input character to the comment
                                  * token's data.
                                  */
-                                adjustDoubleHyphenAndAppendToStrBufAndErr(c);
+                                adjustDoubleHyphenAndAppendToStrBufAndErr(c, reportedConsecutiveHyphens);
+                                reportedConsecutiveHyphens = true;
                                 /*
                                  * Switch to the comment state.
                                  */
@@ -2812,6 +2840,148 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 continue stateloop;
                         }
                     }
+                case COMMENT_LESSTHAN:
+                    for (;;) {
+                        if (++pos == endPos) {
+                            break stateloop;
+                        }
+                        c = checkChar(buf, pos);
+                        switch (c) {
+                            case '!':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN_BANG, reconsume, pos);
+                                continue stateloop;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
+                            case '-':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_END_DASH, reconsume, pos);
+                                continue stateloop;
+                            case '\r':
+                                appendStrBufCarriageReturn();
+                                break stateloop;
+                            case '\n':
+                                appendStrBufLineFeed();
+                                continue;
+                            case '\u0000':
+                                c = '\uFFFD';
+                                // fall thru
+                            default:
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT, reconsume, pos);
+                                continue stateloop;
+                        }
+                    }
+                case COMMENT_LESSTHAN_BANG:
+                    for (;;) {
+                        if (++pos == endPos) {
+                            break stateloop;
+                        }
+                        c = checkChar(buf, pos);
+                        switch (c) {
+                            case '-':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN_BANG_DASH, reconsume, pos);
+                                continue stateloop;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
+                            case '\r':
+                                appendStrBufCarriageReturn();
+                                break stateloop;
+                            case '\n':
+                                appendStrBufLineFeed();
+                                continue;
+                            case '\u0000':
+                                c = '\uFFFD';
+                                // fall thru
+                            default:
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT, reconsume, pos);
+                                continue stateloop;
+                        }
+                    }
+                case COMMENT_LESSTHAN_BANG_DASH:
+                    for (;;) {
+                        if (++pos == endPos) {
+                            break stateloop;
+                        }
+                        c = checkChar(buf, pos);
+                        switch (c) {
+                            case '-':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN_BANG_DASH_DASH, reconsume, pos);
+                                continue stateloop;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
+                            case '\r':
+                                appendStrBufCarriageReturn();
+                                break stateloop;
+                            case '\n':
+                                appendStrBufLineFeed();
+                                continue;
+                            case '\u0000':
+                                c = '\uFFFD';
+                                // fall thru
+                            default:
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT, reconsume, pos);
+                                continue stateloop;
+                        }
+                    }
+                case COMMENT_LESSTHAN_BANG_DASH_DASH:
+                    for (;;) {
+                        if (++pos == endPos) {
+                            break stateloop;
+                        }
+                        c = checkChar(buf, pos);
+                        switch (c) {
+                            case '>':
+                                appendStrBuf(c);
+                                emitComment(3, pos);
+                                state = transition(state, Tokenizer.DATA, reconsume, pos);
+                                continue stateloop;
+                            case '-':
+                                errNestedComment();
+                                adjustDoubleHyphenAndAppendToStrBufAndErr(c, reportedConsecutiveHyphens);
+                                reportedConsecutiveHyphens = true;
+                                state = transition(state, Tokenizer.COMMENT_END, reconsume, pos);
+                                continue stateloop;
+                            case '\r':
+                                errNestedComment();
+                                adjustDoubleHyphenAndAppendToStrBufAndErr(c, reportedConsecutiveHyphens);
+                                reportedConsecutiveHyphens = true;
+                                state = transition(state, Tokenizer.COMMENT, reconsume, pos);
+                                break stateloop;
+                            case '\n':
+                                errNestedComment();
+                                adjustDoubleHyphenAndAppendToStrBufAndErr(c, reportedConsecutiveHyphens);
+                                reportedConsecutiveHyphens = true;
+                                state = transition(state, Tokenizer.COMMENT, reconsume, pos);
+                                continue;
+                            case '\u0000':
+                                c = '\uFFFD';
+                                // fall thru
+                            case '!':
+                                errNestedComment();
+                                adjustDoubleHyphenAndAppendToStrBufAndErr(c, reportedConsecutiveHyphens);
+                                reportedConsecutiveHyphens = true;
+                                state = transition(state, Tokenizer.COMMENT_END_BANG, reconsume, pos);
+                                continue stateloop;
+                            default:
+                                errNestedComment();
+                                adjustDoubleHyphenAndAppendToStrBufAndErr(c, reportedConsecutiveHyphens);
+                                reportedConsecutiveHyphens = true;
+                                state = transition(state, Tokenizer.COMMENT_END, reconsume, pos);
+                                continue stateloop;
+                        }
+                    }
+                    // XXX reorder point
                 case COMMENT_START_DASH:
                     if (++pos == endPos) {
                         break stateloop;
@@ -2840,6 +3010,10 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                              */
                             state = transition(state, Tokenizer.DATA, reconsume, pos);
                             continue stateloop;
+                        case '<':
+                            appendStrBuf(c);
+                            state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                            continue stateloop;
                         case '\r':
                             appendStrBufCarriageReturn();
                             state = transition(state, Tokenizer.COMMENT, reconsume, pos);
@@ -5967,13 +6141,13 @@ private void initDoctypeFields() {
     @Inline private void adjustDoubleHyphenAndAppendToStrBufCarriageReturn()
             throws SAXException {
         silentCarriageReturn();
-        adjustDoubleHyphenAndAppendToStrBufAndErr('\n');
+        adjustDoubleHyphenAndAppendToStrBufAndErr('\n', false);
     }
 
     @Inline private void adjustDoubleHyphenAndAppendToStrBufLineFeed()
             throws SAXException {
         silentLineFeed();
-        adjustDoubleHyphenAndAppendToStrBufAndErr('\n');
+        adjustDoubleHyphenAndAppendToStrBufAndErr('\n', false);
     }
 
     @Inline private void appendStrBufLineFeed() {
@@ -6278,6 +6452,8 @@ public void eof() throws SAXException {
                     break eofloop;
                 case COMMENT_START:
                 case COMMENT:
+                case COMMENT_LESSTHAN:
+                case COMMENT_LESSTHAN_BANG:
                     /*
                      * EOF Parse error.
                      */
@@ -6289,6 +6465,7 @@ public void eof() throws SAXException {
                      */
                     break eofloop;
                 case COMMENT_END:
+                case COMMENT_LESSTHAN_BANG_DASH_DASH:
                     errEofInComment();
                     /* Emit the comment token. */
                     emitComment(2, 0);
@@ -6298,6 +6475,7 @@ public void eof() throws SAXException {
                     break eofloop;
                 case COMMENT_END_DASH:
                 case COMMENT_START_DASH:
+                case COMMENT_LESSTHAN_BANG_DASH:
                     errEofInComment();
                     /* Emit the comment token. */
                     emitComment(1, 0);
@@ -6921,7 +7099,7 @@ protected void errGtInPublicId() throws SAXException {
     protected void errNamelessDoctype() throws SAXException {
     }
 
-    protected void errConsecutiveHyphens() throws SAXException {
+    protected void errNestedComment() throws SAXException {
     }
 
     protected void errPrematureEndOfComment() throws SAXException {
@@ -7064,9 +7242,6 @@ protected void errExpectedSystemId() throws SAXException {
     protected void errMissingSpaceBeforeDoctypeName() throws SAXException {
     }
 
-    protected void errHyphenHyphenBang() throws SAXException {
-    }
-
     protected void errNcrControlChar() throws SAXException {
     }
 

From 9706ad04d115951e718dc995c39ac4d48b2f67f6 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Wed, 20 Jul 2016 00:15:36 +0900
Subject: [PATCH 03/27] =?UTF-8?q?Correct=20error=20for=20EOF=20in=20?=
 =?UTF-8?q?=E2=80=9Cin=20template=E2=80=9D=20state?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Doing `errUnclosedElements(eltPos, "template")` for EOF in the “in
template” state results in the error message “End tag `template` seen, but
there were open elements”, which is all wrong because the actual problem is
that though a `template` end tag was expected, EOF was reached without a
`template` end tag being seen.

So let’s instead when we reach this just report the list of open elements.
---
 src/nu/validator/htmlparser/impl/TreeBuilder.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nu/validator/htmlparser/impl/TreeBuilder.java b/src/nu/validator/htmlparser/impl/TreeBuilder.java
index 7b78b1b7..f48875cb 100644
--- a/src/nu/validator/htmlparser/impl/TreeBuilder.java
+++ b/src/nu/validator/htmlparser/impl/TreeBuilder.java
@@ -1351,7 +1351,7 @@ public final void eof() throws SAXException {
                         break eofloop;
                     }
                     if (errorHandler != null) {
-                        errUnclosedElements(eltPos, "template");
+                        errListUnclosedStartTags(0);
                     }
                     while (currentPtr >= eltPos) {
                         pop();

From 00ca1aa4d08394b4054520bccd61f6ed213e7a37 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Fri, 19 May 2017 00:32:05 +0900
Subject: [PATCH 04/27] =?UTF-8?q?Support=20=E2=80=9Cgenerate=20all=20impli?=
 =?UTF-8?q?ed=20end=20tags=20thoroughly=E2=80=9D?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When the parser encounters a `</template>` end tag and there are other
open elements, the HTML spec requires the parser to “generate all
implied end tags thoroughly”, which unlike “generate implied end tags”
also includes generating implied end tags for table-parts elements
(caption, colgroup, tbody, thead, tfoot, td, th, and tr).
---
 .../htmlparser/impl/TreeBuilder.java          | 25 ++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/src/nu/validator/htmlparser/impl/TreeBuilder.java b/src/nu/validator/htmlparser/impl/TreeBuilder.java
index f48875cb..6756c07c 100644
--- a/src/nu/validator/htmlparser/impl/TreeBuilder.java
+++ b/src/nu/validator/htmlparser/impl/TreeBuilder.java
@@ -3884,7 +3884,7 @@ private void endTagTemplateInHead() throws SAXException {
             errStrayEndTag("template");
             return;
         }
-        generateImpliedEndTags();
+        generateImpliedEndTagsThoroughly();
         if (errorHandler != null && !isCurrent("template")) {
             errUnclosedElements(eltPos, "template");
         }
@@ -4025,6 +4025,29 @@ private void generateImpliedEndTags() throws SAXException {
         }
     }
 
+    private void generateImpliedEndTagsThoroughly() throws SAXException {
+        for (;;) {
+            switch (stack[currentPtr].getGroup()) {
+                case CAPTION:
+                case COLGROUP:
+                case DD_OR_DT:
+                case LI:
+                case OPTGROUP:
+                case OPTION:
+                case P:
+                case RB_OR_RTC:
+                case RT_OR_RP:
+                case TBODY_OR_THEAD_OR_TFOOT:
+                case TD_OR_TH:
+                case TR:
+                    pop();
+                    continue;
+                default:
+                    return;
+            }
+        }
+    }
+
     private boolean isSecondOnStackBody() {
         return currentPtr >= 1 && stack[1].getGroup() == TreeBuilder.BODY;
     }

From f761305662f806caadd71efee3e5627afcf3ab1e Mon Sep 17 00:00:00 2001
From: Henri Sivonen <hsivonen@hsivonen.fi>
Date: Thu, 2 Jul 2020 15:56:47 +0300
Subject: [PATCH 05/27] Mozilla bug 1650066 fixup - Avoid calling
 errListUnclosedStartTags in C++.

Differential Revision: https://phabricator.services.mozilla.com/D82007
---
 src/nu/validator/htmlparser/impl/TreeBuilder.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/nu/validator/htmlparser/impl/TreeBuilder.java b/src/nu/validator/htmlparser/impl/TreeBuilder.java
index 6756c07c..67f9b087 100644
--- a/src/nu/validator/htmlparser/impl/TreeBuilder.java
+++ b/src/nu/validator/htmlparser/impl/TreeBuilder.java
@@ -1350,9 +1350,11 @@ public final void eof() throws SAXException {
                         assert fragment;
                         break eofloop;
                     }
+                    // [NOCPP[
                     if (errorHandler != null) {
                         errListUnclosedStartTags(0);
                     }
+                    // ]NOCPP]
                     while (currentPtr >= eltPos) {
                         pop();
                     }

From 5dbe04189f55adea3127929d032dcce9fb2c1e24 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Thu, 2 Jul 2020 16:07:24 +0900
Subject: [PATCH 06/27] Stay in the COMMENT_LESSTHAN state, annotate
 fall-throughs.

---
 src/nu/validator/htmlparser/impl/Tokenizer.java | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/nu/validator/htmlparser/impl/Tokenizer.java b/src/nu/validator/htmlparser/impl/Tokenizer.java
index 43b4de9d..9cf81d3e 100755
--- a/src/nu/validator/htmlparser/impl/Tokenizer.java
+++ b/src/nu/validator/htmlparser/impl/Tokenizer.java
@@ -2853,8 +2853,7 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 continue stateloop;
                             case '<':
                                 appendStrBuf(c);
-                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
-                                continue stateloop;
+                                continue;
                             case '-':
                                 appendStrBuf(c);
                                 state = transition(state, Tokenizer.COMMENT_END_DASH, reconsume, pos);
@@ -2867,13 +2866,14 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 continue;
                             case '\u0000':
                                 c = '\uFFFD';
-                                // fall thru
+                                // CPPONLY: MOZ_FALLTHROUGH;
                             default:
                                 appendStrBuf(c);
                                 state = transition(state, Tokenizer.COMMENT, reconsume, pos);
                                 continue stateloop;
                         }
                     }
+                    // CPPONLY: MOZ_FALLTHROUGH;
                 case COMMENT_LESSTHAN_BANG:
                     for (;;) {
                         if (++pos == endPos) {
@@ -2897,13 +2897,14 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 continue;
                             case '\u0000':
                                 c = '\uFFFD';
-                                // fall thru
+                                // CPPONLY: MOZ_FALLTHROUGH;
                             default:
                                 appendStrBuf(c);
                                 state = transition(state, Tokenizer.COMMENT, reconsume, pos);
                                 continue stateloop;
                         }
                     }
+                    // CPPONLY: MOZ_FALLTHROUGH;
                 case COMMENT_LESSTHAN_BANG_DASH:
                     for (;;) {
                         if (++pos == endPos) {
@@ -2927,13 +2928,14 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 continue;
                             case '\u0000':
                                 c = '\uFFFD';
-                                // fall thru
+                                // CPPONLY: MOZ_FALLTHROUGH;
                             default:
                                 appendStrBuf(c);
                                 state = transition(state, Tokenizer.COMMENT, reconsume, pos);
                                 continue stateloop;
                         }
                     }
+                    // CPPONLY: MOZ_FALLTHROUGH;
                 case COMMENT_LESSTHAN_BANG_DASH_DASH:
                     for (;;) {
                         if (++pos == endPos) {
@@ -2966,7 +2968,7 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 continue;
                             case '\u0000':
                                 c = '\uFFFD';
-                                // fall thru
+                                // CPPONLY: MOZ_FALLTHROUGH;
                             case '!':
                                 errNestedComment();
                                 adjustDoubleHyphenAndAppendToStrBufAndErr(c, reportedConsecutiveHyphens);
@@ -2981,6 +2983,7 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 continue stateloop;
                         }
                     }
+                    // CPPONLY: MOZ_FALLTHROUGH;
                     // XXX reorder point
                 case COMMENT_START_DASH:
                     if (++pos == endPos) {

From 3aa0900a9f4c9f004b6aa954773c93b105cd638e Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Thu, 2 Jul 2020 20:41:36 +0900
Subject: [PATCH 07/27] Limit ambiguous-ampersand wasteful buffering
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This refines 9ce4bd4 by only buffering if we’re actually inside an
attribute value.
---
 src/nu/validator/htmlparser/impl/Tokenizer.java | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/nu/validator/htmlparser/impl/Tokenizer.java b/src/nu/validator/htmlparser/impl/Tokenizer.java
index 9cf81d3e..124e5ce9 100755
--- a/src/nu/validator/htmlparser/impl/Tokenizer.java
+++ b/src/nu/validator/htmlparser/impl/Tokenizer.java
@@ -3539,7 +3539,13 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                         } else if ((c >= '0' && c <= '9')
                                 || (c >= 'A' && c <= 'Z')
                                 || (c >= 'a' && c <= 'z')) {
-                            appendStrBuf(c);
+                            if (returnState == ATTRIBUTE_VALUE_DOUBLE_QUOTED
+                                    || returnState == ATTRIBUTE_VALUE_SINGLE_QUOTED
+                                    || returnState == ATTRIBUTE_VALUE_UNQUOTED) {
+                                appendStrBuf(c);
+                            }
+                            /* The following pos++ is necessary due to how we’ve
+                             * handled the "reconsume" block for this case. */
                             pos++;
                             continue;
                         }

From b453d957cc13cd5b1b4963f2dc4e7871e98802d6 Mon Sep 17 00:00:00 2001
From: Carlos Amengual <carlosame644+git@gmail.com>
Date: Sat, 18 Apr 2020 22:20:16 +0200
Subject: [PATCH 08/27] POM: add an automatic module name entry to MANIFEST.MF.

---
 pom.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pom.xml b/pom.xml
index 41f46725..29da7dee 100644
--- a/pom.xml
+++ b/pom.xml
@@ -158,6 +158,7 @@
             <Bundle-SymbolicName>nu.validator.htmlparser</Bundle-SymbolicName>
             <Bundle-Version>${project.version}</Bundle-Version>
             <Bundle-RequiredExecutionEnvironment>J2SE-1.5</Bundle-RequiredExecutionEnvironment>
+            <Automatic-Module-Name>nu.validator.htmlparser</Automatic-Module-Name>
             <_removeheaders>Built-By,Bnd-LastModified</_removeheaders>
           </instructions>
         </configuration>

From 767568249097290558ca354ff6fdae4552fa000d Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Sat, 4 Jul 2020 13:59:25 +0900
Subject: [PATCH 09/27] Update xom dependency to 1.3.5+ in pom.xml

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 29da7dee..2286f236 100644
--- a/pom.xml
+++ b/pom.xml
@@ -215,7 +215,7 @@
     <dependency>
       <groupId>xom</groupId>
       <artifactId>xom</artifactId>
-      <version>1.1</version>
+      <version>[1.3.5,)</version>
       <scope>compile</scope>
       <optional>true</optional>
     </dependency>

From 2bc081cc37a53c383f5c35410ec4d251a8e7d661 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Sat, 4 Jul 2020 19:50:49 +0900
Subject: [PATCH 10/27] Remove support for RPM packages from pom.xml

---
 pom.xml | 44 --------------------------------------------
 1 file changed, 44 deletions(-)

diff --git a/pom.xml b/pom.xml
index 2286f236..12a75c63 100644
--- a/pom.xml
+++ b/pom.xml
@@ -163,45 +163,6 @@
           </instructions>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>rpm-maven-plugin</artifactId>
-        <configuration>
-          <release>1</release>
-          <copyright>The MIT License</copyright>
-          <group>Development/Java</group>
-          <workarea>/var/tmp/${project.build.finalName}</workarea>
-          <defineStatements>
-            <defineStatement>_javadir ${rpm.java.dir}</defineStatement>
-            <defineStatement>_javadocdir ${rpm.javadoc.dir}</defineStatement>
-          </defineStatements>
-          <mappings>
-            <mapping>
-              <directory>${rpm.java.dir}</directory>
-              <filemode>644</filemode>
-              <username>root</username>
-              <groupname>root</groupname>
-              <sources>
-                <source>
-                  <location>${project.build.directory}/${project.build.finalName}.jar</location>
-                </source>
-              </sources>
-            </mapping>
-            <mapping>
-              <directory>${rpm.javadoc.dir}/${project.build.finalName}</directory>
-              <filemode>644</filemode>
-              <username>root</username>
-              <groupname>root</groupname>
-              <sources>
-                <source>
-                  <location>${project.build.directory}/apidocs</location>
-                </source>
-              </sources>
-            </mapping>
-          </mappings>
-          <install>%__ln_s ${project.build.finalName}.jar %{buildroot}%{_javadir}/${project.name}.jar</install>
-        </configuration>
-      </plugin>
     </plugins>
   </build>
   <dependencies>
@@ -233,9 +194,4 @@
       <scope>test</scope>
     </dependency>
   </dependencies>
-  <properties>
-    <rpm.java.dir>/usr/share/java</rpm.java.dir>
-    <rpm.javadoc.dir>/usr/share/javadoc</rpm.javadoc.dir>
-    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-  </properties>
 </project>

From 2d67fe39c16173c5bc969b43dc5be6c8e1c7894d Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Sat, 4 Jul 2020 19:57:52 +0900
Subject: [PATCH 11/27] Remove support for OSGi bundles from pom.xml

---
 pom.xml | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/pom.xml b/pom.xml
index 12a75c63..0ab61eaa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -24,7 +24,6 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>nu.validator.htmlparser</groupId>
   <artifactId>htmlparser</artifactId>
-  <packaging>bundle</packaging>
   <version>1.4</version>
   <name>htmlparser</name>
   <url>http://about.validator.nu/htmlparser/</url>
@@ -144,25 +143,6 @@
           <skip>true</skip>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.felix</groupId>
-        <artifactId>maven-bundle-plugin</artifactId>
-        <version>2.3.7</version>
-        <extensions>true</extensions>
-        <configuration>
-          <archive>
-            <addMavenDescriptor>false</addMavenDescriptor>
-          </archive>
-          <instructions>
-            <Bundle-Name>${project.name}</Bundle-Name>
-            <Bundle-SymbolicName>nu.validator.htmlparser</Bundle-SymbolicName>
-            <Bundle-Version>${project.version}</Bundle-Version>
-            <Bundle-RequiredExecutionEnvironment>J2SE-1.5</Bundle-RequiredExecutionEnvironment>
-            <Automatic-Module-Name>nu.validator.htmlparser</Automatic-Module-Name>
-            <_removeheaders>Built-By,Bnd-LastModified</_removeheaders>
-          </instructions>
-        </configuration>
-      </plugin>
     </plugins>
   </build>
   <dependencies>

From 18a7948de925da7f85a2e12d97dbcec93b80ac05 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Sat, 4 Jul 2020 20:30:09 +0900
Subject: [PATCH 12/27] Remove com.sun.tools dependency from pom.xml

---
 pom.xml | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/pom.xml b/pom.xml
index 0ab61eaa..c74e2b19 100644
--- a/pom.xml
+++ b/pom.xml
@@ -86,15 +86,6 @@
       <plugin>
         <artifactId>maven-antrun-plugin</artifactId>
         <version>1.7</version>
-        <dependencies>
-          <dependency>
-            <groupId>com.sun</groupId>
-            <artifactId>tools</artifactId>
-            <version>1.5.0</version>
-            <scope>system</scope>
-            <systemPath>${java.home}/../lib/tools.jar</systemPath>
-          </dependency>
-        </dependencies>
         <executions>
           <execution>
             <id>intitialize-sources</id>

From cfb616c3d9819437c49a095dc007d644d66dd7e6 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Sat, 4 Jul 2020 20:31:41 +0900
Subject: [PATCH 13/27] Update source and target values to 1.7 in pom.xml

---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index c74e2b19..f659ffa1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -79,8 +79,8 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
         <configuration>
-          <source>1.5</source>
-          <target>1.5</target>
+          <source>1.7</source>
+          <target>1.7</target>
         </configuration>
       </plugin>
       <plugin>

From 369fa88a505edb7aa5ad98d5199fc5af25a6df50 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Sat, 4 Jul 2020 20:27:12 +0900
Subject: [PATCH 14/27] Remove ruby-gcj

Fixes https://github.com/validator/htmlparser/issues/16
---
 ruby-gcj/DomUtils.java       |  36 ------
 ruby-gcj/README              |  65 -----------
 ruby-gcj/Rakefile            |  77 -------------
 ruby-gcj/extconf.rb          |  45 --------
 ruby-gcj/test/domencoding.rb |   5 -
 ruby-gcj/test/fonts.rb       |  11 --
 ruby-gcj/test/google.html    |  10 --
 ruby-gcj/test/greek.xml      |   2 -
 ruby-gcj/validator.cpp       | 210 -----------------------------------
 9 files changed, 461 deletions(-)
 delete mode 100644 ruby-gcj/DomUtils.java
 delete mode 100644 ruby-gcj/README
 delete mode 100644 ruby-gcj/Rakefile
 delete mode 100644 ruby-gcj/extconf.rb
 delete mode 100644 ruby-gcj/test/domencoding.rb
 delete mode 100644 ruby-gcj/test/fonts.rb
 delete mode 100644 ruby-gcj/test/google.html
 delete mode 100644 ruby-gcj/test/greek.xml
 delete mode 100644 ruby-gcj/validator.cpp

diff --git a/ruby-gcj/DomUtils.java b/ruby-gcj/DomUtils.java
deleted file mode 100644
index dc43da83..00000000
--- a/ruby-gcj/DomUtils.java
+++ /dev/null
@@ -1,36 +0,0 @@
-import java.util.HashSet;
-import org.w3c.dom.Document;
-import org.w3c.dom.Node;
-import org.w3c.dom.Element;
-
-public class DomUtils {
-
-  private static HashSet<Document> pinned_list = new HashSet<Document>();
-
-  public static synchronized void pin(Document d) {
-    pinned_list.add(d);
-  }
-
-  public static synchronized void unpin(Document d) {
-    pinned_list.remove(d);
-  }
-
-  // return all the text content contained by a single element 
-  public static void getElementContent(Element e, StringBuffer b) {
-    for (Node n = e.getFirstChild(); n!=null; n=n.getNextSibling()) {
-      if (n.getNodeType() == n.TEXT_NODE) {
-        b.append(n.getNodeValue());
-      } else if (n.getNodeType() == n.ELEMENT_NODE) {
-        getElementContent((Element) e, b);
-      }
-    }
-  }
-
-  // replace all child nodes of a given element with a single text element
-  public static void setElementContent(Element e, String s) {
-    while (e.hasChildNodes()) {
-      e.removeChild(e.getFirstChild());
-    }
-    e.appendChild(e.getOwnerDocument().createTextNode(s));
-  }
-}
diff --git a/ruby-gcj/README b/ruby-gcj/README
deleted file mode 100644
index b368437f..00000000
--- a/ruby-gcj/README
+++ /dev/null
@@ -1,65 +0,0 @@
-Disclaimer:
-
-  This code is experimental.
-
-  When some people say experimental, they mean "it may not do what it is
-  intended to do; in fact, it might even wipe out your hard drive".  I mean
-  that too.  But I mean something more than that.
-
-  In this case, experimental means that I don't even know what it is intended
-  to do.  I just have a vague vision, and I am trying out various things in
-  the hopes that one of them will work out.
-
-Vision:
-
-  My vague vision is that I would like to see HTML 5 be a success.  For me to
-  consider it to be a success, it needs to be a standard, be interoperable,
-  and be ubiquitous.
-
-  I believe that the Validator.nu parser can be used to bootstrap that
-  process.  It is written in Java.  Has been compiled into JavaScript.  Has
-  been translated into C++ based on the Mozilla libraries with the intent of
-  being included in Firefox.  It very closely tracks to the standard.
-
-  For the moment, the effort is on extending that to another language (Ruby)
-  on a single environment (i.e., Linux).  Once that is complete, intent is to
-  evaluate the results, decide what needs to be changed, and what needs to be
-  done to support other languages and environments.
-
-  The bar I'm setting for myself isn't just another SWIG generated low level
-  interface to a DOM, but rather a best of breed interface; which for Ruby
-  seems to be the one pioneered by Hpricot and adopted by Nokogiri.  Success
-  will mean passing all of the tests from one of those two parsers as well as
-  all of the HTML5 tests.
-
-Build instructions:
-
-  You'll need icu4j and chardet jars.  If you checked out and ran dldeps you
-  are already all set:
-
-    svn co http://svn.versiondude.net/whattf/build/trunk/ build
-    python build/build.py checkout dldeps
-
-  Fedora 11:
-
-    yum install ruby-devel rubygem-rake java-1.5.0-gcj-devel gcc-c++ 
-
-  Ubuntu 9.04:
-
-    apt-get install ruby ruby1.8-dev rake gcj g++
-
-    Also at this time, you need to install a jdk (e.g. sun-java6-jdk), simply
-    because the javac that comes with gcj doesn't support -sourcepath, and
-    I haven't spent the time to find a replacement.
-
-    Finally, make sure that libjaxp1.3-java is *not* installed.
-
-      http://gcc.gnu.org/ml/java/2009-06/msg00055.html
-
-  If this is done, you should be all set.
- 
-    cd htmlparser/ruby-gcj
-    rake test
-
-  If things are successful, the last lines of the output will list the
-  font attributes and values found in the test/google.html file.
diff --git a/ruby-gcj/Rakefile b/ruby-gcj/Rakefile
deleted file mode 100644
index 7b518025..00000000
--- a/ruby-gcj/Rakefile
+++ /dev/null
@@ -1,77 +0,0 @@
-deps    = ENV['deps'] || '../../dependencies'
-icu4j   = "#{deps}/icu4j-4_0.jar"
-chardet = "#{deps}/mozilla/intl/chardet/java/dist/lib/chardet.jar"
-libgcj  = Dir['/usr/share/java/libgcj*.jar'].grep(/gcj[-\d.]*jar$/).sort.last
-
-task :default => %w(headers libs Makefile validator.so)
-
-# headers
-
-hdb = 'nu/validator/htmlparser/dom/HtmlDocumentBuilder'
-task :headers => %W(headers/DomUtils.h headers/#{hdb}.h)
-
-file 'headers/DomUtils.h' => 'DomUtils.java' do |t|
-  mkdir_p %w(classes headers), :verbose => false
-  sh "javac -d classes #{t.prerequisites.first}"
-  sh "gcjh -force -o #{t.name} -cp #{libgcj}:classes DomUtils"
-end
-
-file "headers/#{hdb}.h" => "../src/#{hdb}.java" do |t|
-  mkdir_p %w(classes headers), :verbose => false
-  sh "javac -cp #{icu4j}:#{chardet} -d classes -sourcepath ../src " +
-    t.prerequisites.first
-  sh "gcjh -force -cp classes -o #{t.name} -cp #{libgcj}:classes " + 
-    hdb.gsub('/','.')
-end
-
-# libs
-
-task :libs => %w(htmlparser chardet icu).map {|name| "lib/libnu-#{name}.so"}
-
-htmlparser = Dir['../src/**/*.java'].reject {|name| name.include? '/xom/'}
-file 'lib/libnu-htmlparser.so' => htmlparser + ['DomUtils.java'] do |t|
-  mkdir_p 'lib', :verbose => false
-  sh "gcj -shared --classpath=#{icu4j}:#{chardet} -fPIC " +
-    "-o #{t.name} #{t.prerequisites.join(' ')}"
-end
-
-file 'lib/libnu-chardet.so' => chardet do |t|
-  mkdir_p 'lib', :verbose => false
-  sh "gcj -shared -fPIC -o #{t.name} #{t.prerequisites.join(' ')}"
-end
-
-file 'lib/libnu-icu.so' => icu4j do |t|
-  mkdir_p 'lib', :verbose => false
-  sh "gcj -shared -fPIC -o #{t.name} #{t.prerequisites.join(' ')}"
-end
-
-# module
-
-file 'Makefile' do
-  sh "ruby extconf.rb --with-gcj=#{libgcj}"
-end
-
-file 'validator.so' => %w(Makefile validator.cpp headers/DomUtils.h) do
-  system 'make'
-end
-
-file 'nu/validator.so' do
-  mkdir_p 'nu', :verbose => false
-  system 'ln -s -t nu ../validator.so'
-end
-
-# tasks
-
-task :test => [:default, 'nu/validator.so'] do
-  ENV['LD_LIBRARY_PATH']='lib'
-  sh 'ruby test/fonts.rb test/google.html'
-end
-
-task :clean do
-  rm_rf %W(classes lib nu mkmf.log headers/DomUtils.h headers/#{hdb}.h) + 
-    Dir['*.o'] + Dir['*.so']
-end
-
-task :clobber => :clean do
-  rm_rf %w(headers Makefile)
-end
diff --git a/ruby-gcj/extconf.rb b/ruby-gcj/extconf.rb
deleted file mode 100644
index 415cf430..00000000
--- a/ruby-gcj/extconf.rb
+++ /dev/null
@@ -1,45 +0,0 @@
-require 'mkmf'
-
-# system dependencies
-gcj  = with_config('gcj', '/usr/share/java/libgcj.jar')
-
-# headers for JAXP
-CONFIG['CC'] = 'g++'
-with_cppflags('-xc++') do
-
-  unless find_header('org/w3c/dom/Document.h', 'headers')
-  
-    `jar tf #{gcj}`.split.each do |file|
-      next unless file =~ /\.class$/
-      next unless file =~ /^(javax|org)\/(w3c|xml)/
-      next if file.include? '$'
-    
-      dest = 'headers/' + file.sub(/\.class$/,'.h')
-      name = file.sub(/\.class$/,'').gsub('/','.')
-    
-      next if File.exist? dest
-    
-      cmd = "gcjh -cp #{gcj} -o #{dest} #{name}"
-      puts cmd
-      break unless system cmd
-      system "ruby -pi -e '$_.sub!(/namespace namespace$/," +
-        "\"namespace namespace$\")' #{dest}"
-      system "ruby -pi -e '$_.sub!(/::namespace::/," +
-        "\"::namespace$::\")' #{dest}"
-    end
-
-    exit unless find_header('org/w3c/dom/Document.h', 'headers')
-  end
-
-  find_header 'nu/validator/htmlparser/dom/HtmlDocumentBuilder.h', 'headers'
-end
-
-# Java libraries
-Config::CONFIG['CC'] = 'g++ -shared'
-dir_config('nu-htmlparser', nil, 'lib')
-have_library 'nu-htmlparser'
-have_library 'nu-icu'
-have_library 'nu-chardet'
-
-# Ruby library
-create_makefile 'nu/validator'
diff --git a/ruby-gcj/test/domencoding.rb b/ruby-gcj/test/domencoding.rb
deleted file mode 100644
index 1beb94c1..00000000
--- a/ruby-gcj/test/domencoding.rb
+++ /dev/null
@@ -1,5 +0,0 @@
-require 'nu/validator'
-
-ARGV.each do |arg|
-  puts Nu::Validator::parse(open(arg)).root.name
-end
diff --git a/ruby-gcj/test/fonts.rb b/ruby-gcj/test/fonts.rb
deleted file mode 100644
index 595e3ae0..00000000
--- a/ruby-gcj/test/fonts.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-require 'nu/validator'
-require 'open-uri'
-
-ARGV.each do |arg|
-  doc = Nu::Validator::parse(open(arg))
-  doc.xpath("//*[local-name()='font']").each do |font|
-    font.attributes.each do |name, attr|
-      puts "#{name} => #{attr.value}"
-    end
-  end
-end
diff --git a/ruby-gcj/test/google.html b/ruby-gcj/test/google.html
deleted file mode 100644
index 8d2183b2..00000000
--- a/ruby-gcj/test/google.html
+++ /dev/null
@@ -1,10 +0,0 @@
-<!doctype html><html><head><meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"><title>Google</title><script>window.google={kEI:"vLhASujeGpTU9QT2iOnWAQ",kEXPI:"17259",kCSIE:"17259",kHL:"en"};
-window.google.sn="webhp";window.google.timers={load:{t:{start:(new Date).getTime()}}};try{window.google.pt=window.gtbExternal&&window.gtbExternal.pageT()||window.external&&window.external.pageT}catch(b){}
-window.google.jsrt_kill=1;
-var _gjwl=location;function _gjuc(){var e=_gjwl.href.indexOf("#");if(e>=0){var a=_gjwl.href.substring(e);if(a.indexOf("&q=")>0||a.indexOf("#q=")>=0){a=a.substring(1);if(a.indexOf("#")==-1){for(var c=0;c<a.length;){var d=c;if(a.charAt(d)=="&")++d;var b=a.indexOf("&",d);if(b==-1)b=a.length;var f=a.substring(d,b);if(f.indexOf("fp=")==0){a=a.substring(0,c)+a.substring(b,a.length);b=c}else if(f=="cad=h")return 0;c=b}_gjwl.href="/search?"+a+"&cad=h";return 1}}}return 0}function _gjp(){!(window._gjwl.hash&&
-window._gjuc())&&setTimeout(_gjp,500)};
-window._gjp && _gjp();</script><style>td{line-height:.8em;}.gac_c{line-height:normal;}form{margin-bottom:20px;}body,td,a,p,.h{font-family:arial,sans-serif}.h{color:#36c;font-size:20px}.q{color:#00c}.ts td{padding:0}.ts{border-collapse:collapse}#gbar{height:22px;padding-left:0px}.gbh,.gbd{border-top:1px solid #c9d7f1;font-size:1px}.gbh{height:0;position:absolute;top:24px;width:100%}#guser{padding-bottom:7px !important;text-align:right}#gbar,#guser{font-size:13px;padding-top:1px !important}@media all{.gb1,.gb3{height:22px;margin-right:.5em;vertical-align:top}#gbar{float:left}}a.gb1,a.gb3{color:#00c !important}.gb3{text-decoration:none}</style><script>google.y={};google.x=function(e,g){google.y[e.id]=[e,g];return false};</script></head><body bgcolor=#ffffff text=#000000 link=#0000cc vlink=#551a8b alink=#ff0000 onload="document.f.q.focus();if(document.images)new Image().src='/images/nav_logo4.png'" topmargin=3 marginheight=3><textarea id=csi style=display:none></textarea><iframe name=wgjf style="display:none"></iframe><div id=gbar><nobr><b class=gb1>Web</b> <a href="http://images.google.com/imghp?hl=en&tab=wi" class=gb1>Images</a> <a href="http://video.google.com/?hl=en&tab=wv" class=gb1>Video</a> <a href="http://maps.google.com/maps?hl=en&tab=wl" class=gb1>Maps</a> <a href="http://news.google.com/nwshp?hl=en&tab=wn" class=gb1>News</a> <a href="http://www.google.com/prdhp?hl=en&tab=wf" class=gb1>Shopping</a> <a href="http://mail.google.com/mail/?hl=en&tab=wm" class=gb1>Gmail</a> <a href="http://www.google.com/intl/en/options/" class=gb3><u>more</u> &raquo;</a></nobr></div><div id=guser width=100%><nobr><a href="/url?sa=p&pref=ig&pval=3&q=http://www.google.com/ig%3Fhl%3Den%26source%3Diglk&usg=AFQjCNFA18XPfgb7dKnXfKz7x7g1GDH1tg">iGoogle</a> | <a href="https://www.google.com/accounts/Login?hl=en&continue=http://www.google.com/">Sign in</a></nobr></div><div class=gbh style=left:0></div><div class=gbh style=right:0></div><center><br clear=all id=lgpd><img alt="Google" height=110 src="/intl/en_ALL/images/logo.gif" width=276 id=logo onload="window.lol&&lol()"><br><br><form action="/search" name=f><table cellpadding=0 cellspacing=0><tr valign=top><td width=25%>&nbsp;</td><td align=center nowrap><input name=hl type=hidden value=en><input type=hidden name=ie value="ISO-8859-1"><input autocomplete="off" maxlength=2048 name=q size=55 title="Google Search" value=""><br><input name=btnG type=submit value="Google Search"><input name=btnI type=submit value="I'm Feeling Lucky"></td><td nowrap width=25% align=left><font size=-2>&nbsp;&nbsp;<a href=/advanced_search?hl=en>Advanced Search</a><br>&nbsp;&nbsp;<a href=/preferences?hl=en>Preferences</a><br>&nbsp;&nbsp;<a href=/language_tools?hl=en>Language Tools</a></font></td></tr></table></form><br><font size=-1><a href="/aclk?sa=L&ai=CqVchLbNASrv7IZa68gS13KTwAc3__IMB29PoogzB2ZzZExABIMFUUK_O0JX______wFgyQaqBAlP0BcDOBRYhqw&num=1&sig=AGiWqty21CD7ixNXZILwCnH7c_3n9v2-tg&q=http://www.allforgood.org#source=hpp">Find an opportunity to volunteer</a> in your community today.</font><br><br><br><font size=-1><a href="/intl/en/ads/">Advertising&nbsp;Programs</a> - <a href="/services/">Business Solutions</a> - <a href="/intl/en/about.html">About Google</a></font><p><font size=-2>&copy;2009 - <a href="/intl/en/privacy.html">Privacy</a></font></p></center><div id=xjsd></div><div id=xjsi><script>if(google.y)google.y.first=[];if(google.y)google.y.first=[];google.dstr=[];google.rein=[];window.setTimeout(function(){var a=document.createElement("script");a.src="/extern_js/f/CgJlbhICdXMgACswCjggQAgsKzAOOAUsKzAYOAQsKzAlOMmIASwrMCY4BCwrMCc4ACw/1t0T7hspHT4.js";(document.getElementById("xjsd")||document.body).appendChild(a)},0);
-;google.y.first.push(function(){google.ac.i(document.f,document.f.q,'','')});google.xjs&&google.j&&google.j.xi&&google.j.xi()</script></div><script>(function(){
-function a(){google.timers.load.t.ol=(new Date).getTime();google.report&&google.report(google.timers.load,{ei:google.kEI,e:google.kCSIE})}if(window.addEventListener)window.addEventListener("load",a,false);else if(window.attachEvent)window.attachEvent("onload",a);google.timers.load.t.prt=(new Date).getTime();
-})();
-</script>
\ No newline at end of file
diff --git a/ruby-gcj/test/greek.xml b/ruby-gcj/test/greek.xml
deleted file mode 100644
index a14d23eb..00000000
--- a/ruby-gcj/test/greek.xml
+++ /dev/null
@@ -1,2 +0,0 @@
-<?xml version='1.0' encoding='iso-8859-7'?>
-<root/>
diff --git a/ruby-gcj/validator.cpp b/ruby-gcj/validator.cpp
deleted file mode 100644
index aadd24ab..00000000
--- a/ruby-gcj/validator.cpp
+++ /dev/null
@@ -1,210 +0,0 @@
-#include <gcj/cni.h>
-
-#include <java/io/ByteArrayInputStream.h>
-#include <java/lang/System.h>
-#include <java/lang/Throwable.h>
-#include <java/util/ArrayList.h>
-#include <javax/xml/xpath/XPath.h>
-#include <javax/xml/xpath/XPathFactory.h>
-#include <javax/xml/xpath/XPathExpression.h>
-#include <javax/xml/xpath/XPathConstants.h>
-#include <javax/xml/parsers/DocumentBuilderFactory.h>
-#include <javax/xml/parsers/DocumentBuilder.h>
-#include <org/w3c/dom/Attr.h>
-#include <org/w3c/dom/Document.h>
-#include <org/w3c/dom/Element.h>
-#include <org/w3c/dom/NodeList.h>
-#include <org/w3c/dom/NamedNodeMap.h>
-#include <org/xml/sax/InputSource.h>
-
-#include "nu/validator/htmlparser/dom/HtmlDocumentBuilder.h"
-
-#include "DomUtils.h"
-
-#include "ruby.h"
-
-using namespace java::io;
-using namespace java::lang;
-using namespace java::util;
-using namespace javax::xml::parsers;
-using namespace javax::xml::xpath;
-using namespace nu::validator::htmlparser::dom;
-using namespace org::w3c::dom;
-using namespace org::xml::sax;
-
-static VALUE jaxp_Document;
-static VALUE jaxp_Attr;
-static VALUE jaxp_Element;
-static ID ID_read;
-static ID ID_doc;
-static ID ID_element;
-
-// convert a Java string into a Ruby string
-static VALUE j2r(String *string) {
-  if (string == NULL) return Qnil;
-  jint len = JvGetStringUTFLength(string);
-  char buf[len];
-  JvGetStringUTFRegion(string, 0, len, buf);
-  return rb_str_new(buf, len);
-}
-
-// convert a Ruby string into a Java string
-static String *r2j(VALUE string) {
-  return JvNewStringUTF(RSTRING(string)->ptr);
-}
-
-// release the Java Document associated with this Ruby Document
-static void vnu_document_free(Document *doc) {
-  DomUtils::unpin(doc);
-}
-
-// Nu::Validator::parse( string|file )
-static VALUE vnu_parse(VALUE self, VALUE input) {
-  HtmlDocumentBuilder *parser = new HtmlDocumentBuilder();
-  
-  // read file-like objects into memory.  TODO: buffer such objects
-  if (rb_respond_to(input, ID_read))
-    input = rb_funcall(input, ID_read, 0);
-
-  // convert input in to a ByteArrayInputStream
-  jbyteArray bytes = JvNewByteArray(RSTRING(input)->len);
-  memcpy(elements(bytes), RSTRING(input)->ptr, RSTRING(input)->len);
-  InputSource *source = new InputSource(new ByteArrayInputStream(bytes));
-
-  // parse, pin, and wrap
-  Document *doc = parser->parse(source);
-  DomUtils::pin(doc);
-  return Data_Wrap_Struct(jaxp_Document, NULL, vnu_document_free, doc);
-}
-
-// Jaxp::parse( string|file )
-static VALUE jaxp_parse(VALUE self, VALUE input) {
-  DocumentBuilderFactory *factory = DocumentBuilderFactory::newInstance();
-  DocumentBuilder *parser = factory->newDocumentBuilder();
-   
-  // read file-like objects into memory.  TODO: buffer such objects
-  if (rb_respond_to(input, ID_read))
-    input = rb_funcall(input, ID_read, 0);
- 
-  try {
-    jbyteArray bytes = JvNewByteArray(RSTRING(input)->len);
-    memcpy(elements(bytes), RSTRING(input)->ptr, RSTRING(input)->len);
-    Document *doc = parser->parse(new ByteArrayInputStream(bytes));
-    DomUtils::pin(doc);
-    return Data_Wrap_Struct(jaxp_Document, NULL, vnu_document_free, doc);
-  } catch (java::lang::Throwable *ex) {
-    ex->printStackTrace();
-    return Qnil;
-  }
-}
-
-
-// Nu::Validator::Document#encoding
-static VALUE jaxp_document_encoding(VALUE rdoc) {
-  Document *jdoc;
-  Data_Get_Struct(rdoc, Document, jdoc);
-  return j2r(jdoc->getXmlEncoding());
-}
-
-// Nu::Validator::Document#root
-static VALUE jaxp_document_root(VALUE rdoc) {
-  Document *jdoc;
-  Data_Get_Struct(rdoc, Document, jdoc);
-
-  Element *jelement = jdoc->getDocumentElement();
-  if (jelement==NULL) return Qnil;
-
-  VALUE relement = Data_Wrap_Struct(jaxp_Element, NULL, NULL, jelement);
-  rb_ivar_set(relement, ID_doc, rdoc);
-  return relement;
-}
-
-// Nu::Validator::Document#xpath
-static VALUE jaxp_document_xpath(VALUE rdoc, VALUE path) {
-  Document *jdoc;
-  Data_Get_Struct(rdoc, Document, jdoc);
-
-  Element *jelement = jdoc->getDocumentElement();
-  if (jelement==NULL) return Qnil;
-
-  XPath *xpath = XPathFactory::newInstance()->newXPath();
-  XPathExpression *expr = xpath->compile(r2j(path));
-  NodeList *list = (NodeList*) expr->evaluate(jdoc, XPathConstants::NODESET);
-
-  VALUE result = rb_ary_new();
-  for (int i=0; i<list->getLength(); i++) {
-    VALUE relement = Data_Wrap_Struct(jaxp_Element, NULL, NULL, list->item(i));
-    rb_ivar_set(relement, ID_doc, rdoc);
-    rb_ary_push(result, relement);
-  }
-  return result;
-}
-
-// Nu::Validator::Element#name
-static VALUE jaxp_element_name(VALUE relement) {
-  Element *jelement;
-  Data_Get_Struct(relement, Element, jelement);
-  return j2r(jelement->getNodeName());
-}
-
-// Nu::Validator::Element#attributes
-static VALUE jaxp_element_attributes(VALUE relement) {
-  Element *jelement;
-  Data_Get_Struct(relement, Element, jelement);
-  VALUE result = rb_hash_new();
-  NamedNodeMap *map = jelement->getAttributes();
-  for (int i=0; i<map->getLength(); i++) {
-    Attr *jattr = (Attr *) map->item(i);
-    VALUE rattr = Data_Wrap_Struct(jaxp_Attr, NULL, NULL, jattr);
-    rb_ivar_set(rattr, ID_element, relement);
-    rb_hash_aset(result, j2r(jattr->getName()), rattr);
-  }
-  return result;
-}
-
-// Nu::Validator::Attribute#value
-static VALUE jaxp_attribute_value(VALUE rattribute) {
-  Attr *jattribute;
-  Data_Get_Struct(rattribute, Attr, jattribute);
-  return j2r(jattribute->getValue());
-}
-
-typedef VALUE (ruby_method)(...);
-
-// Nu::Validator module initialization
-extern "C" void Init_validator() {
-  JvCreateJavaVM(NULL);
-  JvAttachCurrentThread(NULL, NULL);
-  JvInitClass(&DomUtils::class$);
-  JvInitClass(&XPathFactory::class$);
-  JvInitClass(&XPathConstants::class$);
-
-  VALUE jaxp = rb_define_module("Jaxp");
-  rb_define_singleton_method(jaxp, "parse", (ruby_method*)&jaxp_parse, 1);
-
-  VALUE nu = rb_define_module("Nu");
-  VALUE validator = rb_define_module_under(nu, "Validator");
-  rb_define_singleton_method(validator, "parse", (ruby_method*)&vnu_parse, 1);
-
-  jaxp_Document = rb_define_class_under(jaxp, "Document", rb_cObject);
-  rb_define_method(jaxp_Document, "encoding", 
-    (ruby_method*)&jaxp_document_encoding, 0);
-  rb_define_method(jaxp_Document, "root", 
-    (ruby_method*)&jaxp_document_root, 0);
-  rb_define_method(jaxp_Document, "xpath", 
-    (ruby_method*)&jaxp_document_xpath, 1);
-
-  jaxp_Element = rb_define_class_under(jaxp, "Element", rb_cObject);
-  rb_define_method(jaxp_Element, "name", 
-    (ruby_method*)&jaxp_element_name, 0);
-  rb_define_method(jaxp_Element, "attributes", 
-    (ruby_method*)&jaxp_element_attributes, 0);
-
-  jaxp_Attr = rb_define_class_under(jaxp, "Attr", rb_cObject);
-  rb_define_method(jaxp_Attr, "value", 
-    (ruby_method*)&jaxp_attribute_value, 0);
-
-  ID_read = rb_intern("read");
-  ID_doc  = rb_intern("@doc");
-  ID_element = rb_intern("@element");
-}

From 58f05e79541f1c6b8ad2624e3f1fc8b43aa11434 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Tue, 28 Jul 2020 11:12:02 +0900
Subject: [PATCH 15/27] Add .mailmap

---
 .mailmap | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 .mailmap

diff --git a/.mailmap b/.mailmap
new file mode 100644
index 00000000..47921759
--- /dev/null
+++ b/.mailmap
@@ -0,0 +1,7 @@
+Henri Sivonen <hsivonen@hsivonen.fi> hsivonen <none@none>
+Henri Sivonen <hsivonen@hsivonen.fi> unknown <none@none>
+Henri Sivonen <hsivonen@hsivonen.fi> html5-readonly <none@none>
+Henri Sivonen <hsivonen@hsivonen.fi> convert-repo <none@none>
+Michael[tm] Smith <mike@w3.org> Michael(tm) Smith <mike@w3.org>
+Michael[tm] Smith <mike@w3.org> sideshowbarker <none@none>
+Sam Ruby <rubys@intertwingly.net> rubys <none@none>

From c37ed3be83b0498121f9623b96850bc93ef8f8cc Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Tue, 28 Jul 2020 11:26:43 +0900
Subject: [PATCH 16/27] Add .settings/ to .gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index c50ae13c..7eea6ba3 100755
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 .classpath
 .project
+.settings/
 eclipse-classes
 classes
 dist

From 19d24924e8d5814838afad249bcd057c67a27c97 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Mon, 3 Aug 2020 10:45:31 +0900
Subject: [PATCH 17/27] Update comment re: named-charref spec-code clash

---
 src/nu/validator/htmlparser/impl/Tokenizer.java | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/nu/validator/htmlparser/impl/Tokenizer.java b/src/nu/validator/htmlparser/impl/Tokenizer.java
index 124e5ce9..9b67a2c3 100755
--- a/src/nu/validator/htmlparser/impl/Tokenizer.java
+++ b/src/nu/validator/htmlparser/impl/Tokenizer.java
@@ -3455,6 +3455,13 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                      * LATIN CAPITAL LETTER Z, or U+0061 LATIN
                                      * SMALL LETTER A to U+007A LATIN SMALL
                                      * LETTER Z, then, for historical reasons,
+                                     * ---
+                                     * FIXME: The remainder of this comment
+                                     * doesn't actually match the current spec.
+                                     * But the code behavior here also doesn't
+                                     * match the current spec, so it's unclear
+                                     * how to reconcile it...
+                                     * ---
                                      * all the characters that were matched
                                      * after the U+0026 AMPERSAND (&) must be
                                      * unconsumed, and nothing is returned.

From 24477298733345b190585223bf470c4644e22ed2 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Mon, 3 Aug 2020 11:08:13 +0900
Subject: [PATCH 18/27] Flush w/ emitOrAppendCharRefBuf, not ad-hoc code

---
 src/nu/validator/htmlparser/impl/Tokenizer.java | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/nu/validator/htmlparser/impl/Tokenizer.java b/src/nu/validator/htmlparser/impl/Tokenizer.java
index 9b67a2c3..b29dcd17 100755
--- a/src/nu/validator/htmlparser/impl/Tokenizer.java
+++ b/src/nu/validator/htmlparser/impl/Tokenizer.java
@@ -3546,10 +3546,9 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                         } else if ((c >= '0' && c <= '9')
                                 || (c >= 'A' && c <= 'Z')
                                 || (c >= 'a' && c <= 'z')) {
-                            if (returnState == ATTRIBUTE_VALUE_DOUBLE_QUOTED
-                                    || returnState == ATTRIBUTE_VALUE_SINGLE_QUOTED
-                                    || returnState == ATTRIBUTE_VALUE_UNQUOTED) {
-                                appendStrBuf(c);
+                            emitOrAppendCharRefBuf(returnState);
+                            if ((returnState & DATA_AND_RCDATA_MASK) == 0) {
+                                cstart = pos;
                             }
                             /* The following pos++ is necessary due to how we’ve
                              * handled the "reconsume" block for this case. */

From a3d911d1dc6410eb9a955d3b73b890ccee8b6c38 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Mon, 3 Aug 2020 11:12:50 +0900
Subject: [PATCH 19/27] Don't consume char at AMBIGUOUS_AMPERSAND start

---
 .../validator/htmlparser/impl/Tokenizer.java  | 22 ++++++++++---------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/src/nu/validator/htmlparser/impl/Tokenizer.java b/src/nu/validator/htmlparser/impl/Tokenizer.java
index b29dcd17..2b2906e5 100755
--- a/src/nu/validator/htmlparser/impl/Tokenizer.java
+++ b/src/nu/validator/htmlparser/impl/Tokenizer.java
@@ -3533,14 +3533,14 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                     }
                     // XXX reorder point
                 case AMBIGUOUS_AMPERSAND:
+                    /*
+                     * Unlike the definition is the spec, we don't consume the
+                     * next input character right away when entering this state;
+                     * that's because our current implementation differs from
+                     * the spec in that we've already consumed the relevant
+                     * character *before* entering this state.
+                     */
                     ampersandloop: for (;;) {
-                        if (reconsume) {
-                            if (++pos == endPos) {
-                                break stateloop;
-                            }
-                            pos--;
-                            c = checkChar(buf, pos);
-                        }
                         if (c == ';') {
                             errNoNamedCharacterMatch();
                         } else if ((c >= '0' && c <= '9')
@@ -3550,11 +3550,13 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                             if ((returnState & DATA_AND_RCDATA_MASK) == 0) {
                                 cstart = pos;
                             }
-                            /* The following pos++ is necessary due to how we’ve
-                             * handled the "reconsume" block for this case. */
-                            pos++;
+                            if (++pos == endPos) {
+                                break stateloop;
+                            }
+                            c = checkChar(buf, pos);
                             continue;
                         }
+                        reconsume = true;
                         state = transition(state, returnState, reconsume, pos);
                         continue stateloop;
                     }

From 61325c31b46779bf2d97bc6e5dfb113c544df3d0 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Wed, 5 Aug 2020 15:14:29 +0900
Subject: [PATCH 20/27] Fix AMBIGUOUS_AMPERSAND state

---
 src/nu/validator/htmlparser/impl/Tokenizer.java | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/nu/validator/htmlparser/impl/Tokenizer.java b/src/nu/validator/htmlparser/impl/Tokenizer.java
index 2b2906e5..bfa3631e 100755
--- a/src/nu/validator/htmlparser/impl/Tokenizer.java
+++ b/src/nu/validator/htmlparser/impl/Tokenizer.java
@@ -3546,13 +3546,14 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                         } else if ((c >= '0' && c <= '9')
                                 || (c >= 'A' && c <= 'Z')
                                 || (c >= 'a' && c <= 'z')) {
+                            if (++pos == endPos) {
+                                break stateloop;
+                            }
+                            appendCharRefBuf(c);
                             emitOrAppendCharRefBuf(returnState);
                             if ((returnState & DATA_AND_RCDATA_MASK) == 0) {
                                 cstart = pos;
                             }
-                            if (++pos == endPos) {
-                                break stateloop;
-                            }
                             c = checkChar(buf, pos);
                             continue;
                         }

From d205660155b98af341c1dc6f8a1121e029ae7204 Mon Sep 17 00:00:00 2001
From: Carlos Amengual <carlosame644+git@gmail.com>
Date: Sun, 5 Jul 2020 14:03:29 +0200
Subject: [PATCH 21/27] Fix broken maven build due to maven-antrun-plugin being
 unable to locate javac

Fixes: 7b05376c
---
 pom.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pom.xml b/pom.xml
index f659ffa1..5ca80268 100644
--- a/pom.xml
+++ b/pom.xml
@@ -111,6 +111,7 @@
             </goals>
             <configuration>
               <target>
+                <property name="build.compiler" value="extJavac"/>
                 <property name="translator.sources" value="${basedir}/translator-src"/>
                 <property name="translator.classes" value="${project.build.directory}/translator-classes"/>
                 <mkdir dir="${translator.classes}"/>

From 7722be4733943e973dd83b1b41c21c8583a51f11 Mon Sep 17 00:00:00 2001
From: Carlos Amengual <carlosame644+git@gmail.com>
Date: Sun, 5 Jul 2020 15:58:54 +0200
Subject: [PATCH 22/27] General cleanup of the POM file, prerequisite for
 modularization

- Update the file description.
- Enforce the usage of UTF-8 encoding (instead of platform-dependent).
- Build source and javadoc packages.
- Set specific versions for plugins.
- Omit the Maven descriptor in the Jar file (which was lost in 2e8c502b).
---
 pom.xml | 55 +++++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 49 insertions(+), 6 deletions(-)

diff --git a/pom.xml b/pom.xml
index 5ca80268..39fdb1ae 100644
--- a/pom.xml
+++ b/pom.xml
@@ -32,20 +32,18 @@
     Usage notes for this POM:
 
     To build without signing, run:
-    mvn clean source:jar javadoc:jar repository:bundle-create
-    (enter 0 <return> when prompted)
+    mvn clean package
 
     To build and sign, run:
-    mvn clean source:jar javadoc:jar package gpg:sign repository:bundle-create
-    (enter 0 <return> when prompted)
+    mvn clean package gpg:sign
 
-    This POM file is used for creating the bundle for distribution via the
+    This POM file is used for creating the artifacts for distribution via the
     Maven Central Repository. It is not used as part of the normal development
     process of the parser and the maintainer of the parser (Henri Sivonen) 
     isn't experienced in POM tweaking. If you need this POM to do something
     that it currently does not do or do something better, you need to write
     the changes you need yourself and contribute a patch via
-    http://bugzilla.validator.nu/
+    https://github.com/validator/htmlparser
   -->
   <developers>
     <developer>
@@ -71,13 +69,30 @@
     <connection>scm:hg:http://hg.mozilla.org/projects/htmlparser/</connection>
     <url>http://hg.mozilla.org/projects/htmlparser/</url>
   </scm>
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
   <build>
     <sourceDirectory>${project.build.directory}/src</sourceDirectory>
     <testSourceDirectory>${basedir}/test-src</testSourceDirectory>
     <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-source-plugin</artifactId>
+        <version>3.2.1</version>
+        <executions>
+          <execution>
+              <id>attach-sources</id>
+              <goals>
+                <goal>jar-no-fork</goal>
+              </goals>
+            </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.8.1</version>
         <configuration>
           <source>1.7</source>
           <target>1.7</target>
@@ -128,13 +143,41 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <version>3.2.0</version>
+        <configuration>
+        <archive>
+          <addMavenDescriptor>false</addMavenDescriptor>
+        </archive>
+        </configuration>
+      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
+        <version>3.0.0-M5</version>
         <configuration>
           <skip>true</skip>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <version>3.1.0</version>
+        <configuration>
+          <source>7</source>
+          <quiet>true</quiet>
+        </configuration>
+        <executions>
+          <execution>
+            <id>attach-javadocs</id>
+            <goals>
+              <goal>jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
   <dependencies>

From 85705859a5f01f7ef0e2f29f052a41eab0a66861 Mon Sep 17 00:00:00 2001
From: Carlos Amengual <carlosame644+git@gmail.com>
Date: Sun, 5 Jul 2020 16:06:31 +0200
Subject: [PATCH 23/27] Maven POM: update dependency version for jsontools.

Not required for modularization, but no reason to stay on the old version.
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 39fdb1ae..da54d8f8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -205,7 +205,7 @@
     <dependency>
       <groupId>com.sdicons.jsontools</groupId>
       <artifactId>jsontools-core</artifactId>
-      <version>1.4</version>
+      <version>1.7</version>
       <scope>test</scope>
     </dependency>
   </dependencies>

From 2ad928a24e7401e9e31c0ccc925da16dceb48c5f Mon Sep 17 00:00:00 2001
From: Carlos Amengual <carlosame644+git@gmail.com>
Date: Sun, 5 Jul 2020 16:22:33 +0200
Subject: [PATCH 24/27] Maven POM: update the 'scm' information

---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index da54d8f8..789fa5ee 100644
--- a/pom.xml
+++ b/pom.xml
@@ -66,8 +66,8 @@
     </license>
   </licenses>
   <scm>
-    <connection>scm:hg:http://hg.mozilla.org/projects/htmlparser/</connection>
-    <url>http://hg.mozilla.org/projects/htmlparser/</url>
+    <connection>scm:git:https://github.com/validator/htmlparser.git</connection>
+    <url>https://github.com/validator/htmlparser</url>
   </scm>
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

From 4e33cbcaf7f79baf70aba1fd8d4fb6f69f282ef3 Mon Sep 17 00:00:00 2001
From: Carlos Amengual <carlosame644+git@gmail.com>
Date: Sun, 5 Jul 2020 16:33:06 +0200
Subject: [PATCH 25/27] Modularize the library, modifying the POM and adding a
 module-info

Requires JDK 12 or higher (due to bug in JDK 11), but produces bytecode for
Java 7 and 11.
---
 pom.xml              | 47 +++++++++++++++++++++++++++++++++++++-------
 src/module-info.java | 39 ++++++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+), 7 deletions(-)
 create mode 100644 src/module-info.java

diff --git a/pom.xml b/pom.xml
index 789fa5ee..4331a6da 100644
--- a/pom.xml
+++ b/pom.xml
@@ -31,7 +31,7 @@
   <!--
     Usage notes for this POM:
 
-    To build without signing, run:
+    To build without signing, run (with JDK 12 or higher):
     mvn clean package
 
     To build and sign, run:
@@ -93,14 +93,41 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
         <version>3.8.1</version>
-        <configuration>
-          <source>1.7</source>
-          <target>1.7</target>
-        </configuration>
+        <executions>
+          <execution>
+            <id>default-compile</id>
+            <configuration>
+              <release>11</release>
+            </configuration>
+          </execution>
+          <execution>
+            <id>base-compile</id>
+            <goals>
+              <goal>compile</goal>
+            </goals>
+            <configuration>
+              <source>1.7</source>
+              <target>1.7</target>
+              <excludes>
+                <exclude>module-info.java</exclude>
+              </excludes>
+            </configuration>
+          </execution>
+          <execution>
+            <id>default-testCompile</id>
+              <goals>
+                <goal>testCompile</goal>
+              </goals>
+              <configuration>
+                <source>1.7</source>
+                <target>1.7</target>
+              </configuration>
+          </execution>
+        </executions>
       </plugin>
       <plugin>
         <artifactId>maven-antrun-plugin</artifactId>
-        <version>1.7</version>
+        <version>3.0.0</version>
         <executions>
           <execution>
             <id>intitialize-sources</id>
@@ -184,7 +211,13 @@
     <dependency>
       <groupId>com.ibm.icu</groupId>
       <artifactId>icu4j</artifactId>
-      <version>4.0.1</version>
+      <!--
+        A module name was set for version 64.1 (see
+        https://unicode-org.atlassian.net/browse/ICU-20192) but
+        only 64.2 (and later) is available from Maven Central.
+        See also https://github.com/unicode-org/icu/pull/193
+      -->
+      <version>[64.2,)</version>
       <scope>compile</scope>
       <optional>true</optional>
     </dependency>
diff --git a/src/module-info.java b/src/module-info.java
new file mode 100644
index 00000000..729b91c0
--- /dev/null
+++ b/src/module-info.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2020 Carlos Amengual
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+module nu.validator.htmlparser {
+    exports nu.validator.htmlparser.annotation;
+    exports nu.validator.htmlparser.common;
+    exports nu.validator.htmlparser.dom;
+    exports nu.validator.htmlparser.extra;
+    exports nu.validator.htmlparser.impl;
+    exports nu.validator.htmlparser.io;
+    exports nu.validator.htmlparser.rewindable;
+    exports nu.validator.htmlparser.sax;
+    exports nu.validator.htmlparser.xom;
+    exports nu.validator.saxtree;
+
+    requires transitive java.xml;
+    requires static nu.xom;
+    requires static com.ibm.icu;
+    requires static jchardet;
+}

From b047f935b2f7f702102c51344a897e99f2c0976d Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Mon, 6 Jul 2020 13:25:33 +0900
Subject: [PATCH 26/27] Maven POM: Use <doclint>none</doclint> for javadoc

Without this change, the Maven javadoc build fails with fatal errors.
---
 pom.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pom.xml b/pom.xml
index 4331a6da..355bae5e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -195,6 +195,7 @@
         <configuration>
           <source>7</source>
           <quiet>true</quiet>
+          <doclint>none</doclint>
         </configuration>
         <executions>
           <execution>

From cf8e4a41fe1c1b7b02c5d88e9b9a27dabce0c8af Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Mon, 6 Jul 2020 14:08:06 +0900
Subject: [PATCH 27/27] Maven POM: Use https URLs

---
 pom.xml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pom.xml b/pom.xml
index 355bae5e..31b2bace 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
   <artifactId>htmlparser</artifactId>
   <version>1.4</version>
   <name>htmlparser</name>
-  <url>http://about.validator.nu/htmlparser/</url>
+  <url>https://about.validator.nu/htmlparser/</url>
   <description>The Validator.nu HTML Parser is an implementation of the HTML5 parsing algorithm in Java for applications. The parser is designed to work as a drop-in replacement for the XML parser in applications that already support XHTML 1.x content with an XML parser and use SAX, DOM or XOM to interface with the parser.</description>
   <!--
     Usage notes for this POM:
@@ -50,18 +50,18 @@
       <id>hsivonen</id>
       <name>Henri Sivonen</name>
       <email>hsivonen@iki.fi</email>
-      <url>http://hsivonen.iki.fi/</url>
+      <url>https://hsivonen.iki.fi/</url>
     </developer>
   </developers>
   <licenses>
     <license>
       <name>The MIT License</name>
-      <url>http://www.opensource.org/licenses/mit-license.php</url>
+      <url>https://www.opensource.org/licenses/mit-license.php</url>
       <distribution>repo</distribution>
     </license>
     <license>
       <name>The (New) BSD License</name>
-      <url>http://www.opensource.org/licenses/bsd-license.php</url>
+      <url>https://www.opensource.org/licenses/bsd-license.php</url>
       <distribution>repo</distribution>
     </license>
   </licenses>