Formalize bogus comment state.

Author: RReverser

lib/tokenizer/index.js

@@ -63,7 +63,6 @@ var DATA_STATE = 'DATA_STATE',
     AFTER_ATTRIBUTE_VALUE_QUOTED_STATE = 'AFTER_ATTRIBUTE_VALUE_QUOTED_STATE',
     SELF_CLOSING_START_TAG_STATE = 'SELF_CLOSING_START_TAG_STATE',
     BOGUS_COMMENT_STATE = 'BOGUS_COMMENT_STATE',
-    BOGUS_COMMENT_STATE_CONTINUATION = 'BOGUS_COMMENT_STATE_CONTINUATION',
     MARKUP_DECLARATION_OPEN_STATE = 'MARKUP_DECLARATION_OPEN_STATE',
     COMMENT_START_STATE = 'COMMENT_START_STATE',
     COMMENT_START_DASH_STATE = 'COMMENT_START_DASH_STATE',
@@ -724,8 +723,10 @@ _[TAG_OPEN_STATE] = function tagOpenState(cp) {
         this._reconsumeInState(TAG_NAME_STATE);
     }
 
-    else if (cp === $.QUESTION_MARK)
+    else if (cp === $.QUESTION_MARK) {
+        this._createCommentToken();
         this._reconsumeInState(BOGUS_COMMENT_STATE);
+    }
 
     else {
         this._emitChar('<');
@@ -751,8 +752,10 @@ _[END_TAG_OPEN_STATE] = function endTagOpenState(cp) {
         this._emitChar('/');
     }
 
-    else
+    else {
+        this._createCommentToken();
         this._reconsumeInState(BOGUS_COMMENT_STATE);
+    }
 };
 
 
@@ -1554,38 +1557,22 @@ _[SELF_CLOSING_START_TAG_STATE] = function selfClosingStartTagState(cp) {
 
 //12.2.4.44 Bogus comment state
 //------------------------------------------------------------------
-_[BOGUS_COMMENT_STATE] = function bogusCommentState() {
-    this._createCommentToken();
-    this._reconsumeInState(BOGUS_COMMENT_STATE_CONTINUATION);
-};
-
-//HACK: to support streaming and make BOGUS_COMMENT_STATE reentrant we've
-//introduced BOGUS_COMMENT_STATE_CONTINUATION state which will not produce
-//comment token on each call.
-_[BOGUS_COMMENT_STATE_CONTINUATION] = function bogusCommentStateContinuation(cp) {
-    while (true) {
-        if (cp === $.GREATER_THAN_SIGN) {
-            this.state = DATA_STATE;
-            break;
-        }
-
-        else if (cp === $.EOF) {
-            this._reconsumeInState(DATA_STATE);
-            break;
-        }
-
-        else {
-            this.currentToken.data += cp === $.NULL ? UNICODE.REPLACEMENT_CHARACTER : toChar(cp);
-
-            this._hibernationSnapshot();
-            cp = this._consume();
+_[BOGUS_COMMENT_STATE] = function bogusCommentState(cp) {
+    if (cp === $.GREATER_THAN_SIGN) {
+        this.state = DATA_STATE;
+        this._emitCurrentToken();
+    }
 
-            if (this._ensureHibernation())
-                return;
-        }
+    else if (cp === $.EOF) {
+        this._reconsumeInState(DATA_STATE);
+        this._emitCurrentToken();
     }
 
-    this._emitCurrentToken();
+    else if (cp === $.NULL)
+        this.currentToken.data += UNICODE.REPLACEMENT_CHARACTER;
+
+    else
+        this.currentToken.data += toChar(cp);
 };
 
 //12.2.4.45 Markup declaration open state
@@ -1609,8 +1596,10 @@ _[MARKUP_DECLARATION_OPEN_STATE] = function markupDeclarationOpenState(cp) {
         else if (cdataMatch)
             this.state = CDATA_SECTION_STATE;
 
-        else
+        else {
+            this._createCommentToken();
             this._reconsumeInState(BOGUS_COMMENT_STATE);
+        }
     }
 };