Fixed todos; Cleaned up; Improved tokenizer test

Author: Daniel Herzog

src/syntaxhighlight/http-header/tokenizer.js

@@ -4,13 +4,11 @@ window.cls = window.cls || {};
 
 cls.HTTPHeaderTokenizer = function()
 {
-  var CR = "\r";
   var LF = "\n";
   var PUNCTUATOR = ":";
-  var WHITESPACE_CHARS =
-  {
-    '\u0009': 1, //  Tab <TAB>
-    '\u0020': 1, //  Space <SP>
+  var WHITESPACE_CHARS = {
+    "\u0009": 1, //  Tab <TAB>
+    "\u0020": 1, //  Space <SP>
   };
 
   this._buffer = "";
@@ -24,46 +22,39 @@ cls.HTTPHeaderTokenizer = function()
     this._buffer = input_buffer;
     this._emitToken = ontoken;
     while (this._state_handler !== this._state_handlers.EOF)
-    {
       this._state_handler.apply(this);
-    }
 
     this._state_handlers.EOF.apply(this);
   };
 
-  this._state_handlers = 
+  this._state_handlers =
   {
     FIRST_LINE_PART: function()
     {
       if (this._is_EOF())
-      {
         return false;
-      }
+
       var c = this._buffer.charAt(this._current_pos++);
       this._token_type = cls.HTTPHeaderTokenizer.types.FIRST_LINE_PART;
       if (c in WHITESPACE_CHARS)
       {
         this._emitToken(this._token_type ,this._token_buffer);
         this._token_buffer = "";
-        // For now, LF and whitespace add to the next token. Visually that makes no difference.
       }
       else
       if (c === LF)
       {
         this._emitToken(this._token_type ,this._token_buffer);
         this._token_buffer = "";
-        this._emitToken(cls.HTTPHeaderTokenizer.types.LINE_SEPARATOR, c); // todo: don't emit your own token.
         this._state_handler = this._state_handlers.NAME;
-        return false;
       }
       this._token_buffer += c;
     },
     NAME: function()
     {
       if (this._is_EOF())
-      {
         return false;
-      }
+
       var c = this._buffer.charAt(this._current_pos++);
       this._token_type = cls.HTTPHeaderTokenizer.types.NAME;
       if (c === PUNCTUATOR)
@@ -79,9 +70,8 @@ cls.HTTPHeaderTokenizer = function()
     VALUE: function()
     {
       if (this._is_EOF())
-      {
         return false;
-      }
+
       var c = this._buffer.charAt(this._current_pos++);
       this._token_type = cls.HTTPHeaderTokenizer.types.VALUE;
       // LF only means switching to header when the following char is not whitespace.
@@ -90,7 +80,6 @@ cls.HTTPHeaderTokenizer = function()
         this._emitToken(this._token_type ,this._token_buffer);
         this._token_buffer = "";
         this._state_handler = this._state_handlers.NAME;
-        // For now, LF and whitespace add to the next token. Visually that makes no difference.
       }
       this._token_buffer += c;
     },
@@ -108,13 +97,12 @@ cls.HTTPHeaderTokenizer = function()
       return true;
     }
     return false;
-  }
+  };
 }
 
 cls.HTTPHeaderTokenizer.types = {
     FIRST_LINE_PART  : 1,
     NAME             : 2,
     VALUE            : 3,
-    PUNCTUATOR       : 4,
-    LINE_SEPARATOR   : 5
+    PUNCTUATOR       : 4
 };

test-scripts/network-view/header-tokenizer.html

@@ -31,7 +31,7 @@
 </style>
 
 <script src="../../src/scripts/dom.js"></script>
-<script src="../../src/syntaxhighlight/http/tokenizer.js"></script>
+<script src="../../src/syntaxhighlight/http-header/tokenizer.js"></script>
 
 <script>
 
@@ -92,11 +92,13 @@
 
 window.onload = function()
 {
-  document.body.firstElementChild.render(get_tokens(raw_headers).map(token_template));
-  document.body.firstElementChild.render([["br"], ["br"]]);
-  document.body.firstElementChild.render(get_tokens(raw_headers2).map(token_template));
-  document.body.firstElementChild.render([["br"], ["br"]]);
-  document.body.firstElementChild.render(get_tokens(raw_headers3).map(token_template));
+  var headers = [raw_headers, raw_headers2, raw_headers3];
+  headers.forEach(function(header_string){
+    var pre = document.body.firstElementChild.render(["pre"])
+    pre.render(get_tokens(header_string).map(token_template));
+    if (pre.textContent !== header_string)
+      throw(new Error("Tokenized headers produced different result."));
+  });
 }
 
 </script>