lexer.d: make unittest work

dlang · Oct 20, 2015 · c29d1aa · c29d1aa
1 parent 9f5c142
commit c29d1aa
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 37 deletions.
diff --git a/src/lexer.d b/src/lexer.d
@@ -67,24 +67,21 @@ static this()
     }
 }
 
-version (unittest)
+unittest
 {
-    void unittest_lexer()
-    {
-        //printf("unittest_lexer()\n");
-        /* Not much here, just trying things out.
-         */
-        const(char)* text = "int";
-        scope Lexer lex1 = new Lexer(null, cast(char*)text, 0, text.sizeof, 0, 0);
-        TOK tok;
-        tok = lex1.nextToken();
-        //printf("tok == %s, %d, %d\n", Token::toChars(tok), tok, TOKint32);
-        assert(tok == TOKint32);
-        tok = lex1.nextToken();
-        assert(tok == TOKeof);
-        tok = lex1.nextToken();
-        assert(tok == TOKeof);
-    }
+    //printf("lexer.unittest\n");
+    /* Not much here, just trying things out.
+     */
+    string text = "int";
+    scope Lexer lex1 = new Lexer(null, text.ptr, 0, text.length, 0, 0);
+    TOK tok;
+    tok = lex1.nextToken();
+    //printf("tok == %s, %d, %d\n", Token::toChars(tok), tok, TOKint32);
+    assert(tok == TOKint32);
+    tok = lex1.nextToken();
+    assert(tok == TOKeof);
+    tok = lex1.nextToken();
+    assert(tok == TOKeof);
 }
 
 /***********************************************************
@@ -101,12 +98,22 @@ public:
     const(char)* p;         // current character
     const(char)* line;      // start of current line
     Token token;
-    int doDocComment;       // collect doc comment information
-    int anyToken;           // !=0 means seen at least one token
-    int commentToken;       // !=0 means comments are TOKcomment's
+    bool doDocComment;      // collect doc comment information
+    bool anyToken;          // seen at least one token
+    bool commentToken;      // comments are TOKcomment's
     bool errors;            // errors occurred during lexing or parsing
 
-    final extern (D) this(const(char)* filename, const(char)* base, size_t begoffset, size_t endoffset, int doDocComment, int commentToken)
+    /*********************
+     * Creat a Lexer.
+     * Params:
+     *  filename = used for error messages
+     *  base = source code, ending in a 0 byte
+     *  begoffset = starting offset into base[]
+     *  endoffset = last offset into base[]
+     *  doDocComment = handle documentation comments
+     *  commentToken = comments become TOKcomment's
+     */
+    this(const(char)* filename, const(char)* base, size_t begoffset, size_t endoffset, bool doDocComment, bool commentToken)
     {
         scanloc = Loc(filename, 1, 1);
         //printf("Lexer::Lexer(%p,%d)\n",base,length);
@@ -156,16 +163,6 @@ public:
         }
     }
 
-    final static void initLexer()
-    {
-        Identifier.initTable();
-        Token.initTokens();
-        version (unittest)
-        {
-            unittest_lexer();
-        }
-    }
-
     final TOK nextToken()
     {
         if (token.next)

diff --git a/src/mars.d b/src/mars.d
@@ -1191,7 +1191,6 @@ Language changes listed by -transition=id:
     VersionCondition.addPredefinedGlobalIdent("D_HardFloat");
     objc_tryMain_dObjc();
     // Initialization
-    Lexer.initLexer();
     Type._init();
     Id.initialize();
     Module._init();

diff --git a/src/parse.d b/src/parse.d
@@ -235,9 +235,9 @@ public:
      * Input:
      *      loc     location in source file of mixin
      */
-    extern (D) this(Loc loc, Module _module, const(char)* base, size_t length, int doDocComment)
+    extern (D) this(Loc loc, Module _module, const(char)* base, size_t length, bool doDocComment)
     {
-        super(_module ? _module.srcfile.toChars() : null, base, 0, length, doDocComment, 0);
+        super(_module ? _module.srcfile.toChars() : null, base, 0, length, doDocComment, false);
         //printf("Parser::Parser()\n");
         scanloc = loc;
         if (loc.filename)
@@ -254,9 +254,9 @@ public:
         //nextToken();              // start up the scanner
     }
 
-    extern (D) this(Module _module, const(char)* base, size_t length, int doDocComment)
+    extern (D) this(Module _module, const(char)* base, size_t length, bool doDocComment)
     {
-        super(_module ? _module.srcfile.toChars() : null, base, 0, length, doDocComment, 0);
+        super(_module ? _module.srcfile.toChars() : null, base, 0, length, doDocComment, false);
         //printf("Parser::Parser()\n");
         mod = _module;
         linkage = LINKd;

diff --git a/src/tokens.d b/src/tokens.d
@@ -576,8 +576,9 @@ extern (C++) struct Token
 
     static __gshared const(char)*[TOKMAX] tochars;
 
-    static void initTokens()
+    static this()
     {
+        Identifier.initTable();
         foreach (kw; keywords)
         {
             //printf("keyword[%d] = '%s'\n",u, keywords[u].name);