Bam! Robust tokenizing!

src/document.cpp

@@ -2,6 +2,8 @@
 #include "document.hpp"
 
 namespace Sass {
+  using namespace Prelexer;
+
   Document::Document(char* _path, char* _source) {
     path = _path;
     if (!_source) {
@@ -28,4 +30,6 @@ namespace Sass {
   Document::~Document() {
     delete [] source;
   }
+
+
 }

src/document.hpp

@@ -2,15 +2,32 @@
 
 namespace Sass {
   using std::vector;
+  using namespace Prelexer;
 
   struct Document {
     char* path;
     char* source;
-    unsigned int position;
+    char* position;
     unsigned int line_number;
     vector<Node> statements;
+    Token top;
 
     Document(char* _path, char* _source = 0);
     ~Document();
+
+    inline Token& peek() { return top; }
+    template <prelexer mx>
+    bool try_munching() {
+      char* after_whitespace = spaces_and_comments(position);
+      line_number += count_interval<'\n'>(position, after_whitespace);
+      char* after_token = mx(after_whitespace);
+      if (after_token) {
+        top = Token(mx, after_whitespace, after_token, line_number);
+        position = after_token;
+        return true;
+      }
+      else return false;
+    }
+
   };
 }

src/node.hpp

@@ -14,7 +14,9 @@ namespace Sass {
       simple_selector_sequence,
       simple_selector,
       property,
-      value
+      value,
+      lookahead_sequence,
+      lookahead_token
     };
 
     Node_Type type;

src/test.scss

@@ -0,0 +1,9 @@
+div {
+  color: red;
+  background: blue;
+  span {
+    font-weight: bold;
+    display: inline-block;
+  }
+  margin: 10px 5px;
+}

src/token.cpp

@@ -6,9 +6,11 @@ namespace Sass {
     end = 0;
     line_number = 1;
   }
-  Token::Token(const char* _begin,
+  Token::Token(Prelexer::prelexer _type,
+               const char* _begin,
                const char* _end,
                unsigned int _line_number) {
+    type = _type;
     begin = _begin;
     end = _end;
     line_number = _line_number;

src/token.hpp

@@ -1,10 +1,14 @@
+#include "prelexer.hpp"
+
 namespace Sass {
   struct Token {
+    Prelexer::prelexer type;
     const char* begin;
     const char* end;
     unsigned int line_number;
     Token();
-    Token(const char* _begin,
+    Token(Prelexer::prelexer _type,
+          const char* _begin,
           const char* _end,
           unsigned int _line_number);
     inline bool is_null() { return begin == 0 || end == 0; }

src/unit-test-document.cpp

@@ -3,11 +3,32 @@
 
 using namespace Sass;
 
+void print_slice(const char *s, const char *t) {
+  if (t) {
+    printf("succeeded with %ld characters:\t", t - s);
+    while (s < t) putchar(*s++);
+    putchar('\n');
+  }
+  else {
+    printf("failed\n");
+  }
+}
+
 int main(int argc, char* argv[]) {
-  Document doc(argv[1], 0);
-  char* src = doc.source;
-  printf("FILE BEGINS ON NEXT LINE\n");
-  while (*src) std::putchar(*(src++));
-  printf("<EOF>\n");
+  if (argc > 1) {
+    Document doc(argv[1], 0);
+    char* src = doc.source;
+    printf("FILE BEGINS ON NEXT LINE\n");
+    while (*src) std::putchar(*(src++));
+    printf("<EOF>\n");
+
+    doc.try_munching<Prelexer::identifier>();
+    print_slice(doc.top.begin, doc.top.end);
+    doc.try_munching<Prelexer::exactly<'{'> >();
+    print_slice(doc.top.begin, doc.top.end);
+    doc.try_munching<Prelexer::identifier>();
+    print_slice(doc.top.begin, doc.top.end);
+  }
+
   return 0;
 }