Add Unicode support, fix #1

google · Oct 11, 2015 · fef77da · fef77da
1 parent 78c8771
commit fef77da
Show file tree

Hide file tree

Showing 11 changed files with 382 additions and 213 deletions.
diff --git a/core/ast.h b/core/ast.h
@@ -25,6 +25,7 @@ limitations under the License.
 #include <vector>
 
 #include "core/lexer.h"
+#include "core/string.h"
 
 enum ASTType {
     AST_APPLY,
@@ -51,18 +52,17 @@ enum ASTType {
     AST_VAR
 };
 
-
 /** Represents a variable / parameter / field name. */
 struct Identifier {
-    std::string name;
-    Identifier(const std::string &name)
+    String name;
+    Identifier(const String &name)
       : name(name)
     { }
 };
 
 static inline std::ostream &operator<<(std::ostream &o, const Identifier *id)
 {
-    o << id->name;
+    o << encode_utf8(id->name);
     return o;
 }
 
@@ -213,16 +213,16 @@ struct Function : public AST {
 
 /** Represents import "file". */
 struct Import : public AST {
-    std::string file;
-    Import(const LocationRange &lr, const std::string &file)
+    String file;
+    Import(const LocationRange &lr, const String &file)
       : AST(lr, AST_IMPORT), file(file)
     { }
 };
 
 /** Represents importstr "file". */
 struct Importstr : public AST {
-    std::string file;
-    Importstr(const LocationRange &lr, const std::string &file)
+    String file;
+    Importstr(const LocationRange &lr, const String &file)
       : AST(lr, AST_IMPORTSTR), file(file)
     { }
 };
@@ -271,8 +271,8 @@ struct LiteralNumber : public AST {
 
 /** Represents JSON strings. */
 struct LiteralString : public AST {
-    std::string value;
-    LiteralString(const LocationRange &lr, const std::string &value)
+    String value;
+    LiteralString(const LocationRange &lr, const String &value)
       : AST(lr, AST_LITERAL_STRING), value(value)
     { }
 };
@@ -373,7 +373,7 @@ struct Var : public AST {
 /** Allocates ASTs on demand, frees them in its destructor.
  */
 class Allocator {
-    std::map<std::string, const Identifier*> internedIdentifiers;
+    std::map<String, const Identifier*> internedIdentifiers;
     std::vector<AST*> allocated;
     public:
     template <class T, class... Args> T* make(Args&&... args)
@@ -386,7 +386,7 @@ class Allocator {
      *
      * The location used in the Identifier AST is that of the first one parsed.
      */
-    const Identifier *makeIdentifier(const std::string &name)
+    const Identifier *makeIdentifier(const String &name)
     {
         auto it = internedIdentifiers.find(name);
         if (it != internedIdentifiers.end()) {

diff --git a/core/lexer.cpp b/core/lexer.cpp
@@ -19,8 +19,9 @@ limitations under the License.
 #include <string>
 #include <sstream>
 
-#include "core/static_error.h"
 #include "core/lexer.h"
+#include "core/static_error.h"
+#include "core/string.h"
 
 static bool is_upper(char c)
 {
@@ -411,23 +412,8 @@ std::list<Token> jsonnet_lex(const std::string &filename, const char *input)
                                     codepoint += digit;
                                 }
 
-                                // Encode in UTF-8.
-                                if (codepoint < 0x0080) {
-                                    data += codepoint;
-                                } else {
-                                    auto msg = "Codepoint out of ascii range.";
-                                    throw StaticError(filename, begin, msg);
-                                }
-/*
-                                } else if (codepoint < 0x0800) {
-                                    data += 0xC0 | (codepoint >> 6);
-                                    data += 0x80 | (codepoint & 0x3F);
-                                } else {
-                                    data += 0xE0 | (codepoint >> 12);
-                                    data += 0x80 | ((codepoint >> 6) & 0x3F);
-                                    data += 0x80 | (codepoint & 0x3F);
-                                }
-*/
+                                encode_utf8(codepoint, data);
+
                                 // Leave us on the last char, ready for the ++c at
                                 // the outer for loop.
                                 c += 3;

diff --git a/core/lexer.h b/core/lexer.h
@@ -23,6 +23,7 @@ limitations under the License.
 #include <list>
 #include <sstream>
 
+#include "core/string.h"
 #include "core/static_error.h"
 
 struct Token {
@@ -71,6 +72,8 @@ struct Token {
 
     std::string data;
 
+    String data32(void) { return decode_utf8(data); }
+
     LocationRange location;
 
     Token(Kind kind, const std::string &data, const LocationRange &location)