Operators can no-longer end in + - ~ ! or contain // /* |||

sbarzowski · Feb 9, 2016 · 27ddf2c · 27ddf2c
1 parent 8393550
commit 27ddf2c
Show file tree

Hide file tree

Showing 5 changed files with 79 additions and 22 deletions.
diff --git a/core/lexer.cpp b/core/lexer.cpp
@@ -430,7 +430,21 @@ Tokens jsonnet_lex(const std::string &filename, const char *input)
                     kind = Token::IDENTIFIER;
                 }
                 data = id;
-            } else if (is_symbol(*c) || *c == '#') {
+
+            // Single line # comment
+            } else if (*c == '#') {
+                const char *start = c + 1;
+                while (*c != '\0' && *c != '\n') {
+                    ++c;
+                }
+                // Do not include the # or \n in the comment.
+                fodder.emplace_back(FodderElement::COMMENT_HASH, std::string(start, c - start));
+                // Leaving it on the \n allows processing of \n on next iteration,
+                // I.e., managing of the line & column counter.
+                c--;  // Will be iterated again by the for loop.
+                continue;
+
+            } else if (is_symbol(*c)) {
 
                 // Single line C++ style comment
                 if (*c == '/' && *(c+1) == '/') {
@@ -446,20 +460,6 @@ Tokens jsonnet_lex(const std::string &filename, const char *input)
                     continue;
                 }
 
-                // Single line # comment
-                if (*c == '#') {
-                    const char *start = c + 1;
-                    while (*c != '\0' && *c != '\n') {
-                        ++c;
-                    }
-                    // Do not include the # or \n in the comment.
-                    fodder.emplace_back(FodderElement::COMMENT_HASH, std::string(start, c - start));
-                    // Leaving it on the \n allows processing of \n on next iteration,
-                    // I.e., managing of the line & column counter.
-                    c--;  // Will be iterated again by the for loop.
-                    continue;
-                }
-
                 // Multi-line comment.
                 if (*c == '/' && *(c+1) == '*') {
                     c += 2;  // Avoid matching /*/: skip the /* before starting the search for */.
@@ -483,6 +483,7 @@ Tokens jsonnet_lex(const std::string &filename, const char *input)
                     c++;
                     continue;
                 }
+
                 // Text block
                 if (*c == '|' && *(c+1) == '|' && *(c+2) == '|' && *(c+3) == '\n') {
                     std::stringstream block;
@@ -545,12 +546,22 @@ Tokens jsonnet_lex(const std::string &filename, const char *input)
                     break;  // Out of the switch.
                 }
 
-                for (; *c != '\0' ; ++c) {
-                    if (!is_symbol(*c)) {
-                        break;
-                    }
-                    data += *c;
+                const char *operator_begin = c;
+                for (; is_symbol(*c) ; ++c) {
+                    // Not allowed // in operators
+                    if (*c == '/' && *(c+1) == '/') break;
+                    // Not allowed /* in operators
+                    if (*c == '/' && *(c+1) == '*') break;
+                    // Not allowed ||| in operators
+                    if (*c == '|' && *(c+1) == '|' && *(c+2) == '|') break;
+                }
+                // Not allowed to end with a + - ~ ! unless a single char.
+                // So, wind it back if we need to (but not too far).
+                while (c > operator_begin + 1
+                       && (*(c-1) == '+' || *(c-1) == '-' || *(c-1) == '~' || *(c-1) == '!')) {
+                    c--;
                 }
+                data += std::string(operator_begin, c);
                 --c;
                 kind = data == "$" ? Token::DOLLAR : Token::OPERATOR;
             } else {

diff --git a/test_suite/arith_bool.jsonnet b/test_suite/arith_bool.jsonnet
@@ -16,7 +16,8 @@ limitations under the License.
 
 std.assertEqual(!false, true) &&
 std.assertEqual(!true, false) &&
-std.assertEqual(!(!true), true) &&
+std.assertEqual(!!true, true) &&
+std.assertEqual(!!false, false) &&
 
 std.assertEqual(false && false, false) &&
 std.assertEqual(false && true, false) &&

diff --git a/test_suite/comments.jsonnet b/test_suite/comments.jsonnet
@@ -18,4 +18,29 @@ limitations under the License.
 
 # this kind of comment
 
-/* ignore this */ true
+/* ignore this */ true &&
+
+// Test lexing:
+local x = 1-/*foo*/1;
+
+local x = 1-/*foo*/1;
+
+local x = 1-#foo
+1;
+
+local x = 1*/*foo*/1;
+
+local x = 1*#foo
+1;
+
+local x = 1-/*+foo*/1;
+
+local x = 1-#+foo
+1;
+
+local x = 1*/*+foo*/1;
+
+local x = 1*#+foo
+1;
+
+true
diff --git a/test_suite/text_block.jsonnet b/test_suite/text_block.jsonnet
@@ -91,6 +91,16 @@ local blank_line3 = |||
         foo
     |||;
 
+// Interaction with operators (1)
+local op1 = "foo"+|||
+    foo
+|||;
+
+// Interaction with operators (2)
+local op2 = "foo"<|||
+    foo
+|||;
+
 std.assertEqual(blank_line3, "\nfoo\n") &&
 
 

diff --git a/test_suite/text_block.jsonnet.unparse.golden b/test_suite/text_block.jsonnet.unparse.golden
@@ -91,6 +91,16 @@ local blank_line3 = |||
         foo
     |||;
 
+// Interaction with operators (1)
+local op1 = "foo"+|||
+    foo
+|||;
+
+// Interaction with operators (2)
+local op2 = "foo"<|||
+    foo
+|||;
+
 std.assertEqual(blank_line3, "\nfoo\n") &&