Allow _ separator within numeric literals

Closes #504.
ziglang · Sep 29, 2017 · 9652854 · 9652854
1 parent 3fe50cb
commit 9652854
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 6 deletions.
diff --git a/src/bigfloat.cpp b/src/bigfloat.cpp
@@ -61,21 +61,32 @@ void bigfloat_init_bigint(BigFloat *dest, const BigInt *op) {
     }
 }
 
-int bigfloat_init_buf_base10(BigFloat *dest, const uint8_t *buf_ptr, size_t buf_len) {
-    char *str_begin = (char *)buf_ptr;
-    char *str_end;
+int bigfloat_init_buf_base10(BigFloat *dest, const uint8_t *b_start, size_t b_len) {
+    // NOTE: We will not need to know about this when parsing is performed by tokenizer.
+    const char NUMERIC_LIT_SEP = '_';
+
+    Buf *filtered = buf_alloc();
+    for (size_t i = 0; i < b_len; ++i) {
+        if (b_start[i] != NUMERIC_LIT_SEP) {
+            buf_append_char(filtered, b_start[i]);
+        }
+    }
 
     errno = 0;
-    double value = strtod(str_begin, &str_end); // TODO actual f128 parsing
+    char *str_end;
+    double value = strtod(buf_ptr(filtered), &str_end); // TODO actual f128 parsing
+
     if (errno) {
+        buf_deinit(filtered);
         return ErrorOverflow;
     }
 
     float64_t value_f64;
     memcpy(&value_f64, &value, sizeof(double));
     f64_to_f128M(value_f64, &dest->value);
 
-    assert(str_end <= ((char*)buf_ptr) + buf_len);
+    assert(str_end <= (char*) buf_ptr(filtered) + buf_len(filtered));
+    buf_deinit(filtered);
     return 0;
 }
 

diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
@@ -101,6 +101,8 @@
     ALPHA: \
     case '_'
 
+#define NUMERIC_LIT_SEP '_'
+
 struct ZigKeyword {
     const char *text;
     TokenId token_id;
@@ -1186,6 +1188,10 @@ void tokenize(Buf *buf, Tokenization *out) {
                         set_token_id(&t, t.cur_tok, TokenIdFloatLiteral);
                         break;
                     }
+                    if (c == NUMERIC_LIT_SEP) {
+                        break;
+                    }
+
                     uint32_t digit_value = get_digit_value(c);
                     if (digit_value >= t.radix) {
                         if (is_symbol_char(c)) {
@@ -1230,6 +1236,10 @@ void tokenize(Buf *buf, Tokenization *out) {
                         t.state = TokenizeStateFloatExponentUnsigned;
                         break;
                     }
+                    if (c == NUMERIC_LIT_SEP) {
+                        break;
+                    }
+
                     uint32_t digit_value = get_digit_value(c);
                     if (digit_value >= t.radix) {
                         if (is_symbol_char(c)) {
@@ -1279,6 +1289,10 @@ void tokenize(Buf *buf, Tokenization *out) {
                 break;
             case TokenizeStateFloatExponentNumber:
                 {
+                    if (c == NUMERIC_LIT_SEP) {
+                        break;
+                    }
+
                     uint32_t digit_value = get_digit_value(c);
                     if (digit_value >= t.radix) {
                         if (is_symbol_char(c)) {

diff --git a/test/cases/math.zig b/test/cases/math.zig
@@ -136,7 +136,14 @@ test "const number literal" {
 }
 const ten = 10;
 
-
+test "number literal separators" {
+    assert(1 == 1_);
+    assert(1___________________2 == 12);
+    assert(0x1234_1234__1234_1234 == 0x1234123412341234);
+    assert(0x123_456_789___._123p1_023 == 0x123456789.123p1023);
+    assert(0b1001_0001 == 0b10010001);
+    assert(0o755644000== 0o755_644_000);
+}
 
 test "unsigned wrapping" {
     testUnsignedWrappingEval(@maxValue(u32));

diff --git a/test/compile_errors.zig b/test/compile_errors.zig
@@ -2153,4 +2153,9 @@ pub fn addCases(cases: &tests.CompileErrorContext) {
         \\}
     ,
         ".tmp_source.zig:14:17: error: use of undeclared identifier 'HeaderValue'");
+
+    cases.add("numeric literal separator before radix",
+        \\const a = 0_x10;
+    ,
+        ".tmp_source.zig:1:11: error: invalid character: 'x'");
 }