Provide a flag for the integer base

Author: kddnewton

config.yml

@@ -333,6 +333,16 @@ flags:
         comment: "&. operator"
       - name: VARIABLE_CALL
         comment: "a call that could have been a local variable"
+  - name: IntegerBaseFlags
+    values:
+      - name: BINARY
+        comment: "0b prefix"
+      - name: OCTAL
+        comment: "0o or 0 prefix"
+      - name: DECIMAL
+        comment: "0d or no prefix"
+      - name: HEXADECIMAL
+        comment: "0x prefix"
   - name: LoopFlags
     values:
       - name: BEGIN_MODIFIER
@@ -1483,6 +1493,10 @@ nodes:
           @foo = 1
           ^^^^^^^^
   - name: IntegerNode
+    fields:
+      - name: flags
+        type: flags
+        kind: IntegerBaseFlags
     comment: |
       Represents an integer number literal.
 

include/yarp/parser.h

@@ -384,6 +384,12 @@ struct yp_parser {
 
     // This is the list of newline offsets in the source file.
     yp_newline_list_t newline_list;
+
+    // We want to add a flag to integer nodes that indicates their base. We only
+    // want to parse these once, but we don't have space on the token itself to
+    // communicate this information. So we store it here and pass it through
+    // when we find tokens that we need it for.
+    yp_node_flags_t integer_base;
 };
 
 #endif // YARP_PARSER_H

src/yarp.c

@@ -2673,16 +2673,22 @@ yp_else_node_end_keyword_loc_set(yp_else_node_t *node, const yp_token_t *keyword
 
 // Allocate and initialize a new IntegerNode node.
 static yp_integer_node_t *
-yp_integer_node_create(yp_parser_t *parser, const yp_token_t *token) {
+yp_integer_node_create(yp_parser_t *parser, yp_node_flags_t base, const yp_token_t *token) {
     assert(token->type == YP_TOKEN_INTEGER);
     yp_integer_node_t *node = YP_ALLOC_NODE(parser, yp_integer_node_t);
-    *node = (yp_integer_node_t) {{ .type = YP_INTEGER_NODE, .location = YP_LOCATION_TOKEN_VALUE(token) }};
+
+    *node = (yp_integer_node_t) {{
+        .type = YP_INTEGER_NODE,
+        .flags = base,
+        .location = YP_LOCATION_TOKEN_VALUE(token)
+    }};
+
     return node;
 }
 
 // Allocate and initialize a new IntegerNode node from an INTEGER_IMAGINARY token.
 static yp_imaginary_node_t *
-yp_integer_node_imaginary_create(yp_parser_t *parser, const yp_token_t *token) {
+yp_integer_node_imaginary_create(yp_parser_t *parser, yp_node_flags_t base, const yp_token_t *token) {
     assert(token->type == YP_TOKEN_INTEGER_IMAGINARY);
 
     yp_imaginary_node_t *node = YP_ALLOC_NODE(parser, yp_imaginary_node_t);
@@ -2691,7 +2697,7 @@ yp_integer_node_imaginary_create(yp_parser_t *parser, const yp_token_t *token) {
             .type = YP_IMAGINARY_NODE,
             .location = YP_LOCATION_TOKEN_VALUE(token)
         },
-        .numeric = (yp_node_t *) yp_integer_node_create(parser, &((yp_token_t) {
+        .numeric = (yp_node_t *) yp_integer_node_create(parser, base, &((yp_token_t) {
             .type = YP_TOKEN_INTEGER,
             .start = token->start,
             .end = token->end - 1
@@ -2703,7 +2709,7 @@ yp_integer_node_imaginary_create(yp_parser_t *parser, const yp_token_t *token) {
 
 // Allocate and initialize a new IntegerNode node from an INTEGER_RATIONAL token.
 static yp_rational_node_t *
-yp_integer_node_rational_create(yp_parser_t *parser, const yp_token_t *token) {
+yp_integer_node_rational_create(yp_parser_t *parser, yp_node_flags_t base, const yp_token_t *token) {
     assert(token->type == YP_TOKEN_INTEGER_RATIONAL);
 
     yp_rational_node_t *node = YP_ALLOC_NODE(parser, yp_rational_node_t);
@@ -2712,7 +2718,7 @@ yp_integer_node_rational_create(yp_parser_t *parser, const yp_token_t *token) {
             .type = YP_RATIONAL_NODE,
             .location = YP_LOCATION_TOKEN_VALUE(token)
         },
-        .numeric = (yp_node_t *) yp_integer_node_create(parser, &((yp_token_t) {
+        .numeric = (yp_node_t *) yp_integer_node_create(parser, base, &((yp_token_t) {
             .type = YP_TOKEN_INTEGER,
             .start = token->start,
             .end = token->end - 1
@@ -2724,7 +2730,7 @@ yp_integer_node_rational_create(yp_parser_t *parser, const yp_token_t *token) {
 
 // Allocate and initialize a new IntegerNode node from an INTEGER_RATIONAL_IMAGINARY token.
 static yp_imaginary_node_t *
-yp_integer_node_rational_imaginary_create(yp_parser_t *parser, const yp_token_t *token) {
+yp_integer_node_rational_imaginary_create(yp_parser_t *parser, yp_node_flags_t base, const yp_token_t *token) {
     assert(token->type == YP_TOKEN_INTEGER_RATIONAL_IMAGINARY);
 
     yp_imaginary_node_t *node = YP_ALLOC_NODE(parser, yp_imaginary_node_t);
@@ -2733,7 +2739,7 @@ yp_integer_node_rational_imaginary_create(yp_parser_t *parser, const yp_token_t
             .type = YP_IMAGINARY_NODE,
             .location = YP_LOCATION_TOKEN_VALUE(token)
         },
-        .numeric = (yp_node_t *) yp_integer_node_rational_create(parser, &((yp_token_t) {
+        .numeric = (yp_node_t *) yp_integer_node_rational_create(parser, base, &((yp_token_t) {
             .type = YP_TOKEN_INTEGER_RATIONAL,
             .start = token->start,
             .end = token->end - 1
@@ -5193,6 +5199,7 @@ lex_numeric_prefix(yp_parser_t *parser) {
                     yp_diagnostic_list_append(&parser->error_list, parser->current.start, parser->current.end, YP_ERR_INVALID_NUMBER_BINARY);
                 }
 
+                parser->integer_base = YP_INTEGER_BASE_FLAGS_BINARY;
                 break;
 
             // 0o1111 is an octal number
@@ -5205,6 +5212,7 @@ lex_numeric_prefix(yp_parser_t *parser) {
                     yp_diagnostic_list_append(&parser->error_list, parser->current.start, parser->current.end, YP_ERR_INVALID_NUMBER_OCTAL);
                 }
 
+                parser->integer_base = YP_INTEGER_BASE_FLAGS_OCTAL;
                 break;
 
             // 01111 is an octal number
@@ -5218,6 +5226,7 @@ lex_numeric_prefix(yp_parser_t *parser) {
             case '6':
             case '7':
                 parser->current.end += yp_strspn_octal_number(parser->current.end, parser->end - parser->current.end);
+                parser->integer_base = YP_INTEGER_BASE_FLAGS_OCTAL;
                 break;
 
             // 0x1111 is a hexadecimal number
@@ -5230,6 +5239,7 @@ lex_numeric_prefix(yp_parser_t *parser) {
                     yp_diagnostic_list_append(&parser->error_list, parser->current.start, parser->current.end, YP_ERR_INVALID_NUMBER_HEXADECIMAL);
                 }
 
+                parser->integer_base = YP_INTEGER_BASE_FLAGS_HEXADECIMAL;
                 break;
 
             // 0.xxx is a float
@@ -5266,6 +5276,7 @@ lex_numeric_prefix(yp_parser_t *parser) {
 static yp_token_type_t
 lex_numeric(yp_parser_t *parser) {
     yp_token_type_t type = YP_TOKEN_INTEGER;
+    parser->integer_base = YP_INTEGER_BASE_FLAGS_DECIMAL;
 
     if (parser->current.end < parser->end) {
         type = lex_numeric_prefix(parser);
@@ -11272,18 +11283,26 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) {
 
             return node;
         }
-        case YP_TOKEN_INTEGER:
+        case YP_TOKEN_INTEGER: {
+            yp_node_flags_t base = parser->integer_base;
             parser_lex(parser);
-            return (yp_node_t *) yp_integer_node_create(parser, &parser->previous);
-        case YP_TOKEN_INTEGER_IMAGINARY:
+            return (yp_node_t *) yp_integer_node_create(parser, base, &parser->previous);
+        }
+        case YP_TOKEN_INTEGER_IMAGINARY: {
+            yp_node_flags_t base = parser->integer_base;
             parser_lex(parser);
-            return (yp_node_t *) yp_integer_node_imaginary_create(parser, &parser->previous);
-        case YP_TOKEN_INTEGER_RATIONAL:
+            return (yp_node_t *) yp_integer_node_imaginary_create(parser, base, &parser->previous);
+        }
+        case YP_TOKEN_INTEGER_RATIONAL: {
+            yp_node_flags_t base = parser->integer_base;
             parser_lex(parser);
-            return (yp_node_t *) yp_integer_node_rational_create(parser, &parser->previous);
-        case YP_TOKEN_INTEGER_RATIONAL_IMAGINARY:
+            return (yp_node_t *) yp_integer_node_rational_create(parser, base, &parser->previous);
+        }
+        case YP_TOKEN_INTEGER_RATIONAL_IMAGINARY: {
+            yp_node_flags_t base = parser->integer_base;
             parser_lex(parser);
-            return (yp_node_t *) yp_integer_node_rational_imaginary_create(parser, &parser->previous);
+            return (yp_node_t *) yp_integer_node_rational_imaginary_create(parser, base, &parser->previous);
+        }
         case YP_TOKEN_KEYWORD___ENCODING__:
             parser_lex(parser);
             return (yp_node_t *) yp_source_encoding_node_create(parser, &parser->previous);
@@ -13816,7 +13835,8 @@ yp_parser_init(yp_parser_t *parser, const uint8_t *source, size_t size, const ch
         .in_keyword_arg = false,
         .filepath_string = filepath_string,
         .constant_pool = YP_CONSTANT_POOL_EMPTY,
-        .newline_list = YP_NEWLINE_LIST_EMPTY
+        .newline_list = YP_NEWLINE_LIST_EMPTY,
+        .integer_base = 0
     };
 
     yp_accepts_block_stack_push(parser, true);

templates/ext/yarp/api_node.c.erb

@@ -135,7 +135,7 @@ yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding) {
                 <%- nodes.each do |node| -%>
 #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
                 case <%= node.type %>: {
-                    <%- if node.fields.any? { |field| ![YARP::NodeField, YARP::OptionalNodeField].include?(field.class) } -%>
+                    <%- if node.fields.any? { |field| ![YARP::NodeField, YARP::OptionalNodeField, YARP::FlagsField].include?(field.class) } -%>
                     yp_<%= node.human %>_t *cast = (yp_<%= node.human %>_t *) node;
                     <%- end -%>
                     VALUE argv[<%= node.fields.length + 1 %>];

test/yarp/errors_test.rb

@@ -889,8 +889,8 @@ def test_returning_to_optional_parameters_multiple_times
         ParametersNode(
           [RequiredParameterNode(:a)],
           [
-            OptionalParameterNode(:b, Location(), Location(), IntegerNode()),
-            OptionalParameterNode(:d, Location(), Location(), IntegerNode())
+            OptionalParameterNode(:b, Location(), Location(), IntegerNode(IntegerBaseFlags::DECIMAL)),
+            OptionalParameterNode(:d, Location(), Location(), IntegerNode(IntegerBaseFlags::DECIMAL))
           ],
           [RequiredParameterNode(:c), RequiredParameterNode(:e)],
           nil,
@@ -933,7 +933,7 @@ def test_setter_method_cannot_be_defined_in_an_endless_method_definition
         Location(),
         nil,
         nil,
-        StatementsNode([IntegerNode()]),
+        StatementsNode([IntegerNode(IntegerBaseFlags::DECIMAL)]),
         [],
         Location(),
         nil,
@@ -1123,7 +1123,7 @@ def test_duplicated_parameter_names
         :foo,
         Location(),
         nil,
-        ParametersNode([], [OptionalParameterNode(:a, Location(), Location(), IntegerNode())], [RequiredParameterNode(:b)], RestParameterNode(:c, Location(), Location()), [], nil, nil),
+        ParametersNode([], [OptionalParameterNode(:a, Location(), Location(), IntegerNode(IntegerBaseFlags::DECIMAL))], [RequiredParameterNode(:b)], RestParameterNode(:c, Location(), Location()), [], nil, nil),
         nil,
         [:a, :b, :c],
         Location(),