Merge pull request #97 from guineawheek/fix_cpp_int_literals

Allow int literals to be separated with single quotes

cxxheaderparser/lexer.py

@@ -225,18 +225,18 @@ class PlyLexer:
     #
 
     hex_prefix = "0[xX]"
-    hex_digits = "[0-9a-fA-F]+"
+    hex_digits = "[0-9a-fA-F']+"
     bin_prefix = "0[bB]"
-    bin_digits = "[01]+"
+    bin_digits = "[01']+"
 
     # integer constants (K&R2: A.2.5.1)
     integer_suffix_opt = (
         r"(([uU]ll)|([uU]LL)|(ll[uU]?)|(LL[uU]?)|([uU][lL])|([lL][uU]?)|[uU])?"
     )
     decimal_constant = (
-        "(0" + integer_suffix_opt + ")|([1-9][0-9]*" + integer_suffix_opt + ")"
+        "(0" + integer_suffix_opt + ")|([1-9][0-9']*" + integer_suffix_opt + ")"
     )
-    octal_constant = "0[0-7]*" + integer_suffix_opt
+    octal_constant = "0[0-7']*" + integer_suffix_opt
     hex_constant = hex_prefix + hex_digits + integer_suffix_opt
     bin_constant = bin_prefix + bin_digits + integer_suffix_opt
 

tests/test_numeric_literals.py

@@ -0,0 +1,60 @@
+# Note: testcases generated via `python -m cxxheaderparser.gentest`
+
+from cxxheaderparser.types import (
+    FundamentalSpecifier,
+    NameSpecifier,
+    PQName,
+    Token,
+    Type,
+    Value,
+    Variable,
+)
+from cxxheaderparser.simple import (
+    Include,
+    NamespaceScope,
+    Pragma,
+    parse_string,
+    ParsedData,
+)
+
+
+def test_numeric_literals() -> None:
+    content = """
+      #pragma once
+      #include <cstdint>
+
+      int test_binary = 0b01'10'01;
+      int test_decimal = 123'456'789u;
+      int test_octal = 012'42'11l;
+    """
+    data = parse_string(content, cleandoc=True)
+
+    assert data == ParsedData(
+        namespace=NamespaceScope(
+            variables=[
+                Variable(
+                    name=PQName(segments=[NameSpecifier(name="test_binary")]),
+                    type=Type(
+                        typename=PQName(segments=[FundamentalSpecifier(name="int")])
+                    ),
+                    value=Value(tokens=[Token(value="0b01'10'01")]),
+                ),
+                Variable(
+                    name=PQName(segments=[NameSpecifier(name="test_decimal")]),
+                    type=Type(
+                        typename=PQName(segments=[FundamentalSpecifier(name="int")])
+                    ),
+                    value=Value(tokens=[Token(value="123'456'789u")]),
+                ),
+                Variable(
+                    name=PQName(segments=[NameSpecifier(name="test_octal")]),
+                    type=Type(
+                        typename=PQName(segments=[FundamentalSpecifier(name="int")])
+                    ),
+                    value=Value(tokens=[Token(value="012'42'11l")]),
+                ),
+            ]
+        ),
+        pragmas=[Pragma(content=Value(tokens=[Token(value="once")]))],
+        includes=[Include(filename="<cstdint>")],
+    )