Merge pull request #33 from doppioandante/preprocessor-fix-2

Fix preprocessor expansion strategy

src/Tokenizer.zig

@@ -102,16 +102,14 @@ pub const Token = struct {
 
         /// Special token to speed up preprocessing, `loc.end` will be an index to the param list.
         macro_param,
+        /// Special token to signal that the argument must be replaced without expansion (e.g. in concatenation)
+        macro_param_no_expand,
         /// Special token to speed up preprocessing, `loc.end` will be an index to the param list.
         stringify_param,
         /// Same as stringify_param, but for var args
         stringify_va_args,
         /// Special token for when empty argument is passed to macro token.
         empty_arg,
-        /// Special token used to prevent ## passed as arguments from being concatenated
-        hash_hash_from_param,
-        /// Special token used to expand arguments before other tokens.
-        identifier_from_param,
 
         keyword_auto,
         keyword_break,
@@ -293,7 +291,6 @@ pub const Token = struct {
             return switch (id) {
                 .invalid,
                 .identifier,
-                .identifier_from_param,
                 .string_literal,
                 .string_literal_utf_16,
                 .string_literal_utf_8,
@@ -321,6 +318,7 @@ pub const Token = struct {
                 .nl,
                 .eof,
                 .macro_param,
+                .macro_param_no_expand,
                 .stringify_param,
                 .stringify_va_args,
                 .empty_arg,
@@ -373,7 +371,7 @@ pub const Token = struct {
                 .angle_bracket_angle_bracket_right_equal => ">>=",
                 .tilde => "~",
                 .hash => "#",
-                .hash_hash, .hash_hash_from_param => "##",
+                .hash_hash => "##",
 
                 .keyword_auto => "auto",
                 .keyword_break => "break",

test/cases/macro expansion exhaustion.c

@@ -0,0 +1,6 @@
+#define CREATE_CALL F
+#define F(x) (x + 2)
+
+CREATE_CALL(2)
+
+#define EXPECTED_TOKENS (2 + 2)

test/cases/macro re-expansion.c

@@ -0,0 +1,37 @@
+#define EXPECTED_TOKENS z[0] z[0] z[0] z[0] z[0] h(s) 1 13 z[0] z[0] Z1 1 TAU(2) 1 TAU(1) 1 z[0]
+#define f(a) a
+#define z z[0]
+
+#define h(x) f(x(1))
+#define s(x) z
+
+#define K h(s)
+
+#define H h
+
+#define L H
+
+#define SS (s)
+
+#define INCOMPLETE f(1
+#define g(a, b) a##b
+#define INCOMPLETE2 g(1
+
+#define Z1 z Z2
+#define Z2 z Z1
+
+#define THETA(x) x f
+#define TAU(x) x TAU
+#define OMEGA(x) TAU(x)(1) THETA(x)
+
+f(f(z))
+h(s)
+K
+H(s)
+L(s)
+H SS
+INCOMPLETE)
+INCOMPLETE2,3)
+Z1
+TAU(1)(2)
+OMEGA(1)(z)

test/cases/macro token pasting order.c

@@ -0,0 +1,11 @@
+#define CAT(a, ...) a ## __VA_ARGS__
+#define M(val) val
+#define TEST(c) CAT(TEST_, c)
+#define abc 1
+
+TEST(M(0))
+CAT(TEST_, M(0))
+CAT(ab, c)
+CAT(a b, c)
+
+#define EXPECTED_TOKENS TEST_0 TEST_M(0) 1 a bc

test/cases/nested macro call.c

@@ -0,0 +1,7 @@
+#define EXPECTED_TOKENS x x
+
+#define A(x) x
+#define B(x) A(x)
+
+B(B(x))
+A(B(x))

test/cases/placeholder tokens pasting.c

@@ -0,0 +1,6 @@
+#define CAT(a, b) a##b
+
+x CAT(,)x
+CAT(,y)
+
+#define EXPECTED_TOKENS x x y

test/cases/recursive func macro.c

@@ -0,0 +1,6 @@
+#define EXPECTED_TOKENS F(1)
+
+#define F(x) G(x)
+#define G(x) F(x)
+
+F(1)

test/cases/standard-concatenation-strings-example.c

@@ -0,0 +1,37 @@
+// this test doesn't currently work due to the fact that the preprocessor
+// completely removes the spaces between tokens
+// see the commented lines
+
+#define str(s) # s
+#define xstr(s) str(s)
+#define debug(s, t) printf("x" # s "= %d, x" # t "= %s", \
+                    x ## s, x ## t)
+#define INCFILE(n) vers ## n
+#define glue(a, b) a ## b
+#define xglue(a, b) glue(a, b)
+#define HIGHLOW "hello"
+#define LOW LOW ", world"
+debug(1, 2);
+//fputs(str(strncmp("abc\0d", "abc", '\4') // this goes away
+//    == 0) str(: @\n), s);
+xstr(INCFILE(2).h)
+//xstr(INCFILE(2) . h)
+str(INCFILE(2).h)
+//str(INCFILE(2) . h)
+glue(HIGH, LOW);
+xglue(HIGH, LOW)
+
+//#define EXPECTED_TOKENS printf("x" "1" "= %d, x" "2" "= %s", x1, x2); \
+//                        fputs("strncmp(\"abc\\0d\", \"abc\", '\\4') == 0" \
+//                        ": @\n", s); \
+//                        "vers2.h" \
+//                        "vers2 . h" \
+//                        "INCFILE(2).h" \
+//                        "INCFILE(2) . h" \
+//                        "hello"; \
+//                        "hello" ", world"
+#define EXPECTED_TOKENS printf("x" "1" "= %d, x" "2" "= %s", x1, x2); \
+                        "vers2.h" \
+                        "INCFILE(2).h" \
+                        "hello"; \
+                        "hello" ", world"

test/cases/standard-placeholder-example.c

@@ -0,0 +1,6 @@
+// example from the C18 standard draft, 6.10.3.5, example 5
+#define t(x,y,z) x ## y ## z
+int j[] = { t(1,2,3), t(,4,5), t(6,,7), t(8,9,),
+           t(10,,), t(,11,), t(,,12), t(,,) };
+
+#define EXPECTED_TOKENS int j[] = { 123, 45, 67, 89, 10, 11, 12, };

test/cases/standard-redefinition-reexamination-example.c

@@ -0,0 +1,27 @@
+// example from the C18 standard draft, 6.10.3.5, example 3
+#define x 3
+#define f(a) f(x * (a)) 
+#undef x
+#define x 2
+#define g f
+#define z z[0]
+#define h g(\~{ }
+#define m(a) a(w)
+#define w 0,1
+#define t(a) a
+#define p() int
+#define q(x) x
+#define r(x,y) x ## y
+#define str(x) # x
+f(y+1) + f(f(z)) % t(t(g)(0) + t)(1);
+g(x+(3,4)-w) | h 5) & m
+     (f)^m(m);
+p() i[q()] = { q(1), r(2,3), r(4,), r(,5), r(,) };
+char c[2][6] = { str(hello), str() };
+
+#define EXPECTED_TOKENS f(2 * (y+1)) + f(2 * (f(2 * (z[0])))) % f(2 * (0)) + t(1); \
+                        f(2 * (2 +(3,4)-0,1)) | f(2 * (\~{ } 5)) & f(2 * (0,1)) \
+                        ^m(0,1); \
+                        int i[] = { 1, 23, 4, 5, }; \
+                        char c[2][6] = { "hello", "" };
+

test/cases/unmatched macro paren.c

@@ -0,0 +1,10 @@
+#define f(x) x
+
+f(1
+
+#define EXPECTED_ERRORS "unmatched macro paren.c:3:1: error: unterminated function macro argument list" \
+    "unmatched macro paren.c:3:1: warning: type specifier missing, defaults to 'int'" \
+    "unmatched macro paren.c:3:3: error: expected parameter declaration" \
+    "unmatched macro paren.c:3:3: error: expected ')', found 'an integer literal'" \
+    "unmatched macro paren.c:3:2: note: to match this '('" \
+

test/cases/unspecified expansion.c

@@ -0,0 +1,8 @@
+// This can either expand as 2*f(9) or as 2*9*g (see 6.10.3.4 in the standard)
+// Currently arocc does the former, but gcc and clang do the latter
+#define EXPECTED_TOKENS 2*f(9)
+
+#define f(a) a*g
+#define g(a) f(a)
+
+f(2)(9)

test/cases/var args macro functions.c

@@ -1,4 +1,4 @@
-#define EXPECTED_TOKENS "2 , 3 , 4 , 5 , 6" baz bar 3 , 4
+#define EXPECTED_TOKENS "2,3,4,5,6" baz bar 3 , 4
 
 
 #define foo(a,...) #__VA_ARGS__

test/cases/whitespace macro arguments.c

@@ -0,0 +1,17 @@
+#define EXPECTED_TOKENS 1 1 q 1 1 2 3 3
+
+#define h() 1
+#define g(x) x
+#define f(x, y) g(y) x
+
+#define p(...) h(__VA_ARGS__)
+#define q(e, ...) f(e, __VA_ARGS__)
+
+h()
+g()
+f(1,)
+f(1,q)
+f(,)
+p()
+q(3, 2)
+q(, 3)