refactor(c): try for jcpp

Author: phodal

chapi-ast-c/build.gradle.kts

@@ -25,6 +25,8 @@ dependencies {
     // Kotlin reflection.
     testImplementation(kotlin("test"))
 
+    // https://mvnrepository.com/artifact/org.anarres/jcpp
+
     // JUnit 5
     testImplementation("org.junit.jupiter:junit-jupiter-api:5.6.0")
     testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.6.0")
@@ -35,6 +37,7 @@ dependencies {
 
     // coroutines
     testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.0-RC2")
+    testImplementation("org.anarres:jcpp:1.4.14")
 }
 
 sourceSets.main {

chapi-ast-c/src/main/antlr/CLexer.g4

@@ -12,8 +12,9 @@ DELIMITED_COMMENT:       '/*'  .*? '*/'           -> channel(COMMENTS_CHANNEL);
 WHITESPACES:   (Whitespace | Newline)+            -> channel(HIDDEN);
 SHARP:         '#'                                -> mode(DIRECTIVE_MODE);
 
-//MultiLineMacro: '#' (~[\n]*? '\\' '\r'? '\n')+ ~ [\n]+ -> channel (HIDDEN);
-//
+//Directive:      '#' ~ [\n]* -> channel (HIDDEN);
+MultiLineMacro: '#' (~[\n]*? '\\' '\r'? '\n')+ ~ [\n]+ -> channel (HIDDEN);
+
 Auto             : 'auto' ;
 Break            : 'break' ;
 Case             : 'case' ;
@@ -143,12 +144,6 @@ DigitSequence
     : Digit+
     ;
 
-IncludeText
-    : '<' SChar* ('.' | '/' | SChar)* '>'
-    | DIRECTIVE_STRING
-    | CONDITIONAL_SYMBOL
-    ;
-
 STRING
     : EncodingPrefix? '"' SCharSequence? '"'
     ;
@@ -239,7 +234,7 @@ DIRECTIVE_STRING:              '"' ~('"' | [\r\n\u0085\u2028\u2029])* '"' -> cha
 CONDITIONAL_SYMBOL:            Identifier                       -> channel(DIRECTIVE);
 DIRECTIVE_SINGLE_LINE_COMMENT: '//' ~[\r\n\u0085\u2028\u2029]*  -> channel(COMMENTS_CHANNEL), type(SINGLE_LINE_COMMENT);
 DIRECTIVE_NEW_LINE:            Newline                          -> channel(DIRECTIVE), mode(DEFAULT_MODE);
-INCLUDE_TEXT:                  IncludeText                      -> channel(DIRECTIVE), type(IncludeText);
+INCLUDE_TEXT:   ('<' SChar* ('.' | '/' | SChar)* '>' | DIRECTIVE_STRING | CONDITIONAL_SYMBOL)  -> channel(DIRECTIVE);
 
 mode DIRECTIVE_TEXT;
 

chapi-ast-c/src/main/antlr/CParser.g4

@@ -72,8 +72,8 @@ extensionExpression : Extenion? OPEN_PARENS typeName CLOSE_PARENS LeftBrace init
 postixCall
         :LeftBracket expression RightBracket                           #arrayAccessPostfixExpression
         // for macro support: ph_gen(, hpdata_age_heap, hpdata_t, age_link, hpdata_age_comp)
-        | OPEN_PARENS Comma? argumentExpressionList? CLOSE_PARENS                         #functionCallPostfixExpression
-        | (Dot | Arrow) Identifier                                      #memberAccessPostfixExpression
+        | OPEN_PARENS Comma? argumentExpressionList? CLOSE_PARENS      #functionCallPostfixExpression
+        | (Dot | Arrow) Identifier                                     #memberAccessPostfixExpression
         ;
 
 //macroPostixCall

chapi-ast-c/src/main/antlr/CPreprocessorParser.g4

@@ -8,7 +8,7 @@ options { tokenVocab=CLexer; superClass=chapi.ast.antlr.CPreprocessorParserBase;
 
 preprocessor_directive returns [Boolean value]
 	: DEFINE CONDITIONAL_SYMBOL (Identifier | DIGITS | preprocessor_directive)?  directive_new_line_or_sharp { this.OnPreprocessorDirectiveDefine(); }   #preprocessorDeclaration
-	| INCLUDE IncludeText directive_new_line_or_sharp { this.OnPreprocessorDirectiveInclude(); }        #preprocessorIncludeDeclaration
+	| INCLUDE INCLUDE_TEXT directive_new_line_or_sharp { this.OnPreprocessorDirectiveInclude(); }       #preprocessorIncludeDeclaration
 	| UNDEF CONDITIONAL_SYMBOL directive_new_line_or_sharp { this.OnPreprocessorDirectiveUndef(); }     #preprocessorDeclaration
 	| Ifdef CONDITIONAL_SYMBOL directive_new_line_or_sharp { this.OnPreprocessorDirectiveIfdef(); }     #preprocessorIfdefConditional
 	| Ifndef CONDITIONAL_SYMBOL directive_new_line_or_sharp { this.OnPreprocessorDirectiveIfndef(); }   #preprocessorIfndefConditional

chapi-ast-c/src/main/java/chapi/ast/antlr/CPreprocessorParserBase.java

@@ -29,7 +29,7 @@ protected Boolean AllConditions() {
     protected void OnPreprocessorDirectiveInclude() {
         ParserRuleContext c = this._ctx;
         CPreprocessorParser.PreprocessorIncludeDeclarationContext d = (CPreprocessorParser.PreprocessorIncludeDeclarationContext) c;
-        IncludeSymbols.add(d.IncludeText().getText());
+        IncludeSymbols.add(d.INCLUDE_TEXT().getText());
         d.value = AllConditions();
     }
 

chapi-ast-c/src/test/kotlin/chapi/ast/cast/CFullIdentListenerTest.kt

@@ -4,11 +4,12 @@ import kotlinx.coroutines.flow.asFlow
 import kotlinx.coroutines.flow.collect
 import kotlinx.coroutines.flow.mapNotNull
 import kotlinx.coroutines.runBlocking
+import org.anarres.cpp.*
 import org.junit.jupiter.api.Test
 import java.io.File
-import kotlin.test.Ignore
 import kotlin.test.assertEquals
 
+
 internal class CFullIdentListenerTest {
 
     @Test
@@ -338,7 +339,7 @@ typedef struct {
             """.trimIndent()
 
         val codeFile = CAnalyser().analysis(code, "helloworld.c")
-        assertEquals(codeFile.DataStructures.size, 2)
+        assertEquals(codeFile.DataStructures.size, 1)
         assertEquals(codeFile.DataStructures[0].Functions.size, 1)
     }
 
@@ -392,7 +393,7 @@ typedef struct {
             """.trimIndent()
 
         val codeFile = CAnalyser().analysis(code, "helloworld.c")
-        assertEquals(codeFile.DataStructures.size, 0)
+        assertEquals(codeFile.DataStructures.size, 1)
     }
 
     @Test
@@ -664,4 +665,58 @@ typedef struct {
         val codeFile = CAnalyser().analysis(code, "helloworld.c")
         assertEquals(codeFile.DataStructures.size, 0)
     }
+
+
+    @Test
+    fun shouldPreprocessorHandleMacroCall() {
+        val code = """
+            #ifndef HDR_TESTS_H
+            #define HDR_TESTS_H
+            
+            /* These are functions used in tests and are not intended for normal usage. */
+            
+            #include "hdr_histogram.h"
+            
+            #ifdef __cplusplus
+            extern "C" {
+            #endif
+            
+            int32_t counts_index_for(const struct hdr_histogram* h, int64_t value);
+            int hdr_encode_compressed(struct hdr_histogram* h, uint8_t** compressed_histogram, size_t* compressed_len);
+            int hdr_decode_compressed(uint8_t* buffer, size_t length, struct hdr_histogram** histogram);
+            void hdr_base64_decode_block(const char* input, uint8_t* output);
+            void hdr_base64_encode_block(const uint8_t* input, char* output);
+            
+            #ifdef __cplusplus
+            }
+            #endif
+            
+            #endif
+            """.trimIndent()
+
+        val pp = Preprocessor()
+        pp.addInput(StringLexerSource(code))
+        pp.addFeature(Feature.DIGRAPHS);
+        pp.addFeature(Feature.TRIGRAPHS);
+        pp.addFeature(Feature.LINEMARKERS);
+        pp.addWarning(Warning.IMPORT);
+
+        pp.listener = DefaultPreprocessorListener()
+
+        try {
+            while (true) {
+                val tok = pp.token()
+                println(tok.type)
+                if (tok.type == Token.EOF) break
+                print(tok.text)
+            }
+        } catch (e: Exception) {
+            val buf = StringBuilder("Preprocessor failed:\n")
+            println(e)
+        }
+
+
+    }
+
 }
+

chapi-ast-c/src/test/resources/grammar/add.c

@@ -1,12 +1,12 @@
-    int main()
-    {
-        int i, sum = 0;
-       
-        for ( i = 1; i <= LAST; i++ ) {
-          sum += i;
-        } /*-for-*/
-        printf("sum = %d\n", sum);
+int main()
+{
+    int i, sum = 0;
 
-        return 0;
-    }
+    for ( i = 1; i <= LAST; i++ ) {
+      sum += i;
+    } /*-for-*/
+    printf("sum = %d\n", sum);
+
+    return 0;
+}