Implement token counting for C parser

rmosolgo · Apr 22, 2024 · 8acc708 · 8acc708
1 parent c634445
commit 8acc708
Show file tree

Hide file tree

Showing 7 changed files with 147 additions and 104 deletions.
diff --git a/graphql-c_parser/ext/graphql_c_parser_ext/graphql_c_parser_ext.c b/graphql-c_parser/ext/graphql_c_parser_ext/graphql_c_parser_ext.c
@@ -1,7 +1,7 @@
 #include "graphql_c_parser_ext.h"
 
-VALUE GraphQL_CParser_Lexer_tokenize_with_c_internal(VALUE self, VALUE query_string, VALUE fstring_identifiers, VALUE reject_numbers_followed_by_names) {
-  return tokenize(query_string, RTEST(fstring_identifiers), RTEST(reject_numbers_followed_by_names));
+VALUE GraphQL_CParser_Lexer_tokenize_with_c_internal(VALUE self, VALUE query_string, VALUE fstring_identifiers, VALUE reject_numbers_followed_by_names, VALUE max_tokens) {
+  return tokenize(query_string, RTEST(fstring_identifiers), RTEST(reject_numbers_followed_by_names), FIX2INT(max_tokens));
 }
 
 VALUE GraphQL_CParser_Parser_c_parse(VALUE self) {
@@ -13,7 +13,7 @@ void Init_graphql_c_parser_ext() {
   VALUE GraphQL = rb_define_module("GraphQL");
   VALUE CParser = rb_define_module_under(GraphQL, "CParser");
   VALUE Lexer = rb_define_module_under(CParser, "Lexer");
-  rb_define_singleton_method(Lexer, "tokenize_with_c_internal", GraphQL_CParser_Lexer_tokenize_with_c_internal, 3);
+  rb_define_singleton_method(Lexer, "tokenize_with_c_internal", GraphQL_CParser_Lexer_tokenize_with_c_internal, 4);
   setup_static_token_variables();
 
   VALUE Parser = rb_define_class_under(CParser, "Parser", rb_cObject);