Merge pull request #26 from hroptatyr/feat/turtle-chunk-parser

Provide turtle chunk parser
dajobe · Dec 22, 2014 · bd3bfe4 · bd3bfe4
2 parents d33a385 + f79d197
commit bd3bfe4
Show file tree

Hide file tree

Showing 4 changed files with 238 additions and 60 deletions.
diff --git a/src/raptor_uri.c b/src/raptor_uri.c
@@ -134,7 +134,7 @@ raptor_new_uri_from_counted_string(raptor_world* world,
 #if defined(RAPTOR_DEBUG) && RAPTOR_DEBUG > 1
   RAPTOR_DEBUG1("Creating new URI '");
   fwrite(uri_string, sizeof(char), length, RAPTOR_DEBUG_FH);
-  fputs(RAPTOR_DEBUG_FH, "' in hash\n")
+  fputs("' in hash\n", RAPTOR_DEBUG_FH);
 #endif
 
   new_uri = RAPTOR_CALLOC(raptor_uri*, 1, sizeof(*new_uri));

diff --git a/src/turtle_common.h b/src/turtle_common.h
@@ -49,9 +49,6 @@ struct raptor_turtle_parser_s {
   /* buffer length */
   size_t buffer_length;
 
-  /* static statement for use in passing to user code */
-  raptor_statement statement;
-
   raptor_namespace_stack namespaces; /* static */
 
   /* for lexer to store result in */
@@ -63,6 +60,19 @@ struct raptor_turtle_parser_s {
   int scanner_set;
 
   int lineno;
+  int lineno_last_good;
+
+  /* for the chunk parser, how much of the input has been consumed */
+  size_t consumed;
+  /* likewise, how much of the input has been successfully processed */
+  size_t processed;
+  /* indicates what can be processed at most */
+  size_t consumable;
+  /* real end-of-buffer indicator, as we kill the last line */
+  size_t end_of_buffer;
+
+  /* a sequence holding deferred statements */
+  raptor_sequence *deferred;
 
   /* for creating long literals */
   raptor_stringbuffer* sb;
@@ -75,6 +85,9 @@ struct raptor_turtle_parser_s {
 
   /* Allow TRIG extensions */
   int trig;
+
+  /* Last run of many */
+  int is_end;
 };
 
 

diff --git a/src/turtle_lexer.l b/src/turtle_lexer.l
@@ -171,6 +171,9 @@ static char turtle_lexer_oom_text[]="turtle_lexer: Out of memory";
 /* Do not need input() to to read from stdin */
 #define YY_NO_INPUT 1
 
+#define YY_USER_ACTION				\
+  turtle_parser->consumed += yyleng;
+
 %}
 
 /* Tokens from Turtle 2013 spec - lex-ifyed to remove unicode ranges */
@@ -312,6 +315,11 @@ EXPONENT [eE][+-]?[0-9]+
                     BEGIN(INITIAL);
                     raptor_free_stringbuffer(turtle_parser->sb);
                     turtle_parser->sb = NULL;
+                    if(!turtle_parser->is_end) {
+                      /* next run will fix things, hopefully */
+                      return EOF;
+                    }
+                    /* otherwise abort */
                     turtle_syntax_error(rdf_parser, "End of file in middle of \"\"\"literal\"\"\"");
                     yyterminate();
 }
@@ -375,6 +383,11 @@ EXPONENT [eE][+-]?[0-9]+
                     BEGIN(INITIAL);
                     raptor_free_stringbuffer(turtle_parser->sb);
                     turtle_parser->sb = NULL;
+                    if(!turtle_parser->is_end) {
+                      /* next run will fix things, hopefully */
+                      return EOF;
+                    }
+                    /* otherwise abort */
                     turtle_syntax_error(rdf_parser, "End of file in middle of '''literal'''");
                     yyterminate();
 }