Mark start and end of included .re files with line directives.

This fixes bug #386 "#line directives around !include contents"
reported by Cyan Ogilvie.

It is easy to generate line directive at the beginning of the include
file, but much more difficult to find the end of it: file contents are
just a stream of bytes in the input buffer, and the continuation of the
parent file follows immediately after the end of the inlude file.

Therefore we use a hack and "unread" the include directive in the parent
file before filling the buffer with the contents of the include file (as
if the include directive appeared after the contens of include file).
This makes the lexer read the directive for the second time; at this
point we know it's the same include directive because the include file
is already on top of the file stack. So we ignore the directive and
simply generate line info.

The current implementation has a problem with nested includes that have
identical relative file paths. This will be fixed in a follow-up patch
by looking at the resolved paths when comparing files.

bootstrap/src/parse/lex.h

@@ -1,4 +1,4 @@
-/* Generated by re2c 2.1.1 on Fri Jul 16 08:52:38 2021 */
+/* Generated by re2c 2.2 on Sat Nov 27 11:30:21 2021 */
 
 #ifndef _RE2C_PARSE_LEX_
 #define _RE2C_PARSE_LEX_

src/parse/lex.re

@@ -231,7 +231,8 @@ loop:
     "/*!include:re2c" space+ @x dstring @y / ws_or_eoc {
         out.wraw(tok, ptr);
         if (!lex_block_end(out)) return INPUT_ERROR;
-        include(getstr(x + 1, y - 1));
+        include(getstr(x + 1, y - 1), ptr);
+        out.wdelay_stmt(0, code_line_info_input(alc, cur_loc()));
         goto next;
     }
     "/*!include:re2c" {
@@ -504,7 +505,7 @@ scan:
     }
 
     "!include" space+ @x dstring @y space* ";" / ws_or_eoc {
-        include(getstr(x + 1, y - 1));
+        include(getstr(x + 1, y - 1), tok);
         goto scan;
     }
     "!include" {

src/parse/scanner.cc

@@ -48,29 +48,40 @@ bool Scanner::open(const std::string &filename, const std::string *parent)
     return true;
 }
 
-bool Scanner::include(const std::string &filename)
+bool Scanner::include(const std::string &filename, char *at)
 {
+    // This function is called twice for each included file: first time when opening the
+    // file, and second time when it has been fully read and can be closed. Second time
+    // is needed to generate a line directive marking the end of the included file and the
+    // continuation of the parent file. To get the second call, we "unread" include
+    // directive on the first call (essentially just don't move token pointer to cursor)
+    // and let the lexer scan it twice. To differentiate the first and the second times,
+    // we compare the topmost file on stack with the inlude file (after popping all
+    // finished files, as there may be nested includes). This logic can't handle recursive
+    // self-includes, but they would be erroneous anyway.
+    pop_finished_files();
+    if (files.back()->name == filename) return true;
+
     // get name of the current file (before unreading)
     const size_t fidx = get_input_index();
     DASSERT(fidx < files.size());
     const std::string &parent = files[fidx]->escaped_name;
 
-    // unread buffer tail: we'll return to it later
-    // In the buffer nested files go before outer files. In the file stack,
-    // however, outer files go before nested files (nested are at the top).
-    // We want to break from the unreading cycle early, therefore we go in
-    // reverse order of file offsets in buffer and break as soon as the end
-    // offset is less than cursor (current position).
+    // Unread buffer tail: we'll return to it later. In the buffer nested files go before
+    // outer files. In the file stack, however, outer files go before nested files (nested
+    // are at the top). We want to break from the unreading cycle early, therefore we go
+    // in reverse order of file offsets in buffer and break as soon as the end offset is
+    // less than cursor (current position). `at` points at the start of include directive.
     for (size_t i = 0; i < files.size(); ++i) {
         Input *in = files[i];
-        if (in->so >= cur) {
+        if (in->so >= at) {
             // unread whole fragment
             fseek(in->file, in->so - in->eo, SEEK_CUR);
             in->so = in->eo = ENDPOS;
         }
-        else if (in->eo >= cur) {
+        else if (in->eo >= at) {
             // fragment on the boundary, unread partially
-            fseek(in->file, cur - in->eo, SEEK_CUR);
+            fseek(in->file, at - in->eo, SEEK_CUR);
             in->eo = cur - 1;
         }
         else {

src/parse/scanner.h

@@ -50,7 +50,7 @@ class Scanner: private ScannerState
     Scanner(const conopt_t *o, Msg &m);
     ~Scanner();
     bool open(const std::string &filename, const std::string *parent);
-    bool include(const std::string &filename);
+    bool include(const std::string &filename, char *at);
     bool gen_dep_file() const;
     const loc_t &tok_loc() const;
     loc_t cur_loc() const;

test/include/include001.c

@@ -1,16 +1,20 @@
 /* Generated by re2c */
 #line 1 "include/include001.re"
 // re2c $INPUT -o $OUTPUT --depfile $DEPFILE
+#line 1 "include/include001.re.b.inc"
+#line 1 "include/include001.re.c.inc"
 #line 3 "include/include001.re.c.inc"
 
+#line 1 "include/include001.re.b.inc"
 
 
 #line 5 "include/include001.re.b.inc"
 
+#line 2 "include/include001.re"
 
 
 
-#line 14 "include/include001.c"
+#line 18 "include/include001.c"
 {
 	YYCTYPE yych;
 	if (YYLIMIT <= YYCURSOR) YYFILL(1);
@@ -25,12 +29,12 @@
 	++YYCURSOR;
 #line 5 "include/include001.re"
 	{}
-#line 29 "include/include001.c"
+#line 33 "include/include001.c"
 yy5:
 	++YYCURSOR;
 #line 6 "include/include001.re"
 	{}
-#line 34 "include/include001.c"
+#line 38 "include/include001.c"
 }
 #line 7 "include/include001.re"
 

test/include/include002.c

@@ -1,13 +1,16 @@
 /* Generated by re2c */
 #line 1 "include/include002.re"
 // re2c $INPUT -o $OUTPUT --depfile $DEPFILE
+#line 1 "include/nested/include002.re.b.inc"
 
+#line 1 "include/nested/nested/include002.re.bd.inc"
 #line 3 "include/nested/nested/include002.re.bd.inc"
 
+#line 2 "include/nested/include002.re.b.inc"
 
 
 
-#line 11 "include/include002.c"
+#line 14 "include/include002.c"
 {
 	YYCTYPE yych;
 	if (YYLIMIT <= YYCURSOR) YYFILL(1);
@@ -21,22 +24,24 @@
 	++YYCURSOR;
 #line 7 "include/nested/include002.re.b.inc"
 	{}
-#line 25 "include/include002.c"
+#line 28 "include/include002.c"
 yy4:
 	++YYCURSOR;
 #line 6 "include/nested/include002.re.b.inc"
 	{}
-#line 30 "include/include002.c"
+#line 33 "include/include002.c"
 }
 #line 8 "include/nested/include002.re.b.inc"
 
 
+#line 1 "include/nested/nested/include002.re.be.inc"
 #line 3 "include/nested/nested/include002.re.be.inc"
 
+#line 10 "include/nested/include002.re.b.inc"
 
 
 
-#line 40 "include/include002.c"
+#line 45 "include/include002.c"
 {
 	YYCTYPE yych;
 	if (YYLIMIT <= YYCURSOR) YYFILL(1);
@@ -50,19 +55,20 @@
 	++YYCURSOR;
 #line 15 "include/nested/include002.re.b.inc"
 	{}
-#line 54 "include/include002.c"
+#line 59 "include/include002.c"
 yy10:
 	++YYCURSOR;
 #line 14 "include/nested/include002.re.b.inc"
 	{}
-#line 59 "include/include002.c"
+#line 64 "include/include002.c"
 }
 #line 16 "include/nested/include002.re.b.inc"
 
+#line 2 "include/include002.re"
 
 
 
-#line 66 "include/include002.c"
+#line 72 "include/include002.c"
 {
 	YYCTYPE yych;
 	if (YYLIMIT <= YYCURSOR) YYFILL(1);
@@ -77,29 +83,32 @@
 	++YYCURSOR;
 #line 7 "include/include002.re"
 	{}
-#line 81 "include/include002.c"
+#line 87 "include/include002.c"
 yy16:
 	++YYCURSOR;
 #line 5 "include/include002.re"
 	{}
-#line 86 "include/include002.c"
+#line 92 "include/include002.c"
 yy18:
 	++YYCURSOR;
 #line 6 "include/include002.re"
 	{}
-#line 91 "include/include002.c"
+#line 97 "include/include002.c"
 }
 #line 8 "include/include002.re"
 
 
+#line 1 "include/nested/include002.re.c.inc"
 
 
+#line 1 "include/nested/nested/include002.re.cd.inc"
 #line 3 "include/nested/nested/include002.re.cd.inc"
 
+#line 3 "include/nested/include002.re.c.inc"
 
 
 
-#line 103 "include/include002.c"
+#line 112 "include/include002.c"
 {
 	YYCTYPE yych;
 	if (YYLIMIT <= YYCURSOR) YYFILL(1);
@@ -113,22 +122,24 @@
 	++YYCURSOR;
 #line 8 "include/nested/include002.re.c.inc"
 	{}
-#line 117 "include/include002.c"
+#line 126 "include/include002.c"
 yy24:
 	++YYCURSOR;
 #line 7 "include/nested/include002.re.c.inc"
 	{}
-#line 122 "include/include002.c"
+#line 131 "include/include002.c"
 }
 #line 9 "include/nested/include002.re.c.inc"
 
 
+#line 1 "include/nested/nested/include002.re.ce.inc"
 #line 3 "include/nested/nested/include002.re.ce.inc"
 
+#line 11 "include/nested/include002.re.c.inc"
 
 
 
-#line 132 "include/include002.c"
+#line 143 "include/include002.c"
 {
 	YYCTYPE yych;
 	if (YYLIMIT <= YYCURSOR) YYFILL(1);
@@ -142,19 +153,20 @@
 	++YYCURSOR;
 #line 16 "include/nested/include002.re.c.inc"
 	{}
-#line 146 "include/include002.c"
+#line 157 "include/include002.c"
 yy30:
 	++YYCURSOR;
 #line 15 "include/nested/include002.re.c.inc"
 	{}
-#line 151 "include/include002.c"
+#line 162 "include/include002.c"
 }
 #line 17 "include/nested/include002.re.c.inc"
 
+#line 10 "include/include002.re"
 
 
 
-#line 158 "include/include002.c"
+#line 170 "include/include002.c"
 {
 	YYCTYPE yych;
 	if (YYLIMIT <= YYCURSOR) YYFILL(1);
@@ -169,17 +181,17 @@
 	++YYCURSOR;
 #line 15 "include/include002.re"
 	{}
-#line 173 "include/include002.c"
+#line 185 "include/include002.c"
 yy36:
 	++YYCURSOR;
 #line 13 "include/include002.re"
 	{}
-#line 178 "include/include002.c"
+#line 190 "include/include002.c"
 yy38:
 	++YYCURSOR;
 #line 14 "include/include002.re"
 	{}
-#line 183 "include/include002.c"
+#line 195 "include/include002.c"
 }
 #line 16 "include/include002.re"
 

test/include/include003.c

@@ -1,15 +1,19 @@
 /* Generated by re2c */
 #line 1 "include/include003.re"
 // re2c $INPUT -o $OUTPUT --depfile $DEPFILE
+#line 1 "include/include003.re.b.inc"
 #line 3 "include/include003.re.b.inc"
 
+#line 2 "include/include003.re"
 
+#line 1 "include/include003.re.c.inc"
 #line 3 "include/include003.re.c.inc"
 
+#line 3 "include/include003.re"
 
 
 
-#line 13 "include/include003.c"
+#line 17 "include/include003.c"
 {
 	YYCTYPE yych;
 	if (YYLIMIT <= YYCURSOR) YYFILL(1);
@@ -23,17 +27,17 @@
 	++YYCURSOR;
 #line 8 "include/include003.re"
 	{}
-#line 27 "include/include003.c"
+#line 31 "include/include003.c"
 yy4:
 	++YYCURSOR;
 #line 6 "include/include003.re"
 	{}
-#line 32 "include/include003.c"
+#line 36 "include/include003.c"
 yy6:
 	++YYCURSOR;
 #line 7 "include/include003.re"
 	{}
-#line 37 "include/include003.c"
+#line 41 "include/include003.c"
 }
 #line 9 "include/include003.re"