Total rewrite of the code generation subsystem.

Previously the output program was represented as a list of blocks (one
block per each re2c comment in the source file). Each block consisted
of a list of heterogenous fragments. Some fragments contained text that
was generated immediately during the first pass of code generation;
other fragments were placeholders for parts of the program whose
generation was delayed (such as dispatch on YYFILL labels with -f option
that cannot be generated on the first pass because the full list of
labels is known only at the end of the first pass). Individual fragments
used ad-hoc methods to render the generated code, which made rendering
rather complex (for example, it was problematic to decide if the 'if'
statement should be rendered on a single line or on multiple lines).

This commit adds a new AST-like representation of the output program.
The top-level program is still a list of output blocks, but each block
is now an AST. This decouples code generation from rendering, which
simplifies both phases and allows easier customization for different
output language backends (such as C or Go).

Author: skvadrik

bootstrap/src/parse/lex.cc

@@ -1,4 +1,4 @@
-/* Generated by re2c 1.1.1 on Fri Aug  2 10:49:54 2019 */
+/* Generated by re2c 1.3 on Mon Apr 13 20:34:13 2020 */
 
 #ifndef _RE2C_PARSE_LEX_
 #define _RE2C_PARSE_LEX_

bootstrap/src/parse/lex.h

@@ -52,6 +52,10 @@ DFA::DFA
     , rules(dfa.rules)
     , tags(dfa.tags)
     , mtagvers(dfa.mtagvers)
+    , stagnames()
+    , stagvars()
+    , mtagnames()
+    , mtagvars()
     , finvers(dfa.finvers)
     , tcpool(dfa.tcpool)
     , max_fill (0)
@@ -66,6 +70,9 @@ DFA::DFA
     , setup(su)
     , eof_action(eof)
     , msg(msg)
+    , used_labels()
+    , start_label(label_t::first())
+    , initial_label(label_t::first())
 {
     const size_t nstates = dfa.states.size();
     const size_t nchars = dfa.nchars;

src/adfa/adfa.cc

@@ -81,6 +81,10 @@ struct DFA
     std::valarray<Rule> &rules;
     std::vector<Tag> &tags;
     std::set<tagver_t> &mtagvers;
+    std::set<std::string> stagnames;
+    std::set<std::string> stagvars;
+    std::set<std::string> mtagnames;
+    std::set<std::string> mtagvars;
     const tagver_t *finvers;
     tcpool_t &tcpool;
     size_t max_fill;
@@ -96,6 +100,10 @@ struct DFA
     const Code *eof_action;
     Msg &msg;
 
+    std::set<label_t> used_labels;
+    label_t start_label;
+    label_t initial_label;
+
     DFA ( const dfa_t &dfa
         , const std::vector<size_t> &fill
         , size_t def
@@ -111,18 +119,17 @@ struct DFA
     ~DFA ();
     void reorder();
     void prepare(const opt_t *opts);
-    void calc_stats(const opt_t *opts);
-    void emit (Output &, uint32_t &, bool, bool &);
+    void calc_stats(OutputBlock &out);
+    void count_used_labels(const opt_t *opts);
+    void emit_body(Output &output, uint32_t ind, CodeStmts *program) const;
+    void emit_dot(Output &output, CodeStmts *program) const;
 
 private:
     void addState(State*, State *);
     void split (State *);
     void findBaseState(const opt_t *opts);
     void hoist_tags(const opt_t *opts);
     void hoist_tags_and_skip(const opt_t *opts);
-    void count_used_labels(std::set<label_t> &used, label_t start, label_t initial, const opt_t *opts) const;
-    void emit_body (Output &, uint32_t &, const std::set<label_t> & used_labels, label_t initial) const;
-    void emit_dot(Output &o, bool last_cond) const;
 
     FORBID_COPY (DFA);
 };

src/adfa/adfa.h

@@ -9,6 +9,7 @@
 #include "src/adfa/adfa.h"
 #include "src/codegen/bitmap.h"
 #include "src/codegen/go.h"
+#include "src/codegen/emit.h"
 #include "src/debug/debug.h"
 #include "src/dfa/tcmd.h"
 #include "src/msg/msg.h"
@@ -242,14 +243,14 @@ void DFA::prepare(const opt_t *opts)
     }
 }
 
-void DFA::calc_stats(const opt_t *opts)
+void DFA::calc_stats(OutputBlock &out)
 {
+    const opt_t *opts = out.opts;
+
     // calculate 'YYMAXFILL'
     max_fill = 0;
-    for (State * s = head; s; s = s->next)
-    {
-        if (max_fill < s->fill)
-        {
+    for (State * s = head; s; s = s->next) {
+        if (max_fill < s->fill) {
             max_fill = s->fill;
         }
     }
@@ -284,6 +285,33 @@ void DFA::calc_stats(const opt_t *opts)
             "multiple context markers, use '-t, --tags' "
             "option and '/*!stags:re2c ... */' directive");
     }
+
+    if (!oldstyle_ctxmarker) {
+        for (size_t i = 0; i < tags.size(); ++i) {
+            const Tag &tag = tags[i];
+            if (history(tag)) {
+                mtagvars.insert(*tag.name);
+            }
+            else if (tag.name) {
+                stagvars.insert(*tag.name);
+            }
+        }
+        for (tagver_t v = 1; v <= maxtagver; ++v) {
+            const std::string s = vartag_name(v, opts->tags_prefix);
+            if (mtagvers.find(v) != mtagvers.end()) {
+                mtagnames.insert(s);
+            }
+            else {
+                stagnames.insert(s);
+            }
+        }
+        out.stags.insert(stagnames.begin(), stagnames.end());
+        out.mtags.insert(mtagnames.begin(), mtagnames.end());
+    }
+
+    if (!cond.empty()) {
+        out.types.push_back(cond);
+    }
 }
 
 static bool can_hoist_tags(const State *s, const opt_t *opts)

src/adfa/prepare.cc

@@ -47,18 +47,26 @@ const bitmap_t *bitmaps_t::find(const Go *go, const State *s) const
 
 bool bitmaps_t::empty() const { return maps.empty(); }
 
-void bitmaps_t::gen(Output &o, uint32_t ind)
+CodeStmts *bitmaps_t::gen(Output &output)
 {
-    if (empty() || !used) return;
+    if (empty() || !used) return NULL;
+
+    const opt_t *opts = output.block().opts;
+    code_alc_t &alc = output.allocator;
+    Scratchbuf &o = output.scratchbuf;
+
+    CodeStmts *stmts = code_stmts(alc);
 
-    const opt_t *opts = o.block().opts;
     const uint32_t nmap = static_cast<uint32_t>(maps.size());
     riter_t b = maps.rbegin(), e = maps.rend();
 
-    o.wind(ind).ws("static const unsigned char ")
-        .wstring(opts->yybm).ws("[] = {");
+    o.cstr("static const unsigned char ").str(opts->yybm).cstr("[] = {");
+    append_stmt(stmts, code_stmt_text(alc, o.flush()));
+
+    CodeStmts *block = code_stmts(alc);
+    static const uint32_t TABLE_WIDTH = 8;
 
-    for (uint32_t i = 0, t = 1; b != e; i += ncunit, t += 8) {
+    for (uint32_t i = 0, t = 1; b != e; i += ncunit, t += TABLE_WIDTH) {
         memset(buffer, 0, ncunit * sizeof(uint32_t));
 
         for (uint32_t m = 0x80; b != e && m; m >>= 1, ++b) {
@@ -67,25 +75,33 @@ void bitmaps_t::gen(Output &o, uint32_t ind)
             doGen(b->go, b->on, buffer, 0, m);
         }
 
-        if (nmap > 8) {
-            o.ws("\n").wind(ind + 1).ws("/* table ").wu32(t).ws(" .. ")
-                .wu32(std::min(nmap, t + 7)).ws(": ").wu32(i).ws(" */");
+        if (nmap > TABLE_WIDTH) {
+            o.cstr("/* table ").u32(t).cstr(" .. ").u32(std::min(nmap, t + 7))
+                .cstr(": ").u32(i).cstr(" */");
+            append_stmt(block, code_stmt_text(alc, o.flush()));
         }
 
-        for (uint32_t c = 0; c < ncunit; ++c) {
-            if (c % 8 == 0) {
-                o.ws("\n").wind(ind + 1);
+        for (uint32_t i = 0; i < ncunit / TABLE_WIDTH; ++i) {
+            for (uint32_t j = 0; j < TABLE_WIDTH; ++j) {
+                const uint32_t c = buffer[i * TABLE_WIDTH + j];
+                if (opts->yybmHexTable) {
+                    o.u32_hex(c, opts);
+                }
+                else {
+                    o.u32_width(c, 3);
+                }
+                o.cstr(", ");
             }
-            if (opts->yybmHexTable) {
-                o.wu32_hex(buffer[c]);
-            } else {
-                o.wu32_width(buffer[c], 3);
-            }
-            o.ws(", ");
+            append_stmt(block, code_stmt_text(alc, o.flush()));
         }
     }
 
-    o.ws("\n").wind(ind).ws("};\n");
+    append_stmt(stmts, code_block(alc, block, CodeBlock::INDENTED));
+
+    o.cstr("};");
+    append_stmt(stmts, code_stmt_text(alc, o.flush()));
+
+    return stmts;
 }
 
 void doGen(const Go *g, const State *s, uint32_t *bm, uint32_t f, uint32_t m)

src/codegen/bitmap.cc

@@ -13,6 +13,7 @@ namespace re2c {
 class Output;
 struct Go;
 struct State;
+struct CodeStmts;
 
 struct bitmap_t
 {
@@ -40,7 +41,7 @@ class bitmaps_t
     void insert(const Go *go, const State *s);
     const bitmap_t *find(const Go *go, const State *s) const;
     bool empty() const;
-    void gen(Output &o, uint32_t ind);
+    CodeStmts *gen(Output &output);
     FORBID_COPY(bitmaps_t);
 };
 

src/codegen/bitmap.h

@@ -1,21 +1,33 @@
 #ifndef _RE2C_CODE_EMIT_
 #define _RE2C_CODE_EMIT_
 
+#include <vector>
+
 #include "src/codegen/output.h"
 #include "src/adfa/adfa.h"
+#include "src/util/smart_ptr.h"
 
 
 namespace re2c {
 
-typedef std::vector<std::string> code_lines_t;
+typedef std::vector<smart_ptr<DFA> > dfas_t;
+
+void gen_code(Output &output, dfas_t &dfas);
+
+void emit_action(Output &output, uint32_t ind, const DFA &dfa, const State *s,
+    CodeStmts *stmts);
+
+void gen_settags(Output &output, CodeStmts *tag_actions, const DFA &dfa, tcid_t tcid,
+    bool delayed);
+
+void gen_goto(Output &output, CodeStmts *stmts, const State *from, const State *to,
+    const DFA &dfa, tcid_t tcid, bool skip, bool eof);
 
-void emit_action(Output &o, uint32_t ind, const DFA &dfa, const State *s, const std::set<label_t> &used_labels);
-void gen_goto_plain(Output &o, uint32_t ind, const State *from, const State *to, const DFA &dfa, tcid_t tcid, bool skip, bool eof);
-void gen_goto_case(Output &o, uint32_t ind, const State *from, const State *to, const DFA &dfa, tcid_t tcid, bool skip, bool eof);
-void gen_goto_if(Output &o, uint32_t ind, const State *from, const State *to, const DFA &dfa, tcid_t tcid, bool skip, bool eof);
-void gen_settags(code_lines_t &code, const DFA &dfa, tcid_t tcid, const opt_t *opts, bool delayed);
 std::string vartag_name(tagver_t ver, const std::string &prefix);
-std::string vartag_expr(tagver_t ver, const std::string &prefix, const std::string &expression);
+
+std::string vartag_expr(tagver_t ver, const std::string &prefix,
+    const std::string &expression);
+
 std::string tag_expr(const Tag &tag, bool lvalue);
 
 inline std::string indent(uint32_t n, const std::string &s)