Refactored all of the desugaring code to no longer depend on memory b…

…eing initialized to 0. Everything is about 12% faster now.

HACKING

@@ -2,14 +2,16 @@ Privileged arguments
 ====================
 
 As a matter of convenience, there are several identifiers that
-internal macros use. Chances are that if you use these names for other
-things, you're gonna have a bad time.
+internal anaphoric macros use. Chances are that if you use these names
+for other things, you're gonna have a bad time.
 
 In particular, these names, and the macros that use them, are:
 - state:
     Used by a_new and company. Should be an HParseState*
 - mm__:
     Used by h_new and h_free. Should be an HAllocator*
+- stk__:
+    Used in desugaring. Should be an HCFStack*
 
 Function suffixes
 =================

Makefile

@@ -24,17 +24,19 @@ examples/all: src/all
 examples/compile: src/compile
 
 define SUBDIR_TEMPLATE
-$(1)/%:
-	$$(MAKE) -C $(1) $$*
+$(1)/%: force
+	$(MAKE) -C $(1) $$*
 endef
 
+force:
+
 $(foreach dir,$(SUBDIRS),$(eval $(call SUBDIR_TEMPLATE,$(dir))))
 
 #.DEFAULT:
 #	$(if $(findstring ./,$(dir $@)),$(error No rule to make target `$@'),$(MAKE) -C $(dir $@) $(notdir $@))
 
-TAGS: $(shell find * -name "*.c")
-	etags $^
+TAGS: force
+	etags $(shell find * -name "*.c" -o -name "*.h")
 
 config:
 	@printf "%30s %s\n" $(foreach var,$(CONFIG_VARS),$(var) $($(var)) )

src/backends/contextfree.h

@@ -0,0 +1,147 @@
+// This is an internal header; it provides macros to make desugaring cleaner.
+#include <assert.h>
+#include "../internal.h"
+#ifndef HAMMER_CONTEXTFREE__H
+#define HAMMER_CONTEXTFREE__H
+
+
+// HCFStack
+struct HCFStack_ {
+  HCFChoice **stack;
+  int count;
+  int cap;
+  HCFChoice *last_completed; // Last completed choice.
+};
+
+#ifndef UNUSED
+#define UNUSED __attribute__((unused))
+#endif
+
+static inline HCFChoice* h_cfstack_new_choice_raw(HAllocator *mm__, HCFStack *stk__) UNUSED;
+static inline void h_cfstack_begin_choice(HAllocator *mm__, HCFStack *stk__) UNUSED;
+static HCFStack* h_cfstack_new(HAllocator *mm__) UNUSED; 
+static HCFStack* h_cfstack_new(HAllocator *mm__) {
+  HCFStack *stack = h_new(HCFStack, 1);
+  stack->count = 0;
+  stack->cap = 4;
+  stack->stack = h_new(HCFChoice*, stack->cap);
+  return stack;
+}
+
+static void h_cfstack_free(HAllocator *mm__, HCFStack *stk__) UNUSED; 
+static void h_cfstack_free(HAllocator *mm__, HCFStack *stk__) {
+  h_free(stk__->stack);
+  h_free(stk__);
+}
+
+static inline void h_cfstack_add_to_seq(HAllocator *mm__, HCFStack *stk__, HCFChoice *item) UNUSED;
+static inline void h_cfstack_add_to_seq(HAllocator *mm__, HCFStack *stk__, HCFChoice *item) {
+  HCFChoice *cur_top = stk__->stack[stk__->count-1];
+  assert(cur_top->type == HCF_CHOICE);
+  assert(cur_top->seq[0] != NULL); // There must be at least one sequence...
+  stk__->last_completed = item;
+  for (int i = 0;; i++) {
+    if (cur_top->seq[i+1] == NULL) {
+      assert(cur_top->seq[i]->items != NULL);
+      for (int j = 0;; j++) {
+	if (cur_top->seq[i]->items[j] == NULL) {
+	  cur_top->seq[i]->items = mm__->realloc(mm__, cur_top->seq[i]->items, sizeof(HCFChoice*) * (j+2));
+	  cur_top->seq[i]->items[j] = item;
+	  cur_top->seq[i]->items[j+1] = NULL;
+	  return;
+	}
+      }
+    }
+  }
+}
+
+static inline HCFChoice* h_cfstack_new_choice_raw(HAllocator *mm__, HCFStack *stk__) {
+  HCFChoice *ret = h_new(HCFChoice, 1);
+  ret->reshape = NULL;
+  ret->action = NULL;
+  ret->pred = NULL;
+  ret->type = ~0; // invalid type
+  // Add it to the current sequence...
+  if (stk__->count > 0) {
+    h_cfstack_add_to_seq(mm__, stk__, ret);
+  }
+
+  return ret;
+}
+
+static inline void h_cfstack_add_charset(HAllocator *mm__, HCFStack *stk__, HCharset charset) {
+  HCFChoice *ni = h_cfstack_new_choice_raw(mm__, stk__);
+  ni->type = HCF_CHARSET;
+  ni->charset = charset;
+  stk__->last_completed = ni;
+}
+
+
+static inline void h_cfstack_add_char(HAllocator *mm__, HCFStack *stk__, uint8_t chr) {
+  HCFChoice *ni = h_cfstack_new_choice_raw(mm__, stk__);
+  ni->type = HCF_CHAR;
+  ni->chr = chr;
+  stk__->last_completed = ni;
+}
+
+static inline void h_cfstack_add_end(HAllocator *mm__, HCFStack *stk__) {
+  HCFChoice *ni = h_cfstack_new_choice_raw(mm__, stk__);
+  ni->type = HCF_END;
+  stk__->last_completed = ni;
+}
+
+static inline void h_cfstack_begin_choice(HAllocator *mm__, HCFStack *stk__) {
+  HCFChoice *choice = h_cfstack_new_choice_raw(mm__, stk__);
+  choice->type = HCF_CHOICE;
+  choice->seq = h_new(HCFSequence*, 1);
+  choice->seq[0] = NULL;
+
+  if (stk__->count + 1 > stk__->cap) {
+    assert(stk__->cap > 0);
+    stk__->cap *= 2;
+    stk__->stack = mm__->realloc(mm__, stk__->stack, stk__->cap * sizeof(HCFChoice*));
+  }
+  assert(stk__->cap >= 1);
+  stk__->stack[stk__->count++] = choice;
+}
+
+static inline void h_cfstack_begin_seq(HAllocator *mm__, HCFStack *stk__) {
+  HCFChoice *top = stk__->stack[stk__->count-1];
+  for (int i = 0;; i++) {
+    if (top->seq[i] == NULL) {
+      top->seq = mm__->realloc(mm__, top->seq, sizeof(HCFSequence*) * (i+2));
+      HCFSequence *seq = top->seq[i] = h_new(HCFSequence, 1);
+      top->seq[i+1] = NULL;
+      seq->items = h_new(HCFChoice*, 1);
+      seq->items[0] = NULL;
+      return;
+    }
+  }
+}
+
+static inline void h_cfstack_end_seq(HAllocator *mm__, HCFStack *stk__) UNUSED;
+static inline void h_cfstack_end_seq(HAllocator *mm__, HCFStack *stk__) {
+  // do nothing. You should call this anyway.
+}
+
+static inline void h_cfstack_end_choice(HAllocator *mm__, HCFStack *stk__) UNUSED;
+static inline void h_cfstack_end_choice(HAllocator *mm__, HCFStack *stk__) {
+  assert(stk__->count > 0);
+  stk__->last_completed = stk__->stack[stk__->count-1];
+  stk__->count--;
+}
+
+#define HCFS_APPEND(choice) h_cfstack_add_to_seq(mm__, stk__, (choice))
+#define HCFS_DESUGAR(parser) h_desugar(mm__, stk__, parser)
+#define HCFS_ADD_CHARSET(charset) h_cfstack_add_charset(mm__, stk__, (charset))
+#define HCFS_ADD_CHAR(chr) h_cfstack_add_char(mm__, stk__, (chr))
+#define HCFS_ADD_END() h_cfstack_add_end(mm__, stk__)
+// The semicolons on BEGIN macros are intentional; pretend that they
+// are control structures.
+#define HCFS_BEGIN_CHOICE() h_cfstack_begin_choice(mm__, stk__);
+#define HCFS_BEGIN_SEQ() h_cfstack_begin_seq(mm__, stk__);
+#define HCFS_END_CHOICE() h_cfstack_end_choice(mm__, stk__)
+#define HCFS_END_SEQ() h_cfstack_end_seq(mm__, stk__)
+#define HCFS_THIS_CHOICE (stk__->stack[stk__->count-1])
+
+#endif

src/backends/llk.c

@@ -374,7 +374,7 @@ int test_llk(void)
   printf("first(A) = ");
   h_pprint_stringset(stdout, g, h_first(2, g, g->start), 0);
   printf("follow(C) = ");
-  h_pprint_stringset(stdout, g, h_follow(2, g, h_desugar(&system_allocator, c)), 0);
+  h_pprint_stringset(stdout, g, h_follow(2, g, h_desugar(&system_allocator, NULL, c)), 0);
 
   h_compile(p, PB_LLk, NULL);
 

src/backends/regex.c

@@ -354,6 +354,9 @@ static int h_regex_compile(HAllocator *mm__, HParser* parser, const void* params
   if (!parser->vtable->isValidRegular(parser->env))
     return 1;
   HRVMProg *prog = h_new(HRVMProg, 1);
+  prog->length = prog->action_count = 0;
+  prog->insns = NULL;
+  prog->actions = NULL;
   prog->allocator = mm__;
   if (!h_compile_regex(prog, parser)) {
     h_free(prog->insns);

src/cfgrammar.c

@@ -46,7 +46,7 @@ static void collect_geneps(HCFGrammar *grammar);
 HCFGrammar *h_cfgrammar(HAllocator* mm__, const HParser *parser)
 {
   // convert parser to CFG form ("desugar").
-  HCFChoice *desugared = h_desugar(mm__, parser);
+  HCFChoice *desugared = h_desugar(mm__, NULL, parser);
   if(desugared == NULL)
     return NULL;  // -> backend not suitable for this parser
 
@@ -65,6 +65,8 @@ HCFGrammar *h_cfgrammar(HAllocator* mm__, const HParser *parser)
     nt->seq[0]->items[0] = desugared;
     nt->seq[0]->items[1] = NULL;
     nt->seq[1] = NULL;
+    nt->pred = NULL;
+    nt->action = NULL;
     nt->reshape = h_act_first;
     h_hashset_put(g->nts, nt);
     g->start = nt;

src/desugar.c

@@ -1,10 +1,20 @@
 #include "hammer.h"
 #include "internal.h"
+#include "backends/contextfree.h"
 
-HCFChoice *h_desugar(HAllocator *mm__, const HParser *parser) {
+HCFChoice *h_desugar(HAllocator *mm__, HCFStack *stk__, const HParser *parser) {
+  HCFStack *nstk__ = stk__;
   if(parser->desugared == NULL) {
+    if (nstk__ == NULL) {
+      nstk__ = h_cfstack_new(mm__);
+    }
     // we're going to do something naughty and cast away the const to memoize
-    ((HParser *)parser)->desugared = parser->vtable->desugar(mm__, parser->env);
+    parser->vtable->desugar(mm__, nstk__, parser->env);
+    ((HParser *)parser)->desugared = nstk__->last_completed;
+    if (stk__ == NULL)
+      h_cfstack_free(mm__, nstk__);
+  } else if (stk__ != NULL) {
+    HCFS_APPEND(parser->desugared);
   }
 
   return parser->desugared;

src/glue.c

@@ -207,7 +207,7 @@ const HParsedToken *h_seq_flatten(HArena *arena, const HParsedToken *p)
   switch(p->token_type) {
   case TT_SEQUENCE:
     // Flatten and append all.
-    for(size_t i; i<p->seq->used; i++) {
+    for(size_t i = 0; i<p->seq->used; i++) {
       h_seq_append(ret, h_seq_flatten(arena, h_seq_index(p, i)));
     }
     break;

src/internal.h

@@ -49,6 +49,7 @@ static inline void h_generic_free(HAllocator *allocator, void* ptr) {
 }
 
 extern HAllocator system_allocator;
+typedef struct HCFStack_ HCFStack;
 
 
 typedef struct HInputStream_ {
@@ -236,7 +237,7 @@ HParser *h_new_parser(HAllocator *mm__, const HParserVtable *vt, void *env) {
   return p;
 }
 
-HCFChoice *h_desugar(HAllocator *mm__, const HParser *parser);
+HCFChoice *h_desugar(HAllocator *mm__, HCFStack *stk__, const HParser *parser);
 
 HCountedArray *h_carray_new_sized(HArena * arena, size_t size);
 HCountedArray *h_carray_new(HArena * arena);
@@ -276,8 +277,9 @@ HHashValue h_hash_ptr(const void *p);
 
 typedef struct HCFSequence_ HCFSequence;
 
-typedef struct HCFChoice_ {
-  enum {
+
+struct HCFChoice_ {
+  enum HCFChoiceType {
     HCF_END,
     HCF_CHOICE,
     HCF_CHARSET,
@@ -292,7 +294,7 @@ typedef struct HCFChoice_ {
                     // to execute before action and pred are applied.
   HAction action;
   HPredicate pred;
-} HCFChoice;
+};
 
 struct HCFSequence_ {
   HCFChoice **items; // last one is NULL
@@ -303,7 +305,7 @@ struct HParserVtable_ {
   bool (*isValidRegular)(void *env);
   bool (*isValidCF)(void *env);
   bool (*compile_to_rvm)(HRVMProg *prog, void* env); // FIXME: forgot what the bool return value was supposed to mean.
-  HCFChoice* (*desugar)(HAllocator *mm__, void *env);
+  void (*desugar)(HAllocator *mm__, HCFStack *stk__, void *env);
 };
 
 bool h_false(void*);

src/parsers/action.c

@@ -20,20 +20,16 @@ static HParseResult* parse_action(void *env, HParseState *state) {
     return NULL;
 }
 
-static HCFChoice* desugar_action(HAllocator *mm__, void *env) {
+static void desugar_action(HAllocator *mm__, HCFStack *stk__, void *env) {
   HParseAction *a = (HParseAction*)env;
-  HCFSequence *seq = h_new(HCFSequence, 1);
-  seq->items = h_new(HCFChoice*, 2);
-  seq->items[0] = h_desugar(mm__, a->p);
-  seq->items[1] = NULL;
-  HCFChoice *ret = h_new(HCFChoice, 1);
-  ret->type = HCF_CHOICE;
-  ret->seq = h_new(HCFSequence*, 2);
-  ret->seq[0] = seq;
-  ret->seq[1] = NULL;
-  ret->action = a->action;
-  ret->reshape = h_act_first;
-  return ret;
+
+  HCFS_BEGIN_CHOICE() {
+    HCFS_BEGIN_SEQ() {
+      HCFS_DESUGAR(a->p);
+    } HCFS_END_SEQ();
+    HCFS_THIS_CHOICE->action = a->action;
+    HCFS_THIS_CHOICE->reshape = h_act_first;
+  } HCFS_END_CHOICE();
 }
 
 static bool action_isValidRegular(void *env) {

src/parsers/and.c

@@ -9,19 +9,13 @@ static HParseResult *parse_and(void* env, HParseState* state) {
   return NULL;
 }
 
-static HCFChoice* desugar_and(HAllocator *mm__, void *env) {
-  assert_message(0, "Not context-free, can't be desugared");
-  return NULL;
-}
-
 static const HParserVtable and_vt = {
   .parse = parse_and,
   .isValidRegular = h_false, /* TODO: strictly speaking this should be regular,
 				but it will be a huge amount of work and difficult
 				to get right, so we're leaving it for a future
 				revision. --mlp, 18/12/12 */
   .isValidCF = h_false,      /* despite TODO above, this remains false. */
-  .desugar = desugar_and,
   .compile_to_rvm = h_not_regular,
 };
 

src/parsers/attr_bool.c

@@ -33,20 +33,16 @@ static bool ab_isValidCF(void *env) {
   return ab->p->vtable->isValidCF(ab->p->env);
 }
 
-static HCFChoice* desugar_ab(HAllocator *mm__, void *env) {
+static void desugar_ab(HAllocator *mm__, HCFStack *stk__, void *env) {
+
   HAttrBool *a = (HAttrBool*)env;
-  HCFSequence *seq = h_new(HCFSequence, 1);
-  seq->items = h_new(HCFChoice*, 2);
-  seq->items[0] = h_desugar(mm__, a->p);
-  seq->items[1] = NULL;
-  HCFChoice *ret = h_new(HCFChoice, 1);
-  ret->type = HCF_CHOICE;
-  ret->seq = h_new(HCFSequence*, 2);
-  ret->seq[0] = seq;
-  ret->seq[1] = NULL;
-  ret->pred = a->pred;
-  ret->reshape = h_act_first;
-  return ret;
+  HCFS_BEGIN_CHOICE() {
+    HCFS_BEGIN_SEQ() {
+      HCFS_DESUGAR(a->p);
+    } HCFS_END_SEQ();
+    HCFS_THIS_CHOICE->pred = a->pred;
+    HCFS_THIS_CHOICE->reshape = h_act_first;
+  } HCFS_END_CHOICE();
 }
 
 static bool h_svm_action_attr_bool(HArena *arena, HSVMContext *ctx, void* arg) {