Use Aho-Corasick for alts of literals during regexp AST compilation

Makefile

@@ -60,6 +60,7 @@ SUBDIR += tests/minimise
 SUBDIR += tests/native
 SUBDIR += tests/pcre
 SUBDIR += tests/pcre-classes
+SUBDIR += tests/pcre-ac
 SUBDIR += tests/pcre-anchor
 SUBDIR += tests/pcre-repeat
 SUBDIR += tests/pred

examples/bm/libfsm.c

@@ -15,6 +15,7 @@
 #include <fsm/fsm.h>
 #include <fsm/options.h>
 #include <fsm/print.h>
+
 #include <re/re.h>
 
 int
@@ -61,7 +62,7 @@ main(int argc, char *argv[])
 	opt.io = FSM_IO_STR;
 
 	p = argv[0];
-	fsm = re_comp(RE_PCRE, fsm_sgetc, &p, &opt, flags, &e);
+	fsm = re_comp_new(RE_PCRE, fsm_sgetc, &p, &opt, flags, &e);
 	if (fsm == NULL) {
 		re_perror(RE_LITERAL, &e, NULL, s);
 		return 1;

examples/words/main.c

@@ -127,7 +127,7 @@ int main(int argc, char *argv[]) {
 			fsm_state_t rs;
 			struct fsm *r;
 
-			r = re_comp(native ? RE_NATIVE : RE_LITERAL, fsm_sgetc, &p, &opt, 0, &e);
+			r = re_comp_new(native ? RE_NATIVE : RE_LITERAL, fsm_sgetc, &p, &opt, 0, &e);
 			if (r == NULL) {
 				re_perror(native ? RE_NATIVE : RE_LITERAL, &e, NULL, s);
 				return 1;
@@ -161,19 +161,29 @@ int main(int argc, char *argv[]) {
 
 	if (ahocorasick) {
 		struct timespec pre, post;
+		fsm_state_t start;
 
 		if (-1 == clock_gettime(CLOCK_MONOTONIC, &pre)) {
 			perror("clock_gettime");
 			exit(EXIT_FAILURE);
 		}
 
-		fsm = re_strings_build(g,
-			&opt, unanchored ? 0 : (RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT));
+		fsm = fsm_new(&opt);
 		if (fsm == NULL) {
+			perror("fsm_new");
+			exit(EXIT_FAILURE);
+		}
+
+		if (!re_strings_build(fsm, &start, g,
+			unanchored ? 0 : (RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT)))
+		{
 			perror("re_strings_builder_build");
+			fsm_free(fsm);
 			exit(EXIT_FAILURE);
 		}
 
+		fsm_setstart(fsm, start);
+
 		if (-1 == clock_gettime(CLOCK_MONOTONIC, &post)) {
 			perror("clock_gettime");
 			exit(EXIT_FAILURE);

include/re/re.h

@@ -127,9 +127,11 @@ re_getchar_fun(void *opaque);
 
 /*
  * Compile a regexp of the given dialect.
+ * States are populated into the given fsm. The start state is written out
+ * through the *start argument.
  *
- * Returns NULL on error. If non-NULL, the *err struct is populated with the
- * type and 0-indexed byte offset of the error.
+ * Returns 1 on success, or 0 on error. If non-NULL, the *err struct is
+ * populated with the type and 0-indexed byte offset of the error.
  *
  * libfsm provides getc callbacks suitable for use with re_comp; see <fsm/fsm.h>.
  * For example:
@@ -144,8 +146,19 @@ re_getchar_fun(void *opaque);
  * There's nothing special about libfsm's implementation of these; they could
  * equally well be user defined.
  */
+int
+re_comp(struct fsm *fsm, fsm_state_t *start, enum re_dialect dialect,
+	re_getchar_fun *f, void *opaque,
+	const struct fsm_options *opt,
+	enum re_flags flags, struct re_err *err);
+
+/*
+ * A convenience to construct a new fsm.
+ * Returns NULL on error.
+ * See re_comp() for details.
+ */
 struct fsm *
-re_comp(enum re_dialect dialect,
+re_comp_new(enum re_dialect dialect,
 	re_getchar_fun *f, void *opaque,
 	const struct fsm_options *opt,
 	enum re_flags flags, struct re_err *err);

include/re/strings.h

@@ -31,6 +31,10 @@ enum re_strings_flags {
 	RE_STRINGS_AC_AUTOMATON = 1 << 2
 };
 
+/*
+ * A convenience to iterate over an array of strings, and call
+ * re_strings_add_str() for each.
+ */
 struct fsm *
 re_strings(const struct fsm_options *opt, const char *a[], size_t n,
 	enum re_strings_flags flags);
@@ -47,9 +51,14 @@ re_strings_add_raw(struct re_strings *g, const void *p, size_t n);
 int
 re_strings_add_str(struct re_strings *g, const char *s);
 
-struct fsm *
-re_strings_build(struct re_strings *g,
-	const struct fsm_options *opt, enum re_strings_flags flags);
+int
+re_strings_build_into(struct fsm *fsm, fsm_state_t *start,
+	int have_end, fsm_state_t end,
+	struct re_strings *g, enum re_strings_flags flags);
+
+int
+re_strings_build(struct fsm *fsm, fsm_state_t *start,
+	struct re_strings *g, enum re_strings_flags flags);
 
 #endif
 

src/libre/Makefile

@@ -18,7 +18,7 @@ CFLAGS.${src} += -I src # XXX: for internal.h
 DFLAGS.${src} += -I src # XXX: for internal.h
 .endfor
 
-.for src in ${SRC:Msrc/libre/ast.c} ${SRC:Msrc/libre/ast_analysis.c} ${SRC:Msrc/libre/ast_compile.c} ${SRC:Msrc/libre/re.c}
+.for src in ${SRC:Msrc/libre/ast.c} ${SRC:Msrc/libre/ast_analysis.c} ${SRC:Msrc/libre/ast_compile.c} ${SRC:Msrc/libre/re.c} ${SRC:Msrc/libre/ac.c}
 CFLAGS.${src} += -std=c99 # XXX: for ast.h
 DFLAGS.${src} += -std=c99 # XXX: for ast.h
 .endfor

src/libre/ac.c

@@ -6,6 +6,7 @@
 
 #include <assert.h>
 #include <stddef.h>
+#include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
 #include <limits.h>
@@ -14,7 +15,10 @@
 
 #include <fsm/fsm.h>
 
+#include <re/re.h>
+
 #include "ac.h"
+#include "ast.h"
 
 enum { POOL_BLOCK_SIZE = 256 };
 
@@ -162,8 +166,53 @@ trie_add_word(struct trie_graph *g, const char *w, size_t n)
 	return g;
 }
 
+/* internal convenience to avoid constructing a string */
+struct trie_graph *
+trie_add_concat(struct trie_graph *g, const struct ast_expr **a, size_t n)
+{
+	struct trie_state *st;
+	size_t i;
+
+	assert(g != NULL);
+	assert(a != NULL);
+	assert(n > 0);
+
+	st = g->root;
+
+	assert(st != NULL);
+
+	for (i = 0; i < n; i++) {
+		struct trie_state *nx;
+		int idx;
+
+		assert(a[i]->type == AST_EXPR_LITERAL);
+
+		idx = (unsigned char)a[i]->u.literal.c;
+		nx = st->children[idx];
+
+		if (nx == NULL) {
+			nx = newstate(g);
+			if (nx == NULL) {
+				return NULL;
+			}
+
+			st->children[idx] = nx;
+		}
+
+		st = nx;
+	}
+
+	st->output = 1;
+	if (g->depth < n) {
+		g->depth = n;
+	}
+
+	return g;
+}
+
 int
-trie_add_failure_edges(struct trie_graph *g) {
+trie_add_failure_edges(struct trie_graph *g)
+{
 	struct trie_state **q;
 	size_t top,bot;
 	int sym;
@@ -267,6 +316,22 @@ find_next_state(struct trie_state *s, int sym)
 	return nx;
 }
 
+static int
+has_child(const struct trie_state *ts)
+{
+	int sym;
+
+	assert(ts != NULL);
+
+	for (sym = 0; sym < 256; sym++) {
+		if (ts->children[sym] != NULL) {
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
 static int
 trie_to_fsm_state(struct trie_state *ts, struct fsm *fsm,
 	int have_end, fsm_state_t single_end,
@@ -278,7 +343,7 @@ trie_to_fsm_state(struct trie_state *ts, struct fsm *fsm,
 	assert(fsm != NULL);
 	assert(q != NULL);
 
-	if (ts->output && have_end) {
+	if (ts->output && have_end && !has_child(ts)) {
 		*q = single_end;
 		return 1;
 	}
@@ -314,24 +379,34 @@ trie_to_fsm_state(struct trie_state *ts, struct fsm *fsm,
 	}
 
 	if (ts->output) {
-		fsm_setend(fsm, st, 1);
+		if (!have_end) {
+			fsm_setend(fsm, st, 1);
+		} else {
+			/*
+			 * What would usually be an end state in the middle of the trie
+			 * needs an epsilon to hook it up to the single exit state.
+			 *
+			 * We can't set this as an end state, because the single_end
+			 * itself might not actually accept - for example in the middle of
+			 * recursive NFA construction when walking the regexp AST.
+			 */
+			fsm_addedge_epsilon(fsm, st, single_end);
+		}
 	}
 
 	*q = st;
 	return 1;
 }
 
 struct fsm *
-trie_to_fsm(struct fsm *fsm, struct trie_graph *g, int have_end, fsm_state_t end)
+trie_to_fsm(struct fsm *fsm, fsm_state_t *start, struct trie_graph *g, int have_end, fsm_state_t end)
 {
-	fsm_state_t start;
+	assert(start != NULL);
 
-	if (!trie_to_fsm_state(g->root, fsm, have_end, end, &start)) {
+	if (!trie_to_fsm_state(g->root, fsm, have_end, end, start)) {
 		return NULL;
 	}
 
-	fsm_setstart(fsm, start);
-
 	return fsm;
 }
 

src/libre/ac.h

@@ -11,6 +11,7 @@ struct fsm;
 struct fsm_state;
 struct fsm_options;
 
+struct ast_expr;
 struct trie_graph;
 
 struct trie_graph *
@@ -22,11 +23,14 @@ trie_free(struct trie_graph *g);
 struct trie_graph *
 trie_add_word(struct trie_graph *g, const char *w, size_t n);
 
+struct trie_graph *
+trie_add_concat(struct trie_graph *g, const struct ast_expr **a, size_t n);
+
 int
 trie_add_failure_edges(struct trie_graph *g);
 
 struct fsm *
-trie_to_fsm(struct fsm *fsm, struct trie_graph *g,
+trie_to_fsm(struct fsm *fsm, fsm_state_t *start, struct trie_graph *g,
 	int have_end, fsm_state_t end);
 
 void

src/libre/ast.c

@@ -11,6 +11,8 @@
 #include <stdlib.h>
 #include <limits.h>
 
+#include <fsm/fsm.h>
+
 #include <re/re.h>
 
 #include "class.h"
@@ -190,6 +192,7 @@ ast_endpoint_cmp(const struct ast_endpoint *a, const struct ast_endpoint *b)
 
 	default:
 		assert(!"unreached");
+		abort();
 	}
 }
 

src/libre/ast.h

@@ -266,4 +266,8 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque,
 	const struct fsm_options *opt,
 	enum re_flags flags, struct re_err *err, int *unsatisfiable);
 
+struct re_strings;
+int
+re_strings_add_concat(struct re_strings *g, const struct ast_expr **a, size_t n);
+
 #endif

src/libre/ast_analysis.c

@@ -10,6 +10,8 @@
 #include <stdlib.h>
 #include <stdio.h>
 
+#include <fsm/fsm.h>
+
 #include <re/re.h>
 
 #include "class.h"