Remove staDFA algorithm.

Although it is an interesting algorithm, re2c benchmarks show that it
performs worse than TDFA(1): on complex regular expressions staDFA are
often much larger and slower than TDFA(1). See the benchmarks on the
official re2c website: https://re2c.org/benchmarks/benchmarks.html, or
the paper https://arxiv.org/abs/2206.01398 for details.

StaDFA implementation adds complexity to the most complex part of re2c
code -- determinization, and it takes some effort to maintain.

The algorithm is not fully documented and formalized: the original paper
by Chowdhury is incomplete, and the implementation is re2c had to use a
few fixes and modifications to make it work.

The benchmarks for staDFA now use re2c-3.0 to build.

Author: skvadrik

CMakeLists.txt

@@ -165,7 +165,6 @@ set(re2c_sources
     src/dfa/fillpoints.cc
     src/dfa/find_state.cc
     src/dfa/minimization.cc
-    src/dfa/stacmd.cc
     src/dfa/tagver_table.cc
     src/dfa/tcmd.cc
     src/encoding/ebcdic/ebcdic_regexp.cc
@@ -473,7 +472,6 @@ if (RE2C_BUILD_LIBS)
         src/dfa/fillpoints.cc
         src/dfa/find_state.cc
         src/dfa/minimization.cc
-        src/dfa/stacmd.cc
         src/dfa/tagver_table.cc
         src/dfa/tcmd.cc
         src/nfa/estimate_size.cc

MAINTAINERS.md

@@ -14,9 +14,9 @@ outside of lab (to the best of author's knowledge, that is):
   Libtool; remove a few internal options and a bunch of explicit template
   instantiations in TDFA construction code that exist only for libre2c.
 
-- Sta-DFA, TDFA(0): removing these algorithms would clean up the code and make
-  the difficult parts around TDFA construction easier to understand. These
-  algorithms are slower than TDFA(1) and were added for experimental purposes.
+- TDFA(0): removing this algorithm would clean up the code and make the
+  difficult parts around TDFA construction easier to understand. This algorithm
+  is slower than TDFA(1) and was added for experimental purposes.
 
 - One of the build systems: Autotools or CMake, whichever you like less. Someone
   will get unhappy in both cases (Windows is supported only by CMake, but distro

Makefile.am

@@ -30,7 +30,6 @@ re2c_HDR = \
 	src/dfa/determinization.h \
 	src/dfa/dfa.h \
 	src/dfa/posix_precedence.h \
-	src/dfa/stacmd.h \
 	src/dfa/tag_history.h \
 	src/dfa/tagver_table.h \
 	src/dfa/tcmd.h \
@@ -120,7 +119,6 @@ re2c_SRC = \
 	src/dfa/fillpoints.cc \
 	src/dfa/find_state.cc \
 	src/dfa/minimization.cc \
-	src/dfa/stacmd.cc \
 	src/dfa/tagver_table.cc \
 	src/dfa/tcmd.cc \
 	src/encoding/ebcdic/ebcdic_regexp.cc \

Makefile.lib.am

@@ -22,7 +22,6 @@ libre2c_la_HDR = \
 	src/dfa/determinization.h \
 	src/dfa/dfa.h \
 	src/dfa/posix_precedence.h \
-	src/dfa/stacmd.h \
 	src/dfa/tag_history.h \
 	src/dfa/tagver_table.h \
 	src/dfa/tcmd.h \
@@ -111,7 +110,6 @@ libre2c_la_SRC = \
 	src/dfa/fillpoints.cc \
 	src/dfa/find_state.cc \
 	src/dfa/minimization.cc \
-	src/dfa/stacmd.cc \
 	src/dfa/tagver_table.cc \
 	src/dfa/tcmd.cc \
 	src/nfa/estimate_size.cc \

benchmarks/submatch_dfa_aot/CMakeLists.txt

@@ -125,8 +125,15 @@ foreach(bench ${BENCHMARKS})
 endforeach()
 
 # re2c
+file(MAKE_DIRECTORY "${ENG_DIR}/re2c/")
 file(MAKE_DIRECTORY "${GEN_DIR}/re2c/")
 set(RE2C "${CMAKE_BINARY_DIR}/re2c")
+set(RE2C3 "${ENG_DIR}/re2c/re2c3")
+add_custom_command(
+    OUTPUT "${RE2C3}"
+    COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/engines/re2c/getre2c3.sh"
+    WORKING_DIRECTORY "${ENG_DIR}/re2c/"
+)
 set(RE2C_FLAGS "--reusable" "--tags" "--no-generation-date" "--no-version")
 set(COMMON_RE2C
     "${SRC_DIR}/re2c/common.re"
@@ -135,28 +142,44 @@ set(COMMON_RE2C
     "${SRC_DIR}/re2c/include-eof/fill.re"
     "${SRC_DIR}/re2c/include-eof/fill_email.re"
 )
+# deprecated algorithms were removed after re2c-3.0
+set(DEPRECATED_ALGS "stadfa")
 foreach(bench ${BENCHMARKS})
 foreach(eof "" "-eof")
 foreach(alg "tdfa1" "tdfa0" "stadfa")
     set(src_file "${SRC_DIR}/re2c/${bench}.re")
     set(gen_file "${GEN_DIR}/re2c/${bench}${eof}-${alg}.c")
     set(pregen_file "${PREGEN_DIR}/re2c/${bench}${eof}-${alg}.c")
-    # always regenerate re2c benchmarks (regardless of RE2C_REGEN_BENCHMARKS)
-    file(RELATIVE_PATH rel_src_file "${CMAKE_CURRENT_BINARY_DIR}" "${src_file}")
-    file(RELATIVE_PATH rel_gen_file "${CMAKE_CURRENT_BINARY_DIR}" "${gen_file}")
-    file(RELATIVE_PATH rel_inc_path "${CMAKE_CURRENT_BINARY_DIR}" "${SRC_DIR}/re2c/include${eof}")
-    set(re2c_flags ${RE2C_FLAGS} "-I" "${rel_inc_path}")
-    if("${alg}" STREQUAL "tdfa0")
-        set(re2c_flags ${re2c_flags} "--no-lookahead")
-    elseif("${alg}" STREQUAL "stadfa")
-        set(re2c_flags ${re2c_flags} "--stadfa")
+    # always regenerate re2c benchmarks, except for deprecated algorithms
+    if(RE2C_REGEN_BENCHMARKS OR NOT (alg IN_LIST DEPRECATED_ALGS))
+        # for deprecated algorithms use re2c-3.0
+        if(alg IN_LIST DEPRECATED_ALGS)
+            set(re2c_for_gen "${RE2C3}")
+        else()
+            set(re2c_for_gen "${RE2C}")
+        endif()
+        file(RELATIVE_PATH rel_src_file "${CMAKE_CURRENT_BINARY_DIR}" "${src_file}")
+        file(RELATIVE_PATH rel_gen_file "${CMAKE_CURRENT_BINARY_DIR}" "${gen_file}")
+        file(RELATIVE_PATH rel_inc_path "${CMAKE_CURRENT_BINARY_DIR}" "${SRC_DIR}/re2c/include${eof}")
+        set(re2c_flags ${RE2C_FLAGS} "-I" "${rel_inc_path}")
+        if("${alg}" STREQUAL "tdfa0")
+            set(re2c_flags ${re2c_flags} "--no-lookahead")
+        elseif("${alg}" STREQUAL "stadfa")
+            set(re2c_flags ${re2c_flags} "--stadfa")
+        endif()
+        add_custom_command(
+            OUTPUT "${gen_file}"
+            COMMAND "${re2c_for_gen}" ${re2c_flags} "${rel_src_file}" -o "${rel_gen_file}"
+            COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${gen_file}" "${pregen_file}"
+            DEPENDS "${src_file}" ${COMMON_RE2C} "${re2c_for_gen}"
+        )
+    else()
+        add_custom_command(
+            OUTPUT "${gen_file}"
+            COMMAND "${CMAKE_COMMAND}" -E copy "${pregen_file}" "${gen_file}"
+            DEPENDS "${pregen_file}"
+        )
     endif()
-    add_custom_command(
-        OUTPUT "${gen_file}"
-        COMMAND "${RE2C}" ${re2c_flags} "${rel_src_file}" -o "${rel_gen_file}"
-        COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${gen_file}" "${pregen_file}"
-        DEPENDS "${src_file}" ${COMMON_RE2C} "${RE2C}"
-    )
     list(APPEND GEN "${gen_file}")
 endforeach()
 endforeach()

benchmarks/submatch_dfa_aot/Makefile.am

@@ -7,6 +7,7 @@ DAT_DIR = data
 CFLAGS = -O3 -I $(SRC_DIR)
 RAGEL = $(ENG_DIR)/ragel/ragel7
 KLEENEX = $(ENG_DIR)/kleenex/kexc
+RE2C3 = $(ENG_DIR)/re2c/re2c3
 RE2C = $(top_builddir)/re2c
 RE2C_FLAGS_COMMON = --reusable --tags --no-generation-date --no-version
 RE2C_FLAGS     = $(RE2C_FLAGS_COMMON) -I $(SRC_DIR)/re2c/include
@@ -60,21 +61,23 @@ BIN_RAGEL = $(patsubst $(GEN_DIR)%.c, $(BIN_DIR)%, $(GEN_RAGEL))
 
 COMMON_RAGEL = $(COMMON_SRC) $(SRC_DIR)/ragel/common.c
 
-GEN_RE2C_TDFA1      = $(patsubst %, $(GEN_DIR)/re2c/%-tdfa1.c,      $(BENCHMARKS))
-GEN_RE2C_TDFA0      = $(patsubst %, $(GEN_DIR)/re2c/%-tdfa0.c,      $(BENCHMARKS))
-GEN_RE2C_STADFA     = $(patsubst %, $(GEN_DIR)/re2c/%-stadfa.c,     $(BENCHMARKS))
-GEN_RE2C_EOF_TDFA1  = $(patsubst %, $(GEN_DIR)/re2c/%-eof-tdfa1.c,  $(BENCHMARKS))
-GEN_RE2C_EOF_TDFA0  = $(patsubst %, $(GEN_DIR)/re2c/%-eof-tdfa0.c,  $(BENCHMARKS))
-GEN_RE2C_EOF_STADFA = $(patsubst %, $(GEN_DIR)/re2c/%-eof-stadfa.c, $(BENCHMARKS))
+GEN_RE2C_TDFA1     = $(patsubst %, $(GEN_DIR)/re2c/%-tdfa1.c,     $(BENCHMARKS))
+GEN_RE2C_TDFA0     = $(patsubst %, $(GEN_DIR)/re2c/%-tdfa0.c,     $(BENCHMARKS))
+GEN_RE2C_EOF_TDFA1 = $(patsubst %, $(GEN_DIR)/re2c/%-eof-tdfa1.c, $(BENCHMARKS))
+GEN_RE2C_EOF_TDFA0 = $(patsubst %, $(GEN_DIR)/re2c/%-eof-tdfa0.c, $(BENCHMARKS))
 GEN_RE2C = \
     $(GEN_RE2C_TDFA1) \
     $(GEN_RE2C_TDFA0) \
-    $(GEN_RE2C_STADFA) \
     $(GEN_RE2C_EOF_TDFA1) \
-    $(GEN_RE2C_EOF_TDFA0) \
-    $(GEN_RE2C_EOF_STADFA)
+    $(GEN_RE2C_EOF_TDFA0)
 
-BIN_RE2C = $(patsubst $(GEN_DIR)%.c, $(BIN_DIR)%, $(GEN_RE2C))
+GEN_RE2C3_STADFA     = $(patsubst %, $(GEN_DIR)/re2c/%-stadfa.c,     $(BENCHMARKS))
+GEN_RE2C3_EOF_STADFA = $(patsubst %, $(GEN_DIR)/re2c/%-eof-stadfa.c, $(BENCHMARKS))
+GEN_RE2C3 = \
+    $(GEN_RE2C3_STADFA) \
+    $(GEN_RE2C3_EOF_STADFA)
+
+BIN_RE2C = $(patsubst $(GEN_DIR)%.c, $(BIN_DIR)%, $(GEN_RE2C) $(GEN_RE2C3))
 
 COMMON_RE2C = \
     $(COMMON_SRC) \
@@ -97,7 +100,7 @@ GEN_KLEENEX = $(patsubst %, $(GEN_DIR)/kleenex/%.c, \
 
 BIN_KLEENEX = $(patsubst $(GEN_DIR)%.c, $(BIN_DIR)%, $(GEN_KLEENEX))
 
-GEN = $(GEN_RAGEL) $(GEN_RE2C) $(GEN_KLEENEX)
+GEN = $(GEN_RAGEL) $(GEN_RE2C) $(GEN_RE2C3) $(GEN_KLEENEX)
 
 BIN_ = $(BIN_RAGEL) $(BIN_RE2C) $(BIN_KLEENEX)
 BIN_GCC   = $(patsubst %, %-gcc, $(BIN_))
@@ -113,7 +116,7 @@ DAT = $(patsubst %, $(DAT_DIR)/%/big, \
 
 all-local: $(BIN) $(DAT)
 
-# always regenerate re2c benchmarks
+# always regenerate re2c benchmarks (except for deprecated algorithms)
 
 $(GEN_RE2C_TDFA1): $(GEN_DIR)/%-tdfa1.c: $(SRC_DIR)/%.re $(COMMON_RE2C) $(RE2C)
 	$(AM_V_at)mkdir -p $(@D)
@@ -135,19 +138,21 @@ $(GEN_RE2C_EOF_TDFA0): $(GEN_DIR)/%-eof-tdfa0.c: $(SRC_DIR)/%.re $(COMMON_RE2C)
 	$(AM_V_GEN)$(RE2C) $(RE2C_FLAGS_EOF) --no-lookahead $< -o $@
 	$(AM_V_at)if ! cmp -s $@ $(PREGEN_DIR)/re2c/$(@F) ; then cp -f $@ $(PREGEN_DIR)/re2c/$(@F) ; fi
 
-$(GEN_RE2C_STADFA): $(GEN_DIR)/%-stadfa.c: $(SRC_DIR)/%.re $(COMMON_RE2C) $(RE2C)
+# optionally regenerate ragel, kleenex and re2c-3.0 benchmarks
+# (staDFA algorithm was removed in re2c versions after 3.0)
+
+if REGEN_BENCHMARKS
+
+$(GEN_RE2C3_STADFA): $(GEN_DIR)/%-stadfa.c: $(SRC_DIR)/%.re $(COMMON_RE2C) $(RE2C3)
 	$(AM_V_at)mkdir -p $(@D)
-	$(AM_V_GEN)$(RE2C) $(RE2C_FLAGS) --stadfa $< -o $@
+	$(AM_V_GEN)$(RE2C3) $(RE2C_FLAGS) --stadfa $< -o $@
 	$(AM_V_at)if ! cmp -s $@ $(PREGEN_DIR)/re2c/$(@F) ; then cp -f $@ $(PREGEN_DIR)/re2c/$(@F) ; fi
 
-$(GEN_RE2C_EOF_STADFA): $(GEN_DIR)/%-eof-stadfa.c: $(SRC_DIR)/%.re $(COMMON_RE2C) $(RE2C)
+$(GEN_RE2C3_EOF_STADFA): $(GEN_DIR)/%-eof-stadfa.c: $(SRC_DIR)/%.re $(COMMON_RE2C) $(RE2C3)
 	$(AM_V_at)mkdir -p $(@D)
-	$(AM_V_GEN)$(RE2C) $(RE2C_FLAGS_EOF) --stadfa $< -o $@
+	$(AM_V_GEN)$(RE2C3) $(RE2C_FLAGS_EOF) --stadfa $< -o $@
 	$(AM_V_at)if ! cmp -s $@ $(PREGEN_DIR)/re2c/$(@F) ; then cp -f $@ $(PREGEN_DIR)/re2c/$(@F) ; fi
 
-# optionally regenerate ragel and kleenex benchmarks
-if REGEN_BENCHMARKS
-
 $(GEN_RAGEL): $(GEN_DIR)/%.c: $(SRC_DIR)/%.rl $(COMMON_RAGEL) $(RAGEL)
 	$(AM_V_at)mkdir -p $(@D)
 	$(AM_V_GEN)$(RAGEL) -G2 $< -o $@
@@ -160,7 +165,7 @@ $(GEN_KLEENEX): $(GEN_DIR)/%.c: $(SRC_DIR)/%.kex $(KLEENEX)
 
 else
 
-$(GEN_RAGEL) $(GEN_KLEENEX): $(GEN_DIR)/%: $(PREGEN_DIR)/%
+$(GEN_RAGEL) $(GEN_RE2C3) $(GEN_KLEENEX): $(GEN_DIR)/%: $(PREGEN_DIR)/%
 	$(AM_V_at)mkdir -p $(@D)
 	$(AM_V_GEN)cp -f $< $@
 
@@ -188,5 +193,9 @@ $(RAGEL):
 	$(AM_V_at)mkdir -p $(@D) && cp $(srcdir)/engines/ragel/getragel7.sh $(@D)
 	$(AM_V_GEN)( cd $(@D) && ./getragel7.sh )
 
+$(RE2C3):
+	$(AM_V_at)mkdir -p $(@D) && cp $(srcdir)/engines/re2c/getre2c3.sh $(@D)
+	$(AM_V_GEN)( cd $(@D) && ./getre2c3.sh )
+
 clean-local:
 	$(AM_V_at)rm -f $(GEN) $(OBJ) $(BIN) $(DAT)

benchmarks/submatch_dfa_aot/engines/re2c/getre2c3.sh

@@ -0,0 +1,22 @@
+set -ex
+
+if [ -e re2c3 ] ; then
+    echo "re2c3 is already installed, quitting"
+    exit 0
+fi
+
+RE2C_DIR=re2c-3.0
+
+wget https://github.com/skvadrik/re2c/releases/download/3.0/re2c-3.0.tar.xz \
+    && tar xf re2c-3.0.tar.xz \
+    && rm re2c-3.0.tar.xz \
+    && cd "$RE2C_DIR" \
+    && ./configure \
+        --disable-golang \
+        --disable-rust \
+        --prefix=$(pwd)/install \
+    && make -j$(nproc) \
+    && make install \
+    && cd ..
+
+ln -s "$RE2C_DIR"/install/bin/re2c re2c3

benchmarks/submatch_dfa_jit/CMakeLists.txt

@@ -3,8 +3,6 @@ add_executable(bench_submatch_dfa_jit
     "${CMAKE_SOURCE_DIR}/benchmarks/common/common.cc"
 )
 
-set_property(TARGET bench_submatch_dfa_jit PROPERTY CXX_STANDARD 11)
-
 target_link_libraries(bench_submatch_dfa_jit libre2c)
 
 find_package(benchmark REQUIRED)

benchmarks/submatch_dfa_jit/Makefile.am

@@ -1,4 +1,4 @@
-bench_submatch_dfa_jit_CXXFLAGS = $(AM_CXXFLAGS) -std=c++11 -O3 -I $(top_srcdir)
+bench_submatch_dfa_jit_CXXFLAGS = $(AM_CXXFLAGS) -O3 -I $(top_srcdir)
 bench_submatch_dfa_jit_LDFLAGS = $(LDFLAGS_RE2) -lbenchmark -lpthread
 bench_submatch_dfa_jit_LDADD = $(top_builddir)/libre2c.la
 

benchmarks/submatch_nfa/CMakeLists.txt

@@ -3,8 +3,6 @@ add_executable(bench_submatch_nfa
     "${CMAKE_SOURCE_DIR}/benchmarks/common/common.cc"
 )
 
-set_property(TARGET bench_submatch_nfa PROPERTY CXX_STANDARD 11)
-
 target_link_libraries(bench_submatch_nfa libre2c)
 
 find_package(benchmark REQUIRED)