Auto merge of #21452 - bleibig:bison-grammar, r=nikomatsakis

This adds a new lexer/parser combo for the entire Rust language can be generated with with flex and bison, taken from my project at https://github.com/bleibig/rust-grammar. There is also a testing script that runs the generated parser with all *.rs files in the repository (except for tests in compile-fail or ones that marked as "ignore-test" or "ignore-lexer-test"). If you have flex and bison installed, you can run these tests using the new "check-grammar" make target.

This does not depend on or interact with the existing testing code in the grammar, which only provides and tests a lexer specification.

OS X users should take note that the version of bison that comes with the Xcode toolchain (2.3) is too old to work with this grammar, they need to download and install version 3.0 or later.

The parser builds up an S-expression-based AST, which can be displayed by giving the "-v" argument to parser-lalr (normally it only gives output on error). It is only a rough approximation of what is parsed and doesn't capture every detail and nuance of the program.

Hopefully this should be sufficient for issue #2234, or at least a good starting point.

configure

@@ -645,6 +645,8 @@ probe CFG_ISCC             iscc
 probe CFG_JAVAC            javac
 probe CFG_ANTLR4           antlr4
 probe CFG_GRUN             grun
+probe CFG_FLEX             flex
+probe CFG_BISON            bison
 probe CFG_PANDOC           pandoc
 probe CFG_PDFLATEX         pdflatex
 probe CFG_XELATEX          xelatex

mk/grammar.mk

@@ -14,6 +14,11 @@ B = $(CFG_BUILD_DIR)/$(CFG_BUILD)/stage2/
 L = $(B)lib/rustlib/$(CFG_BUILD)/lib
 LD = $(CFG_BUILD)/stage2/lib/rustlib/$(CFG_BUILD)/lib/
 RUSTC = $(STAGE2_T_$(CFG_BUILD)_H_$(CFG_BUILD))
+ifeq ($(CFG_OSTYPE),apple-darwin)
+	FLEX_LDFLAGS=-ll
+else
+	FLEX_LDFLAGS=-lfl
+endif
 
 # Run the reference lexer against libsyntax and compare the tokens and spans.
 # If "// ignore-lexer-test" is present in the file, it will be ignored.
@@ -67,3 +72,46 @@ $(info cfg: javac not available, skipping lexer test...)
 check-lexer:
 
 endif
+
+$(BG)lex.yy.c: $(SG)lexer.l $(BG)
+	@$(call E, flex: $@)
+	$(Q)$(CFG_FLEX) -o $@ $<
+
+$(BG)lexer-lalr.o: $(BG)lex.yy.c $(BG)parser-lalr.tab.h
+	@$(call E, cc: $@)
+	$(Q)$(CFG_CC) -include $(BG)parser-lalr.tab.h -c -o $@ $<
+
+$(BG)parser-lalr.tab.c $(BG)parser-lalr.tab.h: $(SG)parser-lalr.y
+	@$(call E, bison: $@)
+	$(Q)$(CFG_BISON) $< --output=$(BG)parser-lalr.tab.c --defines=$(BG)parser-lalr.tab.h \
+		--name-prefix=rs --warnings=error=all
+
+$(BG)parser-lalr.o: $(BG)parser-lalr.tab.c
+	@$(call E, cc: $@)
+	$(Q)$(CFG_CC) -c -o $@ $<
+
+$(BG)parser-lalr-main.o: $(SG)parser-lalr-main.c
+	@$(call E, cc: $@)
+	$(Q)$(CFG_CC) -std=c99 -c -o $@ $<
+
+$(BG)parser-lalr: $(BG)parser-lalr.o $(BG)parser-lalr-main.o $(BG)lexer-lalr.o
+	@$(call E, cc: $@)
+	$(Q)$(CFG_CC) -o $@ $^ $(FLEX_LDFLAGS)
+
+
+ifdef CFG_FLEX
+ifdef CFG_BISON
+check-grammar: $(BG) $(BG)parser-lalr
+	$(info Verifying grammar ...)
+	$(SG)testparser.py -p $(BG)parser-lalr -s $(S)src
+
+else
+$(info cfg: bison not available, skipping parser test...)
+check-grammar:
+
+endif
+else
+$(info cfg: flex not available, skipping parser test...)
+check-grammar:
+
+endif