Merge pull request #2030 from gasche/ml.ast

makefile rule to test parser changes by comparing ASTs stored in `.ml.ast` files

Changes

@@ -363,6 +363,10 @@ Working version
   Longident.t values.
   (Gabriel Scherer, review by Florian Angeletti and Thomas Refis)
 
+- GPR#2030: makefile targets to build AST files of sources
+  for parser testing. See parsing/HACKING.adoc.
+  (Gabriel Scherer, review by Nicolás Ojeda Bär)
+
 ### Bug fixes:
 
 - MPR#7847, GPR#2019: Fix an infinite loop that could occur when the

Makefile

@@ -1308,6 +1308,37 @@ partialclean::
 
 beforedepend:: bytecomp/opcodes.ml
 
+# Testing the parser -- see parsing/HACKING.adoc
+
+SOURCE_FILES=$(shell \
+  git ls-files '*.ml' '*.mli' \
+| grep -v '^experimental/')
+
+AST_FILES=$(addsuffix .ast,$(SOURCE_FILES))
+
+build-all-asts: $(AST_FILES)
+
+CAMLC_DPARSETREE := \
+	$(CAMLRUN) ./ocamlc -nostdlib -nopervasives \
+	  -stop-after parsing -dparsetree
+
+%.ml.ast: %.ml ocamlc
+	$(CAMLC_DPARSETREE) $< 2> $@ || exit 0
+# `|| exit 0` : some source files will fail to parse
+# (for example, they are meant as toplevel scripts
+# rather than source files, or are parse-error tests),
+# we ignore the failure in that case
+
+%.mli.ast: %.mli ocamlc
+	$(CAMLC_DPARSETREE) $< 2> $@ || exit 0
+
+.PHONY: list-all-asts
+list-all-asts:
+	@for f in $(AST_FILES); do echo "'$$f'"; done
+
+partialclean::
+	rm -f $(AST_FILES)
+
 # Default rules
 
 .SUFFIXES: .ml .mli .cmo .cmi .cmx

parsing/HACKING.adoc

@@ -6,3 +6,71 @@ read before any further exploration of the compiler.
 link:location.mli[Location]:: This module contains utilities
 related to locations and error handling. In particular, it contains
 handlers that are used for all the error reporting in the compiler.
+
+link:parser.mly[parser.mly]:: This file contains the grammar used to
+generated the parser -- using the
+link:http://gallium.inria.fr/~fpottier/menhir/[menhir] parser
+generator, which is an external tool that you need to install if you
+wish to modify the parser.
+
+=== Working on the parser grammar
+
+To avoid depending on an external tool, the compiler build system does
+not rebuild the parser from the source grammar link:parser.mly[] each
+time. It works from a versioned copy of the generated parser stored
+in the `boot/menhir` subdirectory.
+
+If you change link:parser.mly[], you need to run the `promote-menhir`
+target of the root Makefile to rebuild the compiler parser. See
+link:../Makefile.menhir[] for the details of the various
+Menhir-related targets and their use.
+
+==== Testing the grammar
+
+The root Makefile contains a `build-all-asts` target that will build,
+for each source `.ml` or `.mli` file in the repository, a `.ml.ast` or
+`.mli.ast` file describing the parsed abstract syntax tree (AST) in
+`-dparsetree` format.
+This rule is rather slow to run, and can safely be run in parallel, so
+we recommend using `-j` (without a number) to maximize parallelism:
+
+----
+make -j build-all-asts
+----
+
+Finally, the 'list-all-asts' target lists all such '.ast' files.
+
+This is intended to be used to test parser changes, in particular
+those that should not modify the parsed AST at all:
+
+1. Before performing any changes, build all AST files and add them to
+   the git index (`make list-all-asts | xargs git add`).
+
+2. Perform any parser change of interest.
+
+3. To test your changes, build AST files again; `git diff` will show
+   any change to an AST file.
+
+4. Before committing any change, remember to remove the `.ast` files
+   from your index (using `git reset HEAD`), and maybe remove them
+   completely (unless you plan to check further changes).
+
+----
+# save pre-change ASTs
+make -j build-all-asts
+make list-all-asts | xargs git add
+
+# do your parser changes
+# ...
+make promote-menhir
+
+# compare new ASTs
+make -j build-all-asts
+git diff # shows any .ml.ast difference
+
+# remove AST files from the index
+make list-all-asts | xargs git reset HEAD
+
+# remove the files (if no further parser change planned)
+make list-all-asts | xargs rm
+----