Initial ANTLR4 integration attempt using XML grammar from ANTLR book

build.gradle

@@ -68,7 +68,7 @@ configure(subprojects.findAll { it.name.startsWith('gen.antlr3') }) {
 configure(subprojects.findAll { it.name.startsWith('gen.antlr4') }) {
   apply plugin: 'antlr'
   dependencies {
-    antlr 'org.antlr:antlr4:4.5'
+    antlr 'org.antlr:antlr4:4.7'
   }
   if (it.name.startsWith('gen.antlr4-')) {
     dependencies {

gen.antlr3-xml/build.gradle

@@ -1 +1 @@
-description = 'GumTree tree generator for XML code (AntLR based).'
+description = 'GumTree tree generator for XML code (ANTLR3 based).'

gen.antlr4-xml/build.gradle

@@ -0,0 +1,13 @@
+description = 'GumTree tree generator for XML code (ANTLR4 based).'
+
+generateGrammarSource {
+  arguments = ['com/github/gumtreediff/gen/antlr4/xml/XMLLexer.g4', 'com/github/gumtreediff/gen/antlr4/xml/XMLParser.g4']
+}
+
+dependencies {
+    // see https://softnoise.wordpress.com/2014/09/07/gradle-sub-project-test-dependencies-in-multi-project-builds/
+    testCompile project(':gen.antlr4').sourceSets.test.output 
+    testCompile project(':client')
+    testCompile project(':client.diff')
+    testCompile 'junit:junit:4.+'
+}

gen.antlr4-xml/src/main/antlr/com/github/gumtreediff/gen/antlr4/xml/XMLLexer.g4

@@ -0,0 +1,98 @@
+/*
+ [The "BSD licence"]
+ Copyright (c) 2013 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+    derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/** XML lexer derived from ANTLR v4 ref guide book example */
+lexer grammar XMLLexer;
+
+
+@lexer::header {
+package  com.github.gumtreediff.gen.antlr4.xml;
+}
+
+// Default "mode": Everything OUTSIDE of a tag
+COMMENT     :   '<!--' .*? '-->' ;
+CDATA       :   '<![CDATA[' .*? ']]>' ;
+/** Scarf all DTD stuff, Entity Declarations like <!ENTITY ...>,
+ *  and Notation Declarations <!NOTATION ...>
+ */
+DTD         :   '<!' .*? '>'            -> skip ;
+EntityRef   :   '&' Name ';' ;
+CharRef     :   '&#' DIGIT+ ';'
+            |   '&#x' HEXDIGIT+ ';'
+            ;
+SEA_WS      :   (' '|'\t'|'\r'? '\n')+ ;
+
+OPEN        :   '<'                     -> pushMode(INSIDE) ;
+XMLDeclOpen :   '<?xml' S               -> pushMode(INSIDE) ;
+SPECIAL_OPEN:   '<?' Name               -> more, pushMode(PROC_INSTR) ;
+
+TEXT        :   ~[<&]+ ;        // match any 16 bit char other than < and &
+
+// ----------------- Everything INSIDE of a tag ---------------------
+mode INSIDE;
+
+CLOSE       :   '>'                     -> popMode ;
+SPECIAL_CLOSE:  '?>'                    -> popMode ; // close <?xml...?>
+SLASH_CLOSE :   '/>'                    -> popMode ;
+SLASH       :   '/' ;
+EQUALS      :   '=' ;
+STRING      :   '"' ~[<"]* '"'
+            |   '\'' ~[<']* '\''
+            ;
+Name        :   NameStartChar NameChar* ;
+S           :   [ \t\r\n]               -> skip ;
+
+fragment
+HEXDIGIT    :   [a-fA-F0-9] ;
+
+fragment
+DIGIT       :   [0-9] ;
+
+fragment
+NameChar    :   NameStartChar
+            |   '-' | '_' | '.' | DIGIT
+            |   '\u00B7'
+            |   '\u0300'..'\u036F'
+            |   '\u203F'..'\u2040'
+            ;
+
+fragment
+NameStartChar
+            :   [:a-zA-Z]
+            |   '\u2070'..'\u218F'
+            |   '\u2C00'..'\u2FEF'
+            |   '\u3001'..'\uD7FF'
+            |   '\uF900'..'\uFDCF'
+            |   '\uFDF0'..'\uFFFD'
+            ;
+
+// ----------------- Handle <? ... ?> ---------------------
+mode PROC_INSTR;
+
+PI          :   '?>'                    -> popMode ; // close <?...?>
+IGNORE      :   .                       -> more ;

gen.antlr4-xml/src/main/antlr/com/github/gumtreediff/gen/antlr4/xml/XMLParser.g4

@@ -0,0 +1,58 @@
+/*
+ [The "BSD licence"]
+ Copyright (c) 2013 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+    derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/** XML parser derived from ANTLR v4 ref guide book example */
+parser grammar XMLParser;
+
+options { tokenVocab=XMLLexer; }
+
+@parser::header {
+package  com.github.gumtreediff.gen.antlr4.xml;
+}
+
+document    :   prolog? misc* element misc*;
+
+prolog      :   XMLDeclOpen attribute* SPECIAL_CLOSE ;
+
+content     :   chardata?
+                ((element | reference | CDATA | PI | COMMENT) chardata?)* ;
+
+element     :   '<' Name attribute* '>' content '<' '/' Name '>'
+            |   '<' Name attribute* '/>'
+            ;
+
+reference   :   EntityRef | CharRef ;
+
+attribute   :   Name '=' STRING ; // Our STRING is AttValue in spec
+
+/** ``All text that is not markup constitutes the character data of
+ *  the document.''
+ */
+chardata    :   TEXT | SEA_WS ;
+
+misc        :   COMMENT | PI | SEA_WS ;

gen.antlr4-xml/src/main/java/com/github/gumtreediff/gen/antlr4/xml/XmlTreeGenerator.java

@@ -0,0 +1,60 @@
+/*
+ * This file is part of GumTree.
+ *
+ * GumTree is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * GumTree is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with GumTree.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Copyright 2011-2015 Jean-Rémy Falleri <jr.falleri@gmail.com>
+ * Copyright 2011-2015 Floréal Morandat <florealm@gmail.com>
+ * Copyright 2017 Svante Schubert <svante.schubert gmail com>
+ */
+package com.github.gumtreediff.gen.antlr4.xml;
+
+import com.github.gumtreediff.gen.Register;
+import com.github.gumtreediff.gen.antlr4.AbstractAntlr4TreeGenerator;
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.RecognitionException;
+import org.antlr.v4.runtime.Vocabulary;
+
+@Register(id = "xml-antlr4", accept = {"\\.xml$", "\\.xsd$", "\\.wadl$"})
+/**
+ * Creating XML from the grammar used in ANTLR 4 reference book and available at
+ */
+public class XmlTreeGenerator extends AbstractAntlr4TreeGenerator<XMLLexer, XMLParser> {
+
+    @Override
+    public XMLLexer getLexer(CharStream stream) {
+        return new XMLLexer(stream);
+    }
+
+    @Override
+    public XMLParser getParser(CommonTokenStream tokens) {
+        return new XMLParser(tokens);
+    }
+
+    @Override
+    public ParserRuleContext getStartRule(XMLParser parser) throws RecognitionException {
+        return parser.document();
+    }
+
+    /**
+     * Preferable against XMLParser.tokenNames
+     *
+     * @return new ANTLR4 token dictionary
+     */
+    public Vocabulary getVocabulary() {
+        return XMLLexer.VOCABULARY;
+    }
+}

gen.antlr4-xml/src/test/java/com/github/gumtreediff/gen/antlr4/xml/XmlGrammarTest.java

@@ -0,0 +1,55 @@
+/*
+ * This file is part of GumTree.
+ *
+ * GumTree is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * GumTree is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with GumTree.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Copyright 2017 Svante Schubert <svante.schubert gmail com>
+ */
+package com.github.gumtreediff.gen.antlr4.xml;
+
+import com.github.gumtreediff.gen.TreeGenerator;
+import com.github.gumtreediff.gen.antlr4.TreeComparisonBase;
+
+/**
+ * The given test file pairs are being used as input for the tree generator. All
+ * serializers are being used is used to dump the created graphs. In addition,
+ * an edit script showing the difference of the pair is created. Finally, the
+ * new output files from '/build/created-test-files' are being compared with
+ * references from 'src\test\resources\references'.
+ */
+public class XmlGrammarTest extends TreeComparisonBase {
+
+    /**
+     * <b>IMPORTANT</b>
+     * When adding new test pairs to the array, the new test output from the
+     * output folder: '/build/created-test-files' needs to be copied manually to
+     * the reference folder: 'src\test\resources\references
+     */
+    private static final String[][] testCouplesXML = new String[][]{
+        {"books1.xml", "books2.xml"},
+        {"books1.xml", "books1b.xml"},
+        {"books1.xml", "books1c.xml"},
+        {"books1b.xml", "books1c.xml"},
+        {"web.xml", "web1.xml"}
+    };
+
+    public XmlGrammarTest() {
+        super(testCouplesXML);
+    }
+
+    @Override
+    protected TreeGenerator getTreeGenerator() {
+        return new XmlTreeGenerator();
+    }
+}