Add Apache Rat check to gradle

Additional Authors: Dawid Weiss <dweiss@apache.org>

build.gradle

@@ -51,6 +51,7 @@ apply from: file('gradle/validation/git-status.gradle')
 apply from: file('gradle/validation/versions-props-sorted.gradle')
 apply from: file('gradle/validation/validate-source-patterns.gradle')
 apply from: file('gradle/validation/config-file-sanity.gradle')
+apply from: file('gradle/validation/rat-sources.gradle')
 
 // Additional development aids.
 apply from: file('gradle/maven/maven-local.gradle')

gradle/validation/precommit.gradle

@@ -22,6 +22,7 @@ configure(rootProject) {
           "forbiddenApisTest",
           "licenses",
           "javadoc",
+          "rat",
       ]}
     }
   }

gradle/validation/rat-sources.gradle

@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import groovy.xml.NamespaceBuilder
+
+configure(rootProject) {
+    configurations {
+        rat
+    }
+
+    dependencies {
+        rat "org.apache.rat:apache-rat"
+    }
+}
+
+subprojects {
+    plugins.withId("java", {
+        task("rat", type: RatTask) {
+            group = 'Verification'
+            description = 'Runs Apache Rat checks.'
+        }
+
+        // ant had an exclude for ./lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/bocchan.utf-8
+        // lucene/analysis/kuromoji/build.xml:  <property name="rat.excludes" value="**/*.txt,**/bocchan.utf-8"/>
+        // why don't we need it anymore?
+
+        if (project.path == ":lucene:analysis:common") {
+            rat {
+                excludes += [
+                    "**/*.aff",
+                    "**/*.dic",
+                    "**/charfilter/*.htm*",
+                    "**/*LuceneResourcesWikiPage.html"
+                ]
+            }
+        }
+
+        if (project.path == ":lucene:highlighter") {
+            rat {
+                excludes += [
+                    "**/CambridgeMA.utf8"
+                ]
+            }
+        }
+
+        if (project.path == ":lucene:suggest") {
+            rat {
+                excludes += [
+                    "**/Top50KWiki.utf8",
+                    "**/stop-snowball.txt"
+                ]
+            }
+        }
+
+        if (project.path == ":solr:core") {
+            rat {
+                excludes += [
+                    "**/htmlStripReaderTest.html"
+                ]
+            }
+        }
+    })
+}
+
+// Somewhat inspired by existing task from Apache Kafka
+class RatTask extends DefaultTask {
+    @Input
+    List<String> includes = [
+    ]
+
+    @Input
+    List<String> excludes = [
+        "**/TODO",
+        "**/*.txt",
+        "**/*.iml",
+        "**/*.gradle",
+        "build/**"
+    ]
+
+    @Input
+    List<String> additionalExcludes = []
+
+    def reportDir = new File(project.buildDir, 'rat')
+    def xmlReport = new File(reportDir, 'rat-report.xml')
+
+    def generateXmlReport(File reportDir) {
+        def uri = 'antlib:org.apache.rat.anttasks'
+        def ratClasspath = project.rootProject.configurations.rat.asPath
+        ant.taskdef(resource: 'org/apache/rat/anttasks/antlib.xml', uri: uri, classpath: ratClasspath)
+
+        def rat = NamespaceBuilder.newInstance(ant, uri)
+        rat.report(format: 'xml', reportFile: xmlReport, addDefaultLicenseMatchers: true) {
+            ant.fileset(dir: "${project.projectDir}") {
+                ant.include(name: '*.xml')
+                includes.each {
+                    ant.include(name: it)
+                }
+                additionalExcludes.each {
+                    ant.exclude(name: it)
+                }
+            }
+
+            [
+                project.sourceSets.main.java.srcDirs,
+                project.sourceSets.test.java.srcDirs,
+            ].flatten().each { srcLocation ->
+                ant.fileset(dir: srcLocation, erroronmissingdir: false) {
+                    excludes.each { x -> ant.exclude(name: x) }
+                }
+            }
+
+            [
+                project.sourceSets.main.resources.srcDirs
+            ].flatten().each { srcLocation ->
+                ant.fileset(dir: srcLocation, erroronmissingdir: false) {
+                    ant.include(name: "META-INF/**")
+                }
+            }
+
+            // The license rules below were manually copied from lucene/common-build.xml, there is currently no mechanism to sync them
+
+            // BSD 4-clause stuff (is disallowed below)
+            substringMatcher(licenseFamilyCategory: "BSD4 ", licenseFamilyName: "Original BSD License (with advertising clause)") {
+                pattern(substring: "All advertising materials")
+            }
+
+            // BSD-like stuff
+            substringMatcher(licenseFamilyCategory: "BSD  ", licenseFamilyName: "Modified BSD License") {
+                // brics automaton
+                pattern(substring: "Copyright (c) 2001-2009 Anders Moeller")
+                // snowball
+                pattern(substring: "Copyright (c) 2001, Dr Martin Porter")
+                // UMASS kstem
+                pattern(substring: "THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF MASSACHUSETTS AND OTHER CONTRIBUTORS")
+                // Egothor
+                pattern(substring: "Egothor Software License version 1.00")
+                // JaSpell
+                pattern(substring: "Copyright (c) 2005 Bruno Martins")
+                // d3.js
+                pattern(substring: "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS")
+                // highlight.js
+                pattern(substring: "THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS")
+            }
+
+            // MIT-like
+            substringMatcher(licenseFamilyCategory: "MIT  ", licenseFamilyName:"Modified BSD License") {
+                // ICU license
+                pattern(substring: "Permission is hereby granted, free of charge, to any person obtaining a copy")
+            }
+
+            // Apache
+            substringMatcher(licenseFamilyCategory: "AL   ", licenseFamilyName: "Apache") {
+                pattern(substring: "Licensed to the Apache Software Foundation (ASF) under")
+                // this is the old - school one under some files
+                pattern(substring: 'Licensed under the Apache License, Version 2.0 (the "License")')
+            }
+
+            substringMatcher(licenseFamilyCategory: "GEN  ", licenseFamilyName: "Generated") {
+                // svg files generated by gnuplot
+                pattern(substring: "Produced by GNUPLOT")
+                // snowball stemmers generated by snowball compiler
+                pattern(substring: "This file was generated automatically by the Snowball to Java compiler")
+                // parsers generated by antlr
+                pattern(substring: "ANTLR GENERATED CODE")
+            }
+
+            approvedLicense(familyName: "Apache")
+            approvedLicense(familyName: "The MIT License")
+            approvedLicense(familyName: "Modified BSD License")
+            approvedLicense(familyName: "Generated")
+        }
+    }
+
+    def printUnknownFiles() {
+        def ratXml = new XmlParser().parse(xmlReport)
+        def errors = []
+        ratXml.resource.each { resource ->
+            if (resource.'license-approval'.@name[0] == "false") {
+                errors << "Unknown license: ${resource.@name}"
+            }
+        }
+        if (errors) {
+            throw new GradleException("Found " + errors.size() + " file(s) with errors:\n" +
+                    errors.collect{ msg -> "  - ${msg}" }.join("\n"))
+        }
+    }
+
+    @TaskAction
+    def rat() {
+        reportDir.mkdirs()
+        def origEncoding = System.getProperty("file.encoding")
+        try {
+            generateXmlReport(reportDir)
+            printUnknownFiles()
+        } finally {
+            if (System.getProperty("file.encoding") != origEncoding) {
+                throw new GradleException("Insane: rat changed file.encoding to ${System.setProperty('file.encoding')}?")
+            }
+        }
+    }
+}

gradle/validation/validate-source-patterns.gradle

@@ -3,16 +3,16 @@
 // This should be eventually rewritten in plain gradle. For now, delegate to
 // the ant/groovy script we already have.
 
-configurations {
-  checkSourceDeps
-}
+configure(rootProject) {
+  configurations {
+    checkSourceDeps
+  }
 
-dependencies {
-  checkSourceDeps "org.codehaus.groovy:groovy-all:2.4.17"
-  checkSourceDeps "org.apache.rat:apache-rat:0.11"
-}
+  dependencies {
+    checkSourceDeps "org.codehaus.groovy:groovy-all:2.4.17"
+    checkSourceDeps "org.apache.rat:apache-rat"
+  }
 
-configure(rootProject) {
   task validateSourcePatterns() {
     doFirst {
       ant.taskdef(

versions.props

@@ -72,6 +72,7 @@ org.apache.opennlp:opennlp-tools=1.9.1
 org.apache.pdfbox:*=2.0.17
 org.apache.pdfbox:jempbox=1.8.16
 org.apache.poi:*=4.1.1
+org.apache.rat:apache-rat:0.11
 org.apache.tika:*=1.23
 org.apache.velocity.tools:*=3.0
 org.apache.xmlbeans:xmlbeans=3.1.0