diff --git a/build-tools/build-infra/src/main/groovy/lucene.documentation.gradle b/build-tools/build-infra/src/main/groovy/lucene.documentation.gradle deleted file mode 100644 index 4839339b31f2..000000000000 --- a/build-tools/build-infra/src/main/groovy/lucene.documentation.gradle +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension - -if (project != project.rootProject) { - throw new GradleException("Applicable to rootProject only: " + project.path) -} - -configure(rootProject) { - LuceneBuildGlobalsExtension buildGlobals = rootProject.extensions.getByType(LuceneBuildGlobalsExtension) - def urlVersion = buildGlobals.baseVersion.replace('.', '_') - - Provider luceneJavadocUrl = buildOptions.addOption("lucene.javadoc.url", - "External Javadoc URL for documentation generator.", provider { - if (buildGlobals.snapshotBuild) { - // non-release build does not cross-link between modules. - return null - } else { - // release build - return "https://lucene.apache.org/core/${urlVersion}".toString() - } - }) - - def documentationTask = tasks.register("documentation", { - group = 'documentation' - description = 'Generate all documentation' - - dependsOn ':lucene:documentation:assemble' - }) - - assemble.dependsOn documentationTask -} - -configure(project(':lucene:documentation')) { - ext { - docroot = layout.buildDirectory.dir("site").get().asFile - - markdownSrc = file("src/markdown") - assets = file("src/assets") - } - - def documentationTask = tasks.register("documentation", { - group = 'documentation' - description = "Generate ${project.name.capitalize()} documentation" - - dependsOn project(":lucene").subprojects.collect { prj -> - prj.tasks.matching { it.name == 'renderSiteJavadoc' } - } - - dependsOn 'changesToHtml', 'copyDocumentationAssets', - 'markdownToHtml', 'createDocumentationIndex' - }) - - // in CI, fully build documentation. this is very costly, way too - // much for a developer workflow, but it validates build machinery, - // and link references from overview.html, that javac won't catch - LuceneBuildGlobalsExtension buildGlobals = rootProject.extensions.getByType(LuceneBuildGlobalsExtension) - if (buildGlobals.isCIBuild) { - tasks.named("check").configure { - dependsOn 'documentation' - } - } - - tasks.register("copyDocumentationAssets", Copy, { - includeEmptyDirs = false - from project.ext.assets - into project.ext.docroot - }) - - assemble { - dependsOn documentationTask - } - - configurations { - site - } - - artifacts { - site project.ext.docroot, { - builtBy documentationTask - } - } -} - -configure(project(":lucene")) { - ext { - docroot = project('documentation').docroot - } -} diff --git a/build-tools/build-infra/src/main/groovy/lucene.documentation.markdown.gradle b/build-tools/build-infra/src/main/groovy/lucene.documentation.markdown.gradle deleted file mode 100644 index 7ecac2a28196..000000000000 --- a/build-tools/build-infra/src/main/groovy/lucene.documentation.markdown.gradle +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import com.vladsch.flexmark.ast.Heading; -import com.vladsch.flexmark.ext.abbreviation.AbbreviationExtension; -import com.vladsch.flexmark.ext.attributes.AttributesExtension; -import com.vladsch.flexmark.ext.autolink.AutolinkExtension; -import com.vladsch.flexmark.ext.tables.TablesExtension; -import com.vladsch.flexmark.html.HtmlRenderer; -import com.vladsch.flexmark.parser.Parser; -import com.vladsch.flexmark.parser.ParserEmulationProfile; -import com.vladsch.flexmark.util.ast.Document; -import com.vladsch.flexmark.util.data.MutableDataSet; -import com.vladsch.flexmark.util.sequence.Escaping; -import groovy.text.SimpleTemplateEngine -import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension - -configure(project(':lucene:documentation')) { - tasks.register("markdownToHtml", Copy, { - dependsOn "copyDocumentationAssets" - - from(project.parent.projectDir) { - include 'MIGRATE.md' - include 'JRE_VERSION_MIGRATION.md' - include 'SYSTEM_REQUIREMENTS.md' - } - - filteringCharset = 'UTF-8' - includeEmptyDirs = false - - rename(/\.md$/, '.html') - filter(MarkdownFilter) - - into project.ext.docroot - }) - - tasks.register("createDocumentationIndex", MarkdownTemplateTask, { - dependsOn "markdownToHtml" - - outputFile = file("${project.ext.docroot}/index.html") - templateFile = file("${project.ext.markdownSrc}/index.template.md") - - def defaultCodecFile = project(':lucene:core').file('src/java/org/apache/lucene/codecs/Codec.java') - inputs.file(defaultCodecFile) - - // list all properties used by the template here to allow uptodate checks to be correct: - inputs.property('version', project.version) - - def buildGlobals = project.extensions.getByType(LuceneBuildGlobalsExtension) - - binding.put("project", [ - "version": project.version, - "majorVersion": buildGlobals.majorVersion - ]) - - binding.put('defaultCodecPackage', providers.provider { - // static Codec defaultCodec = LOADER . lookup ( "LuceneXXX" ) ; - def regex = ~/\bdefaultCodec\s*=\s*LOADER\s*\.\s*lookup\s*\(\s*"([^"]+)"\s*\)\s*;/ - def matcher = regex.matcher(defaultCodecFile.getText('UTF-8')) - if (!matcher.find()) { - throw new GradleException("Cannot determine default codec from file ${defaultCodecFile}") - } - return matcher.group(1).toLowerCase(Locale.ROOT) - }) - - withProjectList() - }) -} - -// filter that can be used with the "copy" task of Gradle that transforms Markdown files -// from source location to HTML (adding HTML header, styling,...) -class MarkdownFilter extends FilterReader { - - public MarkdownFilter(Reader reader) throws IOException { - // this is not really a filter: it reads whole file in ctor, - // converts it and provides result downstream as a StringReader - super(new StringReader(convert(reader.text))); - } - - public static String convert(String markdownSource) { - // first replace LUCENE and SOLR issue numbers with a markdown link - markdownSource = markdownSource.replaceAll(/(?s)\b(LUCENE|SOLR)\-\d+\b/, - '[$0](https://issues.apache.org/jira/browse/$0)'); - markdownSource = markdownSource.replaceAll(/(?s)\b(GITHUB#|GH-)(\d+)\b/, - '[$0](https://github.com/apache/lucene/issues/$2)'); - - // convert the markdown - MutableDataSet options = new MutableDataSet(); - options.setFrom(ParserEmulationProfile.MARKDOWN); - options.set(Parser.EXTENSIONS, [ - AbbreviationExtension.create(), - AutolinkExtension.create(), - AttributesExtension.create(), - TablesExtension.create(), - ]); - options.set(HtmlRenderer.RENDER_HEADER_ID, true); - options.set(HtmlRenderer.MAX_TRAILING_BLANK_LINES, 0); - Document parsed = Parser.builder(options).build().parse(markdownSource); - - StringBuilder html = new StringBuilder('\n\n'); - CharSequence title = parsed.getFirstChildAny(Heading.class)?.getText(); - if (title != null) { - html.append('').append(Escaping.escapeHtml(title, false)).append('\n'); - } - html.append('\n') - .append('\n\n'); - HtmlRenderer.builder(options).build().render(parsed, html); - html.append('\n\n'); - return html; - } -} - -// Applies a binding of variables using a template and -// produces Markdown, which is converted to HTML -class MarkdownTemplateTask extends DefaultTask { - - @Internal - Project productProject = project.parent - - @InputFile - File templateFile - - @OutputFile - File outputFile - - @Input - @Optional - final MapProperty binding = project.objects.mapProperty(String, Object) - - /** adds a property "projectList" containing all subprojects with javadocs as markdown bullet list */ - void withProjectList() { - binding.put('projectList', project.providers.provider{ - def projects = productProject.subprojects.findAll{ it.tasks.findByName('renderSiteJavadoc')?.enabled } - .sort(false, Comparator.comparing{ - (it.name != 'core') as Boolean - } - .thenComparing(Comparator.comparing{ (it.name != 'solrj') as Boolean }) - .thenComparing(Comparator.comparing{ (it.name == 'test-framework') as Boolean }) - .thenComparing(Comparator.comparing{ it.path })); - return projects.collect{ project -> - def text = "**[${project.relativeDocPath.replace('/','-')}](${project.relativeDocPath}/index.html):** ${project.description}" - if (project.name == 'core') { - text = text.concat(' {style="font-size:larger; margin-bottom:.5em"}') - } - return '* ' + text; - }.join('\n') - }) - } - - @TaskAction - void transform() { - def engine = new SimpleTemplateEngine(); - def resolvedBinding = new HashMap(binding.get()) - String markdown = templateFile.withReader('UTF-8') { - engine.createTemplate(it).make(resolvedBinding).toString(); - } - outputFile.getParentFile().mkdirs(); - outputFile.write(MarkdownFilter.convert(markdown), 'UTF-8'); - } -} diff --git a/build-tools/build-infra/src/main/groovy/lucene.documentation.render-javadoc.gradle b/build-tools/build-infra/src/main/groovy/lucene.documentation.render-javadoc.gradle deleted file mode 100644 index 1d4d5c517f0c..000000000000 --- a/build-tools/build-infra/src/main/groovy/lucene.documentation.render-javadoc.gradle +++ /dev/null @@ -1,558 +0,0 @@ -import org.gradle.internal.jvm.Jvm -import org.apache.lucene.gradle.plugins.java.RenderJavadocTaskBase -import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// generate javadocs by calling javadoc tool -// see https://docs.oracle.com/en/java/javase/11/tools/javadoc.html - -def resources = rootProject.file("gradle/documentation/render-javadoc") - -allprojects { - plugins.withType(JavaPlugin).configureEach { - configurations { - missingdoclet - } - - dependencies { - missingdoclet project(":build-tools:missing-doclet") - } - - project.ext { - relativeDocPath = project.path.replaceFirst(/:\w+:/, "").replace(':', '/') - } - - JavaVersion minJavaVersion = buildGlobals.minJavaVersion.get() - - def renderJavadoc = tasks.register("renderJavadoc", RenderJavadocTask, { - description = "Generates Javadoc API documentation for each module. This directly invokes javadoc tool." - group = "documentation" - - taskResources = resources - dependsOn sourceSets.main.compileClasspath - classpath = sourceSets.main.compileClasspath - srcDirSet = sourceSets.main.java - releaseVersion = minJavaVersion - - outputDir = project.tasks.named("javadoc").get().destinationDir - }) - - // We disable the default javadoc task and have our own - // javadoc rendering task below. The default javadoc task - // will just invoke 'renderJavadoc' (to allow people to call - // conventional task name). - tasks.named("javadoc").configure { - enabled = false - dependsOn renderJavadoc - } - - if (project.path == ':lucene:luke' || !(project in rootProject.ext.mavenProjects)) { - // These projects are not part of the public API so we don't render their javadocs - // as part of the site's creation. Linting happens via javac - } else { - tasks.register("renderSiteJavadoc", RenderJavadocTask, { - description = "Generates Javadoc API documentation for the site (relative links)." - group = "documentation" - - taskResources = resources - dependsOn sourceSets.main.compileClasspath - classpath = sourceSets.main.compileClasspath - srcDirSet = sourceSets.main.java - releaseVersion = minJavaVersion - - relativeProjectLinks = true - - enableSearch = true - - // Place the documentation under the documentation directory. - // docroot is defined in 'documentation.gradle' - outputDir = project.docroot.toPath().resolve(project.ext.relativeDocPath).toFile() - }) - } - } -} - -// Set up titles and link up some offline docs for all documentation -// (they may be unused but this doesn't do any harm). - -def minJava = deps.versions.minJava.get() -def javaJavadocPackages = rootProject.file("${resources}/java-${minJava}/") -if (!javaJavadocPackages.exists()) { - throw new GradleException("Prefetched javadoc element-list is missing at " + javaJavadocPackages + ", " + - "create this directory and fetch the element-list file from " + - "from https://docs.oracle.com/en/java/javase/${minJava}/docs/api/element-list") -} - -def junitVersion = deps.versions.junit.get() -def junitJavadocPackages = rootProject.file("${resources}/junit-${junitVersion}/") -if (!junitJavadocPackages.exists()) { - throw new GradleException("Prefetched javadoc package-list is missing at " + junitJavadocPackages + ", " + - "create this directory and fetch the package-list file from " + - "from https://junit.org/junit4/javadoc/${junitVersion}/package-list") -} - -allprojects { - project.tasks.withType(RenderJavadocTask).configureEach { - title = "Lucene ${project.version} ${project.name} API" - - offlineLinks += [ - ("https://docs.oracle.com/en/java/javase/${minJava}/docs/api/".toString()): javaJavadocPackages, - ("https://junit.org/junit4/javadoc/${junitVersion}/".toString()): junitJavadocPackages - ] - - luceneDocUrl = project.rootProject.buildOptions.getOption('lucene.javadoc.url').asStringProvider() - - // Set up custom doclet. - dependsOn configurations.missingdoclet - docletpath = configurations.missingdoclet - } -} - -// Configure project-specific tweaks and to-dos. -configure(project(":lucene:analysis:common")) { - project.tasks.withType(RenderJavadocTask).configureEach { - // TODO: fix missing javadocs - javadocMissingLevel = "class" - } -} - -configure([ - project(":lucene:analysis:kuromoji"), - project(":lucene:analysis:nori"), - project(":lucene:analysis:opennlp"), - project(":lucene:analysis:smartcn"), - project(":lucene:benchmark"), - project(":lucene:codecs"), - project(":lucene:grouping"), - project(":lucene:highlighter"), - project(":lucene:luke"), - project(":lucene:monitor"), - project(":lucene:queries"), - project(":lucene:queryparser"), - project(":lucene:replicator"), - project(":lucene:spatial-extras"), -]) { - project.tasks.withType(RenderJavadocTask).configureEach { - // TODO: fix missing javadocs - javadocMissingLevel = "class" - } -} - -configure([ - project(":lucene:analysis:icu"), - project(":lucene:analysis:morfologik"), - project(":lucene:analysis:phonetic"), - project(":lucene:analysis:stempel"), - project(":lucene:classification"), - project(":lucene:demo"), - project(":lucene:expressions"), - project(":lucene:facet"), - project(":lucene:join"), - project(":lucene:spatial3d"), - project(":lucene:suggest"), -]) { - project.tasks.withType(RenderJavadocTask).configureEach { - // TODO: fix missing @param tags - javadocMissingLevel = "method" - } -} - -configure(project(":lucene:backward-codecs")) { - project.tasks.withType(RenderJavadocTask).configureEach { - // TODO: fix missing @param tags - javadocMissingLevel = "method" - } -} - -configure(project(":lucene:test-framework")) { - project.tasks.withType(RenderJavadocTask).configureEach { - // TODO: fix missing javadocs - javadocMissingLevel = "class" - } -} - -configure(project(":lucene:sandbox")) { - project.tasks.withType(RenderJavadocTask).configureEach { - // TODO: fix missing javadocs - javadocMissingLevel = "class" - } -} - -configure(project(":lucene:spatial-test-fixtures")) { - project.tasks.withType(RenderJavadocTask).configureEach { - // TODO: fix missing javadocs - javadocMissingLevel = "class" - } -} - -configure(project(":lucene:misc")) { - project.tasks.withType(RenderJavadocTask).configureEach { - // TODO: fix missing javadocs - javadocMissingLevel = "class" - } -} - -configure(project(":lucene:core")) { - project.tasks.withType(RenderJavadocTask).configureEach { - // TODO: fix missing javadocs - javadocMissingLevel = "class" - // some packages are fixed already - javadocMissingMethod = [ - "org.apache.lucene.util.automaton", - "org.apache.lucene.analysis.standard", - "org.apache.lucene.analysis.tokenattributes", - "org.apache.lucene.document", - "org.apache.lucene.search.similarities", - "org.apache.lucene.index", - "org.apache.lucene.codecs", - "org.apache.lucene.codecs.lucene50", - "org.apache.lucene.codecs.lucene60", - "org.apache.lucene.codecs.lucene80", - "org.apache.lucene.codecs.lucene84", - "org.apache.lucene.codecs.lucene86", - "org.apache.lucene.codecs.lucene87", - "org.apache.lucene.codecs.perfield" - ] - } -} - -configure(project(':lucene:demo')) { - project.tasks.withType(RenderJavadocTask).configureEach { - // For the demo, we link the example source in the javadocs, as it's ref'ed elsewhere - linksource = true - } -} - -// Add cross-project documentation task dependencies: -// - each RenderJavaDocs task gets a dependency to all tasks with the same name in its dependencies -// - the dependency is using dependsOn with a closure to enable lazy evaluation -configure(subprojects) { - project.tasks.withType(RenderJavadocTask).configureEach { task -> - task.dependsOn { - task.project.configurations.implementation.allDependencies.withType(ProjectDependency).collect { dep -> - return dep.path + ":" + task.name - } - } - } -} - -class OfflineLink implements Serializable { - @Input - String url - - @InputDirectory - @PathSensitive(PathSensitivity.RELATIVE) - @IgnoreEmptyDirectories - File location - - OfflineLink(String url, File location) { - this.url = url - this.location = location - } -} - -@CacheableTask -abstract class RenderJavadocTask extends RenderJavadocTaskBase { - @InputFiles - @PathSensitive(PathSensitivity.RELATIVE) - @IgnoreEmptyDirectories - @SkipWhenEmpty - SourceDirectorySet srcDirSet; - - @OutputDirectory - File outputDir - - @CompileClasspath - FileCollection classpath - - @CompileClasspath - FileCollection docletpath - - @Input - String title - - @Input - boolean linksource = false - - @Input - boolean enableSearch = false - - @Input - boolean relativeProjectLinks = false - - @Input - JavaVersion releaseVersion - - @Internal - Map offlineLinks = [:] - - // Computes cacheable inputs from the map in offlineLinks. - @Nested - List getCacheableOfflineLinks() { - return offlineLinks.collect { url, location -> new OfflineLink(url, location) } - } - - @Input - @Optional - final Property luceneDocUrl = project.objects.property(String) - - // default is to require full javadocs - @Input - String javadocMissingLevel = "parameter" - - // anything in these packages is checked with level=method. This allows iteratively fixing one package at a time. - @Input - List javadocMissingMethod = [] - - // default is not to ignore any elements, should only be used to workaround split packages - @Input - List javadocMissingIgnore = [] - - @InputDirectory - @PathSensitive(PathSensitivity.RELATIVE) - @IgnoreEmptyDirectories - File taskResources - - /** Utility method to recursively collect all tasks with same name like this one that we depend on */ - private Set findRenderTasksInDependencies() { - Set found = [] - def collectDeps - collectDeps = { task -> - project.gradle.taskGraph.getDependencies(task).findAll{ it.name == this.name && it.enabled && !found.contains(it) }.each{ - found << it - collectDeps(it) - } - } - collectDeps(this) - return found - } - - @TaskAction - public void render() { - def srcDirs = srcDirSet.sourceDirectories.filter { dir -> dir.exists() } - - def optionsFile = project.file("${getTemporaryDir()}/javadoc-options.txt") - - // create the directory, so relative link calculation knows that it's a directory: - outputDir.mkdirs(); - - def opts = [] - - def overviewFiles = srcDirs - .collect { dir -> project.file("${dir}/overview.html") } - .findAll { overviewFile -> overviewFile.exists() } - - assert overviewFiles.size() == 1 : "Must be exactly one overview.html file: " + overviewFiles - opts << [ - '-overview', - project.file(overviewFiles.getFirst()) - ] - - opts << ['-d', outputDir] - opts << '-protected' - opts << ['-encoding', 'UTF-8'] - opts << ['-charset', 'UTF-8'] - opts << ['-docencoding', 'UTF-8'] - if (!enableSearch) { - opts << '-noindex' - } - opts << '-author' - opts << '-version' - if (linksource) { - opts << '-linksource' - } - opts << '-use' - opts << ['-locale', 'en_US'] - opts << ['-windowtitle', title] - opts << ['-doctitle', title] - if (!classpath.isEmpty()) { - opts << [ - '-classpath', - classpath.asPath - ] - } - - LuceneBuildGlobalsExtension buildGlobals = project.rootProject.extensions.getByType(LuceneBuildGlobalsExtension) - opts << [ - '-bottom', - "Copyright © 2000-${buildGlobals.buildYear} Apache Software Foundation. All Rights Reserved." - ] - - opts << [ - '-tag', - 'lucene.experimental:a:WARNING: This API is experimental and might change in incompatible ways in the next release.' - ] - opts << [ - '-tag', - 'lucene.internal:a:NOTE: This API is for internal purposes only and might change in incompatible ways in the next release.' - ] - opts << [ - '-tag', - "lucene.spi:t:SPI Name (case-insensitive: if the name is 'htmlStrip', 'htmlstrip' can be used when looking up the service)." - ] - - opts << [ - '-doclet', - "org.apache.lucene.missingdoclet.MissingDoclet" - ] - opts << [ - '-docletpath', - docletpath.asPath - ] - opts << [ - '--missing-level', - javadocMissingLevel - ] - if (javadocMissingIgnore) { - opts << [ - '--missing-ignore', - String.join(',', javadocMissingIgnore) - ] - } - if (javadocMissingMethod) { - opts << [ - '--missing-method', - String.join(',', javadocMissingMethod) - ] - } - - opts << ['-quiet'] - - // Add all extra options, if any. - opts.addAll(extraOpts.orElse([]).get()) - - def allOfflineLinks = [:] - allOfflineLinks.putAll(offlineLinks) - - // Resolve inter-project links: - // - find all (enabled) tasks this tasks depends on (with same name), calling findRenderTasksInDependencies() - // - sort the tasks preferring those whose project name equals 'core', then lexigraphical by path - // - for each task get output dir to create relative or absolute link - findRenderTasksInDependencies() - .sort(false, Comparator.comparing { (it.project.name != 'core') as Boolean }.thenComparing(Comparator.comparing { it.path })) - .each { otherTask -> - def otherProject = otherTask.project - // For relative links we compute the actual relative link between projects. - if (relativeProjectLinks) { - def pathTo = otherTask.outputDir.toPath().toAbsolutePath() - def pathFrom = outputDir.toPath().toAbsolutePath() - def relative = pathFrom.relativize(pathTo).toString().replace(File.separator, '/') - opts << ['-link', relative] - } else { - // For absolute links, we determine the target URL by assembling the full URL (if base is available). - def value = luceneDocUrl.getOrElse(null) - if (value) { - allOfflineLinks.put("${value}/${otherProject.relativeDocPath}/".toString(), otherTask.outputDir) - } - } - } - - // Add offline links. - allOfflineLinks.each { url, dir -> - // Some sanity check/ validation here to ensure dir/package-list or dir/element-list is present. - if (!project.file("$dir/package-list").exists() && - !project.file("$dir/element-list").exists()) { - throw new GradleException("Expected pre-rendered package-list or element-list at ${dir}.") - } - logger.info("Linking ${url} to ${dir}") - opts << ['-linkoffline', url, dir] - } - - opts << [ - '--release', - releaseVersion.toString() - ] - opts << '-Xdoclint:all,-missing' - - // Increase Javadoc's heap. - opts += ["-J-Xmx512m"] - // Force locale to be "en_US" (fix for: https://bugs.openjdk.java.net/browse/JDK-8222793) - opts += [ - "-J-Duser.language=en", - "-J-Duser.country=US" - ] - - // -J options have to be passed on command line, they are not interpreted if passed via args file. - def jOpts = opts.findAll { opt -> opt instanceof String && opt.startsWith("-J") } - opts.removeAll(jOpts) - - // Collect all source files, for now excluding module descriptors. - opts.addAll( - srcDirs.collectMany { dir -> - project.fileTree(dir: dir, include: "**/*.java", exclude: "**/module-info.java").files - }.collect { - it.toString() - } - ) - - // handle doc-files manually since in explicit source file mode javadoc does not copy them. - srcDirs.each { File dir -> - project.copy { - into outputDir - - from(dir, { - include "**/doc-files/**" - }) - } - } - - // Temporary file that holds all javadoc options for the current task (except jOpts) - optionsFile.withWriter("UTF-8", { writer -> - // escapes an option with single quotes or whitespace to be passed in the options.txt file for - def escapeJavadocOption = { String s -> (s =~ /[ '"]/) ? ("'" + s.replaceAll(/[\\'"]/, /\\$0/) + "'") : s } - - opts.each { entry -> - if (entry instanceof List) { - writer.write(entry.collect { escapeJavadocOption(it as String) }.join(" ")) - } else { - writer.write(escapeJavadocOption(entry as String)) - } - writer.write('\n') - } - }) - - def javadocCmd = project.file(executable.get()) - logger.info("Javadoc executable used: ${javadocCmd}") - - buildGlobals.quietExec(this, { - executable javadocCmd - - args += ["@${optionsFile}"] - args += jOpts - }) - - // append some special table css, prettify css - ant.concat(destfile: "${outputDir}/stylesheet.css", append: "true", fixlastline: "true", encoding: "UTF-8") { - filelist(dir: taskResources, files: - [ - "table_padding.css", - "custom_styles.css", - "prettify/prettify.css" - ].join(" ") - ) - } - - // append prettify to scripts - ant.concat(destfile: "${outputDir}/script.js", append: "true", fixlastline: "true", encoding: "UTF-8") { - filelist(dir: project.file("${taskResources}/prettify"), files: "prettify.js inject-javadocs.js") - } - - ant.fixcrlf(srcdir: outputDir, includes: "stylesheet.css script.js", eol: "lf", fixlast: "true", encoding: "UTF-8") - } -} diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/documentation/ChangesToHtmlTask.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/documentation/ChangesToHtmlTask.java index 2be8ff417a50..cb246d7eb2f8 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/documentation/ChangesToHtmlTask.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/documentation/ChangesToHtmlTask.java @@ -40,6 +40,7 @@ import org.gradle.process.ExecOperations; import org.gradle.process.ExecResult; +/** Convert {@code CHANGES.txt} into html using plenty of perl hackery. */ public abstract class ChangesToHtmlTask extends DefaultTask { @Input public abstract Property getProductName(); diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/documentation/DocumentationConfigPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/documentation/DocumentationConfigPlugin.java new file mode 100644 index 000000000000..b7dfe6dcda8e --- /dev/null +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/documentation/DocumentationConfigPlugin.java @@ -0,0 +1,393 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.gradle.plugins.documentation; + +import com.vladsch.flexmark.ast.Heading; +import com.vladsch.flexmark.ext.abbreviation.AbbreviationExtension; +import com.vladsch.flexmark.ext.attributes.AttributesExtension; +import com.vladsch.flexmark.ext.autolink.AutolinkExtension; +import com.vladsch.flexmark.ext.tables.TablesExtension; +import com.vladsch.flexmark.html.HtmlRenderer; +import com.vladsch.flexmark.parser.Parser; +import com.vladsch.flexmark.parser.ParserEmulationProfile; +import com.vladsch.flexmark.util.ast.Document; +import com.vladsch.flexmark.util.data.MutableDataSet; +import com.vladsch.flexmark.util.sequence.Escaping; +import groovy.text.SimpleTemplateEngine; +import java.io.File; +import java.io.FilterReader; +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.nio.file.Files; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.apache.lucene.gradle.plugins.LuceneGradlePlugin; +import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension; +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleException; +import org.gradle.api.Project; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.provider.MapProperty; +import org.gradle.api.tasks.Copy; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.Optional; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.TaskContainer; + +/** Configures documentation generation and other documentation-related aspects. */ +public class DocumentationConfigPlugin extends LuceneGradlePlugin { + public static final String OPT_JAVADOC_URL = "lucene.javadoc.url"; + + @Override + public void apply(Project project) { + applicableToRootProjectOnly(project); + + LuceneBuildGlobalsExtension buildGlobals = getLuceneBuildGlobals(project); + + getBuildOptions(project) + .addOption( + OPT_JAVADOC_URL, + "External Javadoc URL for documentation generator.", + project + .getProviders() + .provider( + () -> { + if (buildGlobals.snapshotBuild) { + // non-release build does not cross-link between modules. + return null; + } else { + // release build + var urlVersion = buildGlobals.baseVersion.replace('.', '_'); + return "https://lucene.apache.org/core/" + urlVersion; + } + })); + + TaskContainer tasks = project.getTasks(); + var documentationTask = + tasks.register( + "documentation", + task -> { + task.setGroup("documentation"); + task.setDescription("Generate all documentation"); + + task.dependsOn(":lucene:documentation:assemble"); + }); + + tasks.named("assemble").configure(task -> task.dependsOn(documentationTask)); + + project.configure( + List.of(project.project(":lucene:documentation")), this::configureDocumentationProject); + } + + private void configureDocumentationProject(Project project) { + File docroot = getDocumentationRoot(project); + File markdownSrc = project.file("src/markdown"); + File assets = project.file("src/assets"); + + var tasks = project.getTasks(); + + configureMarkdownConversion(markdownSrc, docroot, project); + + var copyDocumentationAssets = + tasks.register( + "copyDocumentationAssets", + Copy.class, + task -> { + task.setIncludeEmptyDirs(false); + task.from(assets); + task.into(docroot); + }); + + var documentationTask = + tasks.register( + "documentation", + task -> { + task.setGroup("documentation"); + task.setDescription("Generate Lucene documentation"); + + task.dependsOn( + project + .getProviders() + .provider( + () -> + project.project(":lucene").getSubprojects().stream() + .flatMap( + prj -> + prj + .getTasks() + .matching( + t -> t.getName().equals("renderSiteJavadoc")) + .stream()) + .toList())); + + task.dependsOn( + copyDocumentationAssets, + "changesToHtml", + "markdownToHtml", + "createDocumentationIndex"); + }); + + // Limit building the full documentation to CI runs only. + // This is very costly so we only validate the build machinery there. + var buildGlobals = getLuceneBuildGlobals(project); + if (buildGlobals.isCIBuild) { + tasks.named("check").configure(task -> task.dependsOn(documentationTask)); + } + + // assemble always builds the docs. + tasks.named("assemble").configure(task -> task.dependsOn(documentationTask)); + + // Expose the documentation as an artifact. + String confName = "site"; + project.getConfigurations().create(confName); + project + .getArtifacts() + .add( + confName, + docroot, + configurablePublishArtifact -> { + configurablePublishArtifact.builtBy(documentationTask); + }); + } + + public static File getDocumentationRoot(Project project) { + return project + .project(":lucene:documentation") + .getLayout() + .getBuildDirectory() + .dir("site") + .get() + .getAsFile(); + } + + private void configureMarkdownConversion(File markdownSrc, File docroot, Project project) { + TaskContainer tasks = project.getTasks(); + var markdownToHtml = + tasks.register( + "markdownToHtml", + Copy.class, + task -> { + task.dependsOn("copyDocumentationAssets"); + + task.from( + project.project(":lucene").getLayout().getProjectDirectory().getAsFile(), + copySpec -> { + copySpec.include("MIGRATE.md"); + copySpec.include("JRE_VERSION_MIGRATION.md"); + copySpec.include("SYSTEM_REQUIREMENTS.md"); + }); + + task.setFilteringCharset("UTF-8"); + task.setIncludeEmptyDirs(false); + + task.rename(Pattern.compile("\\.md$"), ".html"); + task.filter(MarkdownFilterImpl.class); + + task.into(docroot); + }); + + tasks.register( + "createDocumentationIndex", + MarkdownTemplateTask.class, + task -> { + task.dependsOn(markdownToHtml); + + task.getOutputFile().set(docroot.toPath().resolve("index.html").toFile()); + task.getTemplateFile().set(markdownSrc.toPath().resolve("index.template.md").toFile()); + + // list all properties used by the template here to allow uptodate checks to be correct: + task.getInputs().property("version", project.getVersion()); + + var defaultCodecFile = + project.project(":lucene:core").file("src/java/org/apache/lucene/codecs/Codec.java"); + task.getInputs().file(defaultCodecFile); + + var buildGlobals = getLuceneBuildGlobals(project); + task.getBinding() + .put( + "project", + Map.ofEntries( + Map.entry("version", project.getVersion().toString()), + Map.entry("majorVersion", buildGlobals.majorVersion))); + + task.getBinding() + .put( + "defaultCodecPackage", + project + .getProviders() + .provider( + () -> { + var regex = + Pattern.compile( + "Codec defaultCodec = LOADER\\.lookup\\((?\"([^\"]+)\")\\);"); + var matcher = + regex.matcher(Files.readString(defaultCodecFile.toPath())); + if (!matcher.find()) { + throw new GradleException( + "Cannot determine default codec from file: " + defaultCodecFile); + } + return matcher.group("codec").toLowerCase(Locale.ROOT); + })); + + task.withProjectList(); + }); + } + + /** + * A filter that can be used with gradle's "copy" task to transforms Markdown files into HTML + * (adding HTML header, styling,...). + */ + public static final class MarkdownFilterImpl extends FilterReader { + public MarkdownFilterImpl(Reader reader) throws IOException { + // this is not really a filter: it reads the whole file in ctor, + // converts it and provides result downstream as a StringReader + super(new StringReader(convert(reader.readAllAsString()))); + } + + public static String convert(String markdownSource) { + // replace LUCENE or SOLR issue numbers with a markdown link + markdownSource = + markdownSource.replaceAll( + "(?s)\\b(LUCENE|SOLR)-\\d+\\b", "[$0](https://issues.apache.org/jira/browse/$0)"); + // then follow with github issues/ pull requests. + markdownSource = + markdownSource.replaceAll( + "(?s)\\b(GITHUB#|GH-)(\\d+)\\b", "[$0](https://github.com/apache/lucene/issues/$2)"); + + // convert the markdown + MutableDataSet options = new MutableDataSet(); + options.setFrom(ParserEmulationProfile.MARKDOWN); + options.set( + Parser.EXTENSIONS, + List.of( + AbbreviationExtension.create(), + AutolinkExtension.create(), + AttributesExtension.create(), + TablesExtension.create())); + + options.set(HtmlRenderer.RENDER_HEADER_ID, true); + options.set(HtmlRenderer.MAX_TRAILING_BLANK_LINES, 0); + Document parsed = Parser.builder(options).build().parse(markdownSource); + + StringBuilder html = new StringBuilder("\n\n"); + var headingNode = parsed.getFirstChildAny(Heading.class); + if (headingNode != null) { + var title = ((Heading) headingNode).getText(); + html.append("").append(Escaping.escapeHtml(title, false)).append("\n"); + } + html.append( + """ + + + + """); + HtmlRenderer.builder(options).build().render(parsed, html); + html.append("\n\n"); + return html.toString(); + } + } + + /** + * Applies a binding of variables using a template and produces Markdown, which is converted to + * HTML. + */ + public abstract static class MarkdownTemplateTask extends DefaultTask { + @InputFile + public abstract RegularFileProperty getTemplateFile(); + + @OutputFile + public abstract RegularFileProperty getOutputFile(); + + @Input + @Optional + public abstract MapProperty getBinding(); + + @TaskAction + public void transform() throws IOException, ClassNotFoundException { + var engine = new SimpleTemplateEngine(); + HashMap resolvedBinding = new HashMap<>(getBinding().get()); + String markdown = + engine + .createTemplate(Files.readString(getTemplateFile().get().getAsFile().toPath())) + .make(resolvedBinding) + .toString(); + + var outputPath = getOutputFile().get().getAsFile().toPath(); + Files.createDirectories(outputPath.getParent()); + Files.writeString(outputPath, MarkdownFilterImpl.convert(markdown)); + } + + /** + * adds a property "projectList" containing all subprojects with javadocs as markdown bullet + * list + */ + public void withProjectList() { + var projectListProvider = + getProject() + .getProviders() + .provider( + () -> { + var projectList = + getProject().project(":lucene").getSubprojects().stream() + .filter( + p -> + p + .getTasks() + .matching(t -> t.getName().equals("renderSiteJavadoc")) + .stream() + .findAny() + .isPresent()) + .sorted( + Comparator.comparing((Project t) -> !t.getName().equals("core")) + .thenComparing(t -> t.getName().equals("test-framework")) + .thenComparing(Project::getPath)) + .toList(); + + return projectList.stream() + .map( + project -> { + var text = + String.format( + Locale.ROOT, + "**[%s](%s/index.html):** %s", + relativeDocPath(project).replace('/', '-'), + relativeDocPath(project), + project.getDescription()); + if (project.getName().equals("core")) { + text = text + " {style='font-size:larger; margin-bottom:.5em'}"; + } + return "* " + text; + }) + .collect(Collectors.joining("\n")); + }); + + getBinding().put("projectList", projectListProvider); + } + } + + public static String relativeDocPath(Project p) { + return p.getPath().replaceFirst(":\\w+:", "").replace(':', '/'); + } +} diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/globals/LuceneBuildGlobalsExtension.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/globals/LuceneBuildGlobalsExtension.java index 7d227b315ce3..c20845167bc2 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/globals/LuceneBuildGlobalsExtension.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/globals/LuceneBuildGlobalsExtension.java @@ -23,6 +23,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import javax.inject.Inject; import org.apache.lucene.gradle.plugins.misc.QuietExec; @@ -71,6 +72,11 @@ public abstract class LuceneBuildGlobalsExtension { */ public boolean isCIBuild; + /** + * @see #getPublishedProjects() + */ + Set publishedProjects; + /** Returns per-project seed for randomization. */ public abstract Property getProjectSeedAsLong(); @@ -86,6 +92,14 @@ public abstract class LuceneBuildGlobalsExtension { /** If set, returns certain flags helpful for configuring the build for the intellij idea IDE. */ public abstract Property getIntellijIdea(); + /** + * Return a set of "maven-published" projects (those that are published to sonatype/maven + * central). + */ + public Set getPublishedProjects() { + return publishedProjects; + } + /** * Returns the path to the provided named external tool. Developers may set up different tool * paths using local build options. diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/globals/RegisterBuildGlobalsPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/globals/RegisterBuildGlobalsPlugin.java index b89745638cda..2b8c655ef210 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/globals/RegisterBuildGlobalsPlugin.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/globals/RegisterBuildGlobalsPlugin.java @@ -21,10 +21,13 @@ import com.carrotsearch.randomizedtesting.SeedUtils; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.Comparator; +import java.util.LinkedHashSet; import java.util.Locale; import java.util.Random; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.apache.lucene.gradle.plugins.LuceneGradlePlugin; import org.gradle.api.GradleException; import org.gradle.api.JavaVersion; @@ -84,6 +87,38 @@ public void apply(Project project) { boolean isIdeaSync = Boolean.parseBoolean(System.getProperty("idea.sync.active", "false")); boolean isIdeaBuild = (isIdea && !isIdeaSync); + // Determine the set of projects whose artifacts are published to maven central. + LinkedHashSet publishedProjects = + project.project(":lucene").getSubprojects().stream() + .filter( + subproject -> { + // Exclude all modular-test projects. + if (subproject.getPath().endsWith(".tests")) { + return false; + } + + // Exclude build tools. + if (subproject.getPath().startsWith(":build-tools:")) { + return false; + } + + // Exclude these explicitly. + return switch (subproject.getPath()) { + // Exclude distribution assembly, tests & documentation. + case ":lucene:distribution", ":lucene:documentation" -> false; + // Exclude the parent container project for analysis modules (no artifacts). + case ":lucene:analysis" -> false; + // Exclude other misc modules: native, test and benchmarks. + case ":lucene:misc:native", + ":lucene:spatial-test-fixtures", + ":lucene:benchmarks-jmh" -> + false; + default -> true; + }; + }) + .sorted(Comparator.comparing(Project::getPath)) + .collect(Collectors.toCollection(LinkedHashSet::new)); + project.allprojects( p -> { var globals = @@ -96,6 +131,7 @@ public void apply(Project project) { globals.buildTime = buildTime; globals.buildYear = buildYear; globals.isCIBuild = isCIBuild; + globals.publishedProjects = publishedProjects; globals.getRootSeed().set(rootSeed); globals.getRootSeedAsLong().set(rootSeedLong); globals diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/help/BuildOptionGroupsPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/help/BuildOptionGroupsPlugin.java index 3cc7f72e4b48..83c39b782c03 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/help/BuildOptionGroupsPlugin.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/help/BuildOptionGroupsPlugin.java @@ -20,6 +20,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.lucene.gradle.plugins.documentation.DocumentationConfigPlugin; import org.apache.lucene.gradle.plugins.hacks.DumpGradleStateOnStalledBuildsPlugin; import org.apache.lucene.gradle.plugins.ide.EclipseSupportPlugin; import org.apache.lucene.gradle.plugins.java.ErrorPronePlugin; @@ -77,7 +78,7 @@ public void apply(Project project) { optionGroups.group( "Options useful for release managers", - explicitList("lucene.javadoc.url", "sign", "useGpg")); + explicitList(DocumentationConfigPlugin.OPT_JAVADOC_URL, "sign", "useGpg")); optionGroups.group( "Build control and information", diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/JavaProjectConventionsPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/JavaProjectConventionsPlugin.java index 3b331c8ab845..78b513d590ad 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/JavaProjectConventionsPlugin.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/JavaProjectConventionsPlugin.java @@ -57,5 +57,6 @@ private void applyJavaPlugins(Project project) { plugins.apply(ShowSlowestTestsAtEndPlugin.class); plugins.apply(ShowFailedTestsAtEndPlugin.class); plugins.apply(ErrorPronePlugin.class); + plugins.apply(RenderJavadocPlugin.class); } } diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/RenderJavadocPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/RenderJavadocPlugin.java new file mode 100644 index 000000000000..e500f3be365e --- /dev/null +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/RenderJavadocPlugin.java @@ -0,0 +1,766 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.gradle.plugins.java; + +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import javax.inject.Inject; +import org.apache.lucene.gradle.plugins.LuceneGradlePlugin; +import org.apache.lucene.gradle.plugins.documentation.DocumentationConfigPlugin; +import org.apache.tools.ant.taskdefs.Concat; +import org.apache.tools.ant.taskdefs.FixCRLF; +import org.apache.tools.ant.types.FileList; +import org.gradle.api.GradleException; +import org.gradle.api.JavaVersion; +import org.gradle.api.Project; +import org.gradle.api.Task; +import org.gradle.api.artifacts.ProjectDependency; +import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.Directory; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.file.RegularFile; +import org.gradle.api.file.SourceDirectorySet; +import org.gradle.api.internal.file.FileOperations; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.plugins.JavaPluginExtension; +import org.gradle.api.provider.ListProperty; +import org.gradle.api.provider.MapProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.CacheableTask; +import org.gradle.api.tasks.CompileClasspath; +import org.gradle.api.tasks.IgnoreEmptyDirectories; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.Nested; +import org.gradle.api.tasks.Optional; +import org.gradle.api.tasks.OutputDirectory; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; +import org.gradle.api.tasks.SkipWhenEmpty; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.TaskContainer; +import org.gradle.api.tasks.javadoc.Javadoc; + +/** Configures all projects to manually invoke Javadoc instead of relying on gradle's defaults. */ +public class RenderJavadocPlugin extends LuceneGradlePlugin { + @Override + public void apply(Project project) { + requiresAppliedPlugin(project, JavaPlugin.class); + + var resources = + getProjectRootPath(project).resolve("gradle/documentation/render-javadoc").toFile(); + + var missingdocletConfiguration = project.getConfigurations().create("missingdoclet"); + project.getDependencies().add("missingdoclet", project.project(":build-tools:missing-doclet")); + + TaskContainer tasks = project.getTasks(); + var renderJavadoc = + tasks.register( + "renderJavadoc", + RenderJavadocTask.class, + task -> { + task.setGroup("documentation"); + task.setDescription( + "Generates Javadoc API documentation for each module. This directly invokes javadoc tool."); + + task.getTaskResources().set(resources); + + SourceSet mainSrcSet = + project + .getExtensions() + .getByType(JavaPluginExtension.class) + .getSourceSets() + .getByName("main"); + + var compileCp = mainSrcSet.getCompileClasspath(); + task.dependsOn(compileCp); + task.getClasspath().from(compileCp); + task.getSrcDirSet().set(mainSrcSet.getJava()); + + JavaVersion minJavaVersion = getLuceneBuildGlobals(project).getMinJavaVersion().get(); + task.getReleaseVersion().set(minJavaVersion); + + var javadocOutputDir = + project + .getProviders() + .provider( + () -> + tasks + .withType(Javadoc.class) + .named("javadoc") + .get() + .getDestinationDir()); + task.getOutputDir().set(project.getLayout().dir(javadocOutputDir)); + }); + + // We disable the default javadoc task and have our own + // javadoc rendering task below. The default javadoc task + // will just invoke 'renderJavadoc' (to allow people to call + // conventional task name). + tasks + .named("javadoc") + .configure( + task -> { + task.setEnabled(false); + task.dependsOn(renderJavadoc); + }); + + // Add a rendering task that produces the output suitable for the Lucene site. + Set publishedProjects = getLuceneBuildGlobals(project).getPublishedProjects(); + + if (project.getPath().equals(":lucene:luke") || !(publishedProjects.contains(project))) { + // These projects are not part of the public API so we don't render their javadocs + // as part of the site's creation. + } else { + tasks.register( + "renderSiteJavadoc", + RenderJavadocTask.class, + task -> { + task.setGroup("documentation"); + task.setDescription( + "Generates Javadoc API documentation for the site (relative links)."); + + task.getTaskResources().set(resources); + + { + SourceSet mainSrcSet = + project + .getExtensions() + .getByType(JavaPluginExtension.class) + .getSourceSets() + .getByName("main"); + + var compileCp = mainSrcSet.getCompileClasspath(); + task.dependsOn(compileCp); + task.getClasspath().from(compileCp); + task.getSrcDirSet().set(mainSrcSet.getJava()); + } + + JavaVersion minJavaVersion = getLuceneBuildGlobals(project).getMinJavaVersion().get(); + task.getReleaseVersion().set(minJavaVersion); + + // site-creation specific settings. + task.getRelativeProjectLinks().set(true); + task.getEnableSearch().set(true); + // Place the documentation under the documentation directory. + task.getOutputDir() + .set( + DocumentationConfigPlugin.getDocumentationRoot(project) + .toPath() + .resolve(DocumentationConfigPlugin.relativeDocPath(project)) + .toFile()); + }); + } + + // Set up titles and link up some offline docs for all documentation + // (they may be unused but this doesn't do any harm). + String minJava = getVersionCatalog(project).findVersion("minJava").get().toString(); + Path javaJavadocPackages = resources.toPath().resolve("java-" + minJava + "/"); + if (!Files.exists(javaJavadocPackages)) { + throw new GradleException( + "Prefetched javadoc element-list is missing at " + + javaJavadocPackages + + ", " + + "create this directory and fetch the element-list file from " + + "from https://docs.oracle.com/en/java/javase/" + + minJava + + "/docs/api/element-list"); + } + + String junitVersion = getVersionCatalog(project).findVersion("junit").get().toString(); + Path junitJavadocPackages = resources.toPath().resolve("junit-" + junitVersion + "/"); + if (!Files.exists(junitJavadocPackages)) { + throw new GradleException( + "Prefetched javadoc package-list is missing at " + + junitJavadocPackages + + ", " + + "create this directory and fetch the package-list file from " + + "from https://junit.org/junit4/javadoc/" + + junitVersion + + "/package-list"); + } + + project + .getTasks() + .withType(RenderJavadocTask.class) + .configureEach( + task -> { + task.getTitle() + .set("Lucene " + project.getVersion() + " " + project.getName() + " API"); + + task.getOfflineLinks() + .put( + "https://docs.oracle.com/en/java/javase/" + minJava + "/docs/api/", + javaJavadocPackages.toFile()); + task.getOfflineLinks() + .put( + "https://junit.org/junit4/javadoc/" + junitVersion + "/", + junitJavadocPackages.toFile()); + + task.getLuceneDocUrl() + .set( + getBuildOptions(project.getRootProject()) + .getOption(DocumentationConfigPlugin.OPT_JAVADOC_URL) + .asStringProvider()); + + // Set up custom doclet. + task.dependsOn(missingdocletConfiguration); + task.getDocletpath().from(missingdocletConfiguration); + }); + + // Configure project-specific tweaks and to-dos. + project + .getTasks() + .withType(RenderJavadocTask.class) + .configureEach( + task -> { + // TODO: fix these + switch (task.getProject().getPath()) { + case ":lucene:core": + task.getJavadocMissingLevel().set("class"); + task.getJavadocMissingMethod() + .set( + List.of( + "org.apache.lucene.util.automaton", + "org.apache.lucene.analysis.standard", + "org.apache.lucene.analysis.tokenattributes", + "org.apache.lucene.document", + "org.apache.lucene.search.similarities", + "org.apache.lucene.index", + "org.apache.lucene.codecs", + "org.apache.lucene.codecs.lucene50", + "org.apache.lucene.codecs.lucene60", + "org.apache.lucene.codecs.lucene80", + "org.apache.lucene.codecs.lucene84", + "org.apache.lucene.codecs.lucene86", + "org.apache.lucene.codecs.lucene87", + "org.apache.lucene.codecs.perfield")); + break; + + case ":lucene:analysis:common": + case ":lucene:analysis:kuromoji": + case ":lucene:analysis:nori": + case ":lucene:analysis:opennlp": + case ":lucene:analysis:smartcn": + case ":lucene:benchmark": + case ":lucene:codecs": + case ":lucene:grouping": + case ":lucene:highlighter": + case ":lucene:luke": + case ":lucene:misc": + case ":lucene:monitor": + case ":lucene:queries": + case ":lucene:queryparser": + case ":lucene:replicator": + case ":lucene:sandbox": + case ":lucene:spatial-extras": + case ":lucene:spatial-test-fixtures": + case ":lucene:test-framework": + task.getJavadocMissingLevel().set("class"); + break; + + case ":lucene:analysis:icu": + case ":lucene:analysis:morfologik": + case ":lucene:analysis:phonetic": + case ":lucene:analysis:stempel": + case ":lucene:backward-codecs": + case ":lucene:classification": + case ":lucene:expressions": + case ":lucene:facet": + case ":lucene:join": + case ":lucene:spatial3d": + case ":lucene:suggest": + task.getJavadocMissingLevel().set("method"); + break; + + case ":lucene:demo": + task.getJavadocMissingLevel().set("method"); + // For the demo, we link the example source in the javadocs, as it's ref'ed + // elsewhere + task.getLinksource().set(true); + break; + } + }); + + // Add cross-project documentation task dependencies: + // - each RenderJavaDocs task gets a dependency to all tasks with the same name + // present in this project's dependency configuration 'implementation'. + // - a lazy provider is used to collect these dependencies. + project + .getTasks() + .withType(RenderJavadocTask.class) + .configureEach( + task -> { + task.dependsOn( + project + .getProviders() + .provider( + () -> { + var allDeps = + task.getProject() + .getConfigurations() + .getByName("implementation") + .getAllDependencies(); + var subtasks = + allDeps.withType(ProjectDependency.class).stream() + .map(dep -> dep.getPath() + ":" + task.getName()) + .toList(); + + task.getLogger() + .info( + "Task {} depends on -> {}", + task.getPath(), + String.join(", ", subtasks)); + + return subtasks; + })); + }); + } + + @CacheableTask + public abstract static class RenderJavadocTask extends RenderJavadocTaskBase { + @Inject + public abstract FileOperations getFileOps(); + + @InputFiles + @PathSensitive(PathSensitivity.RELATIVE) + @IgnoreEmptyDirectories + @SkipWhenEmpty + public abstract Property getSrcDirSet(); + + @OutputDirectory + public abstract DirectoryProperty getOutputDir(); + + @CompileClasspath + public abstract ConfigurableFileCollection getClasspath(); + + @CompileClasspath + public abstract ConfigurableFileCollection getDocletpath(); + + @Input + public abstract Property getTitle(); + + @Input + public abstract Property getLinksource(); + + @Input + public abstract Property getEnableSearch(); + + @Input + public abstract Property getRelativeProjectLinks(); + + @Input + public abstract Property getReleaseVersion(); + + @Internal + public abstract MapProperty getOfflineLinks(); + + // Computes cacheable inputs from the map in offlineLinks. + @Nested + public List getCacheableOfflineLinks() { + return getOfflineLinks().get().entrySet().stream() + .map( + e -> + getProject() + .getObjects() + .newInstance(OfflineLink.class, e.getKey(), e.getValue())) + .toList(); + } + + @Input + @Optional + public abstract Property getLuceneDocUrl(); + + // default is to require full javadocs + @Input + public abstract Property getJavadocMissingLevel(); + + // anything in these packages is checked with level=method. This allows iteratively fixing one + // package at a time. + @Input + public abstract ListProperty getJavadocMissingMethod(); + + // default is not to ignore any elements, should only be used to workaround split packages + @Input + public abstract ListProperty getJavadocMissingIgnore(); + + @InputDirectory + @PathSensitive(PathSensitivity.RELATIVE) + @IgnoreEmptyDirectories + public abstract DirectoryProperty getTaskResources(); + + public RenderJavadocTask() { + getLinksource().convention(false); + getEnableSearch().convention(false); + getRelativeProjectLinks().convention(false); + getJavadocMissingLevel().convention("parameter"); + getJavadocMissingIgnore().convention(List.of()); + } + + @TaskAction + public void render() throws IOException { + List srcDirs = + getSrcDirSet().get().getSourceDirectories().getFiles().stream() + .filter(f -> Files.exists(f.toPath())) + .toList(); + + Path optionsFile = getTemporaryDir().toPath().resolve("javadoc-options.txt"); + Files.createDirectories(optionsFile.getParent()); + + // if we are re-rendering, wipe any previous data. + getFileOps().delete(getOutputDir().get().getAsFile()); + + // create the directory, so relative link calculation knows that it's a directory. + Files.createDirectories(getOutputDir().get().getAsFile().toPath()); + + List opts = new ArrayList<>(); + + var overviewFiles = + srcDirs.stream() + .map(dir -> dir.toPath().resolve("overview.html")) + .filter(Files::exists) + .toList(); + if (overviewFiles.size() != 1) { + throw new GradleException("Must be exactly one overview.html file: " + overviewFiles); + } + + opts.add( + List.of("-overview", getProject().file(overviewFiles.getFirst().toFile()).toString())); + opts.add(List.of("-d", getOutputDir().get().getAsFile().toString())); + opts.add("-protected"); + + opts.add(List.of("-encoding", "UTF-8")); + opts.add(List.of("-charset", "UTF-8")); + opts.add(List.of("-docencoding", "UTF-8")); + + if (!getEnableSearch().getOrElse(false)) { + opts.add("-noindex"); + } + + opts.add("-author"); + opts.add("-version"); + if (getLinksource().get()) { + opts.add("-linksource"); + } + + opts.add("-use"); + + opts.add(List.of("-locale", "en_US")); + + opts.add(List.of("-windowtitle", getTitle().get())); + opts.add(List.of("-doctitle", getTitle().get())); + + if (!getClasspath().isEmpty()) { + opts.add(List.of("-classpath", getClasspath().getAsPath())); + } + + var buildGlobals = getLuceneBuildGlobals(getProject()); + opts.add( + List.of( + "-bottom", + "Copyright © 2000-" + + buildGlobals.buildYear + + " Apache Software Foundation. All Rights Reserved.")); + + opts.add( + List.of( + "-tag", + "lucene.experimental:a:WARNING: This API is experimental and might change in incompatible ways in the next release.")); + opts.add( + List.of( + "-tag", + "lucene.internal:a:NOTE: This API is for internal purposes only and might change in incompatible ways in the next release.")); + opts.add( + List.of( + "-tag", + "lucene.spi:t:SPI Name (case-insensitive: if the name is 'htmlStrip', 'htmlstrip' can be used when looking up the service).")); + + opts.add(List.of("-doclet", "org.apache.lucene.missingdoclet.MissingDoclet")); + opts.add(List.of("-docletpath", getDocletpath().getAsPath())); + + opts.add(List.of("--missing-level", getJavadocMissingLevel().get())); + + var missingIgnored = getJavadocMissingIgnore().getOrElse(List.of()); + if (!missingIgnored.isEmpty()) { + opts.add(List.of("--missing-ignore", String.join(",", missingIgnored))); + } + + var missingMethod = getJavadocMissingMethod().getOrElse(List.of()); + if (!missingMethod.isEmpty()) { + opts.add(List.of("--missing-method", String.join(",", missingMethod))); + } + + opts.add("-quiet"); + + // Add all extra options, if any. + opts.addAll(getExtraOpts().getOrElse(List.of())); + + Map allOfflineLinks = new LinkedHashMap<>(); + allOfflineLinks.putAll(getOfflineLinks().get()); + + addOfflineOrRelativeLinksToDependencies(opts, allOfflineLinks); + + allOfflineLinks.forEach( + (url, dir) -> { + // Some sanity check/ validation here to ensure dir/package-list or dir/element-list is + // present. + if (!Files.exists(dir.toPath().resolve("package-list")) + && !Files.exists(dir.toPath().resolve("element-list"))) { + throw new GradleException( + "Expected pre-rendered package-list or element-list at: " + dir); + } + getLogger().info("Offline link: {} to {}", url, dir); + opts.add(List.of("-linkoffline", url, dir.toString())); + }); + + opts.add(List.of("--release", getReleaseVersion().get().getMajorVersion())); + + opts.add("-Xdoclint:all,-missing"); + + // Increase Javadoc's heap. + opts.add("-J-Xmx512m"); + + // Force locale to be "en_US" (fix for: https://bugs.openjdk.java.net/browse/JDK-8222793) + opts.add("-J-Duser.language=en"); + opts.add("-J-Duser.country=US"); + + // add custom scripts and css. + { + // append some special table css, prettify css. + Provider customCss = + getOutputDir().file("resource-files/lucene-stylesheet.css"); + concat( + customCss, + getTaskResources(), + "table_padding.css", + "custom_styles.css", + "prettify/prettify.css"); + + // append prettify to scripts + Provider customScript = getOutputDir().file("script-files/lucene-script.js"); + concat( + customScript, getTaskResources().dir("prettify"), "prettify.js", "inject-javadocs.js"); + + opts.add(List.of("--add-script", customScript.get().getAsFile().toString())); + opts.add(List.of("--add-stylesheet", customCss.get().getAsFile().toString())); + } + + // -J options have to be passed on command line, they are not interpreted if passed via args + // file. + var jOpts = + opts.stream() + .filter(opt -> opt instanceof String && ((String) opt).startsWith("-J")) + .toList(); + opts.removeAll(jOpts); + + // Collect all source files, for now excluding module descriptors. + opts.addAll( + srcDirs.stream() + .flatMap( + dir -> + getProject() + .fileTree( + dir, + cfg -> { + cfg.include("**/*.java"); + cfg.exclude("**/module-info.java"); + }) + .getFiles() + .stream()) + .map(File::toString) + .toList()); + + // handle doc-files manually since in explicit source file mode javadoc does not copy them. + for (var dir : srcDirs) { + getFileOps() + .copy( + spec -> { + spec.into(getOutputDir()); + spec.from( + dir, + cfg -> { + cfg.include("**/doc-files/**"); + }); + }); + } + + // Temporary file that holds all javadoc options for the current task (except jOpts) + try (var writer = Files.newBufferedWriter(optionsFile)) { + // escapes an option with single quotes or whitespace to be passed in the options.txt file + // for + Function escapeJavadocOption = + s -> { + if (Pattern.compile("[ '\"]").matcher(s).find()) { + String escaped = s.replaceAll("[\\\\'\"]", "\\\\$0"); + return "'" + escaped + "'"; + } else { + return s; + } + }; + + for (var entry : opts) { + if (entry instanceof List asList) { + writer.write( + asList.stream() + .map(v -> escapeJavadocOption.apply(v.toString())) + .collect(Collectors.joining(" "))); + } else { + writer.write(escapeJavadocOption.apply(entry.toString())); + } + writer.write("\n"); + } + } + + var javadocCmd = getProject().file(getExecutable().get()); + getLogger().info("Javadoc executable used: " + javadocCmd); + + buildGlobals.quietExec( + this, + execSpec -> { + execSpec.executable(javadocCmd); + execSpec.args("@" + optionsFile); + execSpec.args(jOpts); + }); + } + + private void concat( + Provider targetFile, Provider dir, String... files) { + var concat = (Concat) getAnt().getAntProject().createTask("concat"); + concat.setDestfile(targetFile.get().getAsFile()); + concat.setAppend(true); + concat.setFixLastLine(true); + concat.setEncoding("UTF-8"); + concat.addFilelist(fileList(dir, files)); + concat.execute(); + + var fixcrlf = (FixCRLF) getAnt().getAntProject().createTask("fixcrlf"); + fixcrlf.setSrcdir(dir.get().getAsFile()); + fixcrlf.setIncludes(String.join(" ", files)); + fixcrlf.setEncoding("UTF-8"); + fixcrlf.setFixlast(true); + var lf = new FixCRLF.CrLf(); + lf.setValue("lf"); + fixcrlf.setEol(lf); + fixcrlf.execute(); + } + + private FileList fileList(Provider dir, String... files) { + var fileList = new FileList(); + fileList.setDir(dir.get().getAsFile()); + fileList.setFiles(String.join(" ", files)); + return fileList; + } + + /** + * Resolve inter-project links: + * + *
    + *
  • find all (enabled) tasks from the subgraph of tasks this task depends on (with same + * name) + *
  • sort these tasks, ordering the 'core' first, then lexigraphically by path + *
  • for each task, get the output dir to create relative or absolute link. + *
+ */ + private void addOfflineOrRelativeLinksToDependencies( + List opts, Map allOfflineLinks) { + List sortedTaskDeps; + { + Set taskDeps = new HashSet<>(); + var taskGraph = getProject().getGradle().getTaskGraph(); + ArrayDeque remaining = new ArrayDeque<>(List.of(this)); + var thisTaskName = getName(); + while (!remaining.isEmpty()) { + var task = remaining.pop(); + taskGraph.getDependencies(task).stream() + .filter(t -> t.getName().equals(thisTaskName) && t.getEnabled()) + .forEach( + t -> { + if (taskDeps.add(((RenderJavadocTask) t))) { + remaining.add(t); + } + }); + } + + sortedTaskDeps = + taskDeps.stream() + .sorted( + Comparator.comparing( + t -> !t.getProject().getName().equals("core")) + .thenComparing(Task::getPath)) + .toList(); + } + + for (var otherTask : sortedTaskDeps) { + Project otherProject = otherTask.getProject(); + // For relative links we compute the actual relative link between projects. + if (getRelativeProjectLinks().getOrElse(true)) { + Path pathTo = otherTask.getOutputDir().get().getAsFile().toPath().toAbsolutePath(); + Path pathFrom = getOutputDir().get().getAsFile().toPath().toAbsolutePath(); + String relative = pathFrom.relativize(pathTo).toString().replace(File.separatorChar, '/'); + getLogger().info("Relative link: {} to {}", otherProject.getPath(), relative); + opts.add(List.of("-link", relative)); + } else { + // For absolute links, we determine the target URL by assembling the full URL (if base is + // available). + if (getLuceneDocUrl().isPresent()) { + allOfflineLinks.put( + getLuceneDocUrl().get() + + "/" + + DocumentationConfigPlugin.relativeDocPath(otherProject), + otherTask.getOutputDir().get().getAsFile()); + } else { + // Ignore, not linking relative modules at all. + } + } + } + } + } + + public abstract static class OfflineLink implements Serializable { + @Input + public abstract Property getUrl(); + + @InputDirectory + @PathSensitive(PathSensitivity.RELATIVE) + @IgnoreEmptyDirectories + public abstract DirectoryProperty getLocation(); + + @Inject + public OfflineLink(String url, File location) { + this.getUrl().set(url); + this.getLocation().set(location); + } + } +} diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.java index 8ffe33969cf4..12cfe457c42e 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.java @@ -20,6 +20,7 @@ import java.util.Collection; import org.apache.lucene.gradle.plugins.LuceneGradlePlugin; import org.apache.lucene.gradle.plugins.astgrep.AstGrepPlugin; +import org.apache.lucene.gradle.plugins.documentation.DocumentationConfigPlugin; import org.apache.lucene.gradle.plugins.eclint.EditorConfigLintPlugin; import org.apache.lucene.gradle.plugins.gitgrep.GitGrepPlugin; import org.apache.lucene.gradle.plugins.gitinfo.GitInfoPlugin; @@ -87,6 +88,8 @@ public void apply(Project rootProject) { plugins.apply(MeasureTaskTimesPlugin.class); plugins.apply(DumpGradleStateOnStalledBuildsPlugin.class); + plugins.apply(DocumentationConfigPlugin.class); + // Apply more convention plugins to all projects. rootProject .getAllprojects() diff --git a/build.gradle b/build.gradle index 07f8a7df1298..ef40c760964c 100644 --- a/build.gradle +++ b/build.gradle @@ -33,10 +33,6 @@ plugins { id "lucene.misc.pylucene" - id "lucene.documentation" - id "lucene.documentation.markdown" - id "lucene.documentation.render-javadoc" - id "lucene.regenerate" id "lucene.regenerate.jflex" id "lucene.regenerate.forUtil"