From 50a064809902ddfbe88ef2db282f8fac656c6f89 Mon Sep 17 00:00:00 2001 From: Colvin Cowie <51863265+colvinco@users.noreply.github.com> Date: Mon, 6 Feb 2023 15:18:27 +0000 Subject: [PATCH] SOLR-16641 - Generate gradle.properties from gradlew (#1320) --- .github/workflows/bin-solr-test.yml | 2 - .github/workflows/docker-test.yml | 2 - .github/workflows/gradle-precommit.yml | 3 - .github/workflows/solrj-test.yml | 2 - .gitignore | 2 +- .../gradle/GradlePropertiesGenerator.java | 72 +++++++++++++ dev-docs/how-to-contribute.adoc | 1 + dev-docs/solr-source-code.adoc | 2 + dev-tools/scripts/releaseWizard.yaml | 2 - gradle/generation/local-settings.gradle | 81 +------------- gradle/template.gradle.properties | 102 ++++++++++++++++++ .../validation/gradlew-scripts-tweaked.gradle | 4 +- gradlew | 12 ++- gradlew.bat | 15 ++- help/localSettings.txt | 60 +---------- solr/CHANGES.txt | 2 + 16 files changed, 207 insertions(+), 157 deletions(-) create mode 100644 buildSrc/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java create mode 100644 gradle/template.gradle.properties diff --git a/.github/workflows/bin-solr-test.yml b/.github/workflows/bin-solr-test.yml index 6daadd1817e..d8a6076da64 100644 --- a/.github/workflows/bin-solr-test.yml +++ b/.github/workflows/bin-solr-test.yml @@ -34,8 +34,6 @@ jobs: restore-keys: | ${{ runner.os }}-gradle-binsolr- ${{ runner.os }}-gradle- - - name: Initialize gradle settings - run: ./gradlew localSettings - name: Test the bin/solr script run: ./gradlew integrationTests - name: Archive logs diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index 511cee11e8c..ddf6292d46f 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -42,8 +42,6 @@ jobs: restore-keys: | ${{ runner.os }}-gradle-docker- ${{ runner.os }}-gradle- - - name: Initialize gradle settings - run: ./gradlew localSettings - name: Build Docker image with Gradle run: ./gradlew solr:docker:docker - name: Run tests on Docker image diff --git a/.github/workflows/gradle-precommit.yml b/.github/workflows/gradle-precommit.yml index 73bb5367605..6cc1425db84 100644 --- a/.github/workflows/gradle-precommit.yml +++ b/.github/workflows/gradle-precommit.yml @@ -34,9 +34,6 @@ jobs: ${{ runner.os }}-gradle-precommit- ${{ runner.os }}-gradle- - - name: Initialize gradle settings - run: ./gradlew localSettings - - name: Run gradle check (without tests) run: ./gradlew check -x test -Ptask.times=true diff --git a/.github/workflows/solrj-test.yml b/.github/workflows/solrj-test.yml index a935a5598be..625ab7ee437 100644 --- a/.github/workflows/solrj-test.yml +++ b/.github/workflows/solrj-test.yml @@ -33,7 +33,5 @@ jobs: restore-keys: | ${{ runner.os }}-gradle-solrj- ${{ runner.os }}-gradle- - - name: Initialize gradle settings - run: ./gradlew localSettings - name: Test the SolrJ Package run: ./gradlew solr:solrj:test diff --git a/.gitignore b/.gitignore index 419ffff531c..18542bd60d4 100644 --- a/.gitignore +++ b/.gitignore @@ -37,7 +37,7 @@ __pycache__ *~ # Ignore lucene included build -lucene/ +/lucene # Ignore gradle wrapper gradle/wrapper/gradle-wrapper.jar diff --git a/buildSrc/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java b/buildSrc/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java new file mode 100644 index 00000000000..db4f804f12e --- /dev/null +++ b/buildSrc/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.gradle; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.Map; + +/** + * Standalone class that generates a populated gradle.properties from a template. + * + *

Has no dependencies outside of standard java libraries + */ +public class GradlePropertiesGenerator { + public static void main(String[] args) { + if (args.length != 2) { + System.err.println("Usage: java GradlePropertiesGenerator.java "); + System.exit(2); + } + + try { + new GradlePropertiesGenerator().run(Paths.get(args[0]), Paths.get(args[1])); + } catch (Exception e) { + System.err.println("ERROR: " + e.getMessage()); + System.exit(3); + } + } + + public void run(Path source, Path destination) throws IOException { + if (!Files.exists(source)) { + throw new IOException("template file not found: " + source); + } + if (Files.exists(destination)) { + System.out.println(destination + " already exists, skipping generation."); + return; + } + + // Approximate a common-sense default for running gradle/tests with parallel + // workers: half the count of available cpus but not more than 12. + var cpus = Runtime.getRuntime().availableProcessors(); + var maxWorkers = (int) Math.max(1d, Math.min(cpus * 0.5d, 12)); + var testsJvms = (int) Math.max(1d, Math.min(cpus * 0.5d, 12)); + + var replacements = Map.of("@MAX_WORKERS@", maxWorkers, "@TEST_JVMS@", testsJvms); + + System.out.println("Generating gradle.properties"); + String fileContent = Files.readString(source, StandardCharsets.UTF_8); + for (var entry : replacements.entrySet()) { + fileContent = fileContent.replace(entry.getKey(), String.valueOf(entry.getValue())); + } + Files.writeString( + destination, fileContent, StandardCharsets.UTF_8, StandardOpenOption.CREATE_NEW); + } +} diff --git a/dev-docs/how-to-contribute.adoc b/dev-docs/how-to-contribute.adoc index ef82760399e..d072d69260c 100644 --- a/dev-docs/how-to-contribute.adoc +++ b/dev-docs/how-to-contribute.adoc @@ -17,6 +17,7 @@ In order to make a new contribution to Solr you will use the fork you have creat 1. Create a new Jira issue in the Solr project: https://issues.apache.org/jira/projects/SOLR/issues 2. Create a new branch in your Solr fork to provide a PR for your contribution on the newly created issue. Make any necessary changes for the given bug/feature in that branch. You can use additional information in these dev-docs to build and test your code as well as ensure it passes code quality checks. 3. Once you are satisfied with your changes, get your branch ready for a PR by running `./gradlew tidy updateLicenses check -x test`. This will format your source code, update licenses of any dependency version changes and run all pre-commit tests. Commit the changes. +* Note: the `check` command requires `perl` and `python3` to be present on your `PATH` to validate documentation. 4. Open a PR of your branch against the `main` branch of the apache/solr repository. When you open a PR on your fork, this should be the default option. * The title of your PR should include the Solr Jira issue that you opened, i.e. `SOLR-12345: New feature`. * The PR description will automatically populate with a pre-set template that you will need to fill out. diff --git a/dev-docs/solr-source-code.adoc b/dev-docs/solr-source-code.adoc index 9f9cf79e021..9daf86cd2d4 100644 --- a/dev-docs/solr-source-code.adoc +++ b/dev-docs/solr-source-code.adoc @@ -38,6 +38,8 @@ If you want to build the documentation, type `./gradlew -p solr documentation`. `./gradlew check` will assemble Solr and run all validation tasks unit tests. +NOTE: the `check` command requires `perl` and `python3` to be present on your `PATH` to validate documentation. + To build the final Solr artifacts run `./gradlew assemble`. Lastly, there is developer oriented documentation in `./dev-docs/README.adoc` that diff --git a/dev-tools/scripts/releaseWizard.yaml b/dev-tools/scripts/releaseWizard.yaml index 747c62d22a4..7e6875889aa 100644 --- a/dev-tools/scripts/releaseWizard.yaml +++ b/dev-tools/scripts/releaseWizard.yaml @@ -356,8 +356,6 @@ groups: - !Command cmd: git pull --ff-only stdout: true - - !Command - cmd: "{{ gradle_cmd }} localSettings" - !Command cmd: "{{ gradle_cmd }} clean check -x test" - !Todo diff --git a/gradle/generation/local-settings.gradle b/gradle/generation/local-settings.gradle index 263183b57e7..e164b890706 100644 --- a/gradle/generation/local-settings.gradle +++ b/gradle/generation/local-settings.gradle @@ -41,87 +41,12 @@ configure(rootProject) { throw new GradleException( "Certain gradle tasks and plugins require access to jdk.compiler" + " internals, your gradle.properties might have just been generated or could be" + - " out of sync (see help/localSettings.txt)") - } - } - } - - task localSettings() { - doFirst { - // If we don't have the defaults yet, create them. - if (hasDefaults) { - logger.lifecycle("Local settings already exist, skipping generation.") - } else { - // Approximate a common-sense default for running gradle/tests with parallel - // workers: half the count of available cpus but not more than 12. - def cpus = Runtime.runtime.availableProcessors() - def maxWorkers = (int) Math.max(1d, Math.min(cpus * 0.5d, 12)) - def testsJvms = (int) Math.max(1d, Math.min(cpus * 0.5d, 12)) - - // Write the defaults for this machine. - rootProject.file("gradle.properties").write(""" -# These settings have been generated automatically on the first run. -# See gradlew :helpLocalSettings for more information. -systemProp.file.encoding=UTF-8 - -# Set up gradle JVM defaults. -# -# We also open up internal compiler modules for spotless/ google java format. -org.gradle.jvmargs=-Xmx1g -XX:TieredStopAtLevel=1 -XX:+UseParallelGC -XX:ActiveProcessorCount=1 \\ - --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \\ - --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \\ - --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \\ - --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \\ - --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED - -# Run at normal priority, in parallel -org.gradle.parallel=true -org.gradle.priority=normal - -# This setting enables local task output caches. This will speed up -# your local builds in most cases but will also consume disk space in your -# gradle home. See SOLR-15603 for details. -# org.gradle.caching=true - -# Silence gradle warnings. We'll deal with them when we upgrade the wrapper. -org.gradle.warning.mode=none - -# You may disable the background daemon if it consumes too much memory. -org.gradle.daemon=true -# timeout after 15 mins of inactivity. -org.gradle.daemon.idletimeout=900000 - -# Maximum number of parallel gradle workers. -org.gradle.workers.max=${maxWorkers} - -# Maximum number of test JVMs forked per test task. -tests.jvms=${testsJvms} - -# Disable auto JVM provisioning (we don't use toolchains yet but want no surprises). -org.gradle.java.installations.auto-download=false - -# Set these to enable automatic JVM location discovery. -org.gradle.java.installations.fromEnv=JDK11,JDK12,JDK13,JDK14,JDK15,JDK16,JDK17 -org.gradle.java.installations.paths=(custom paths) - -""", "UTF-8") - - logger.log(LogLevel.WARN, "\nIMPORTANT. This is the first time you ran the build. " + - "I wrote some sane defaults (for this machine) to 'gradle.properties', " + - "they will be picked up on consecutive gradle invocations (not this one).\n\n" + - "Run gradlew :helpLocalSettings for more information.") + " out of sync (see gradle/template.gradle.properties)") } } } } -if (!hasDefaults) { - // Make all tasks depend on local setup to make sure it'll run. - allprojects { - tasks.all { task -> - if (task != rootProject.localSettings) { - task.dependsOn rootProject.localSettings - } - } - } +task localSettings() { + // This is just a placeholder until all references to the localSettings task are removed #SOLR-16641 } diff --git a/gradle/template.gradle.properties b/gradle/template.gradle.properties new file mode 100644 index 00000000000..c52e3048a77 --- /dev/null +++ b/gradle/template.gradle.properties @@ -0,0 +1,102 @@ +############################# +# Local developer settings # +############################# +# +# The first invocation of any task in Solr gradlew will generate and save this project-local 'gradle.properties' file. +# This file contains the defaults you may (but don't have to) tweak for your particular hardware (or taste). Note there +# are certain settings in that file that may be _required_ at runtime for certain plugins (an example is the spotless/ +# google java format plugin, which requires adding custom exports to JVM modules). gradlew only generates this file +# if it's not already present (it never overwrites the defaults) -- occasionally you may have to manually delete (or move) +# this file and regenerate from scratch. +# +# This is an overview of some of these settings. +# +############### +# Parallelism # +############### +# +# Gradle build can run tasks in parallel but by default it consumes all CPU cores which +# is too optimistic a default for Solr tests. You can disable the parallelism +# entirely or assign it a 'low' priority with these properties: +# +# org.gradle.parallel=[true, false] +# org.gradle.priority=[normal, low] +# +# The default level of parallelism is computed based on the number of cores on +# your machine (on the first run of gradle build). By default these are fairly conservative +# settings (half the number of cores for workers, for example): +# +# org.gradle.workers.max=[X] +# tests.jvms=[N <= X] +# +# The number of test JVMs can be lower than the number of workers: this just means +# that two projects can run tests in parallel to saturate all the workers. The I/O and memory +# bandwidth limits will kick in quickly so even if you have a very beefy machine bumping +# it too high may not help. +# +# You can always override these settings locally using command line as well: +# gradlew -Ptests.jvms=N --max-workers=X +# +############# +# Test JVMS # +############# +# +# Test JVMs have their own set of arguments which can be customized. These are configured +# separately from the gradle workers, for example: +# +# tests.jvms=3 +# tests.heapsize=512m +# tests.minheapsize=512m +# tests.jvmargs=-XX:+UseParallelGC -XX:TieredStopAtLevel=1 -XX:ActiveProcessorCount=1 +# +################# +# Gradle Daemon # +################# +# +# The gradle daemon is a background process that keeps an evaluated copy of the project +# structure, some caches, etc. It speeds up repeated builds quite a bit but if you don't +# like the idea of having a (sizeable) background process running in the background, +# disable it. +# +# org.gradle.daemon=[true, false] +# org.gradle.jvmargs=... +############################################################################################# + +# UTF-8 as standard file encoding +systemProp.file.encoding=UTF-8 + +# Set up gradle JVM defaults. +# +# We also open up internal compiler modules for spotless/ google java format. +org.gradle.jvmargs=-Xmx1g -XX:TieredStopAtLevel=1 -XX:+UseParallelGC -XX:ActiveProcessorCount=1 \ + --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ + --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ + --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \ + --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \ + --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + +# Run at normal priority, in parallel +org.gradle.parallel=true +org.gradle.priority=normal + +# This setting enables local task output caches. This will speed up +# your local builds in most cases but will also consume disk space in your +# gradle home. See SOLR-15603 for details. +# org.gradle.caching=true + +# Silence gradle warnings. We'll deal with them when we upgrade the wrapper. +org.gradle.warning.mode=none + +# You may disable the background daemon if it consumes too much memory. +org.gradle.daemon=true +# timeout after 15 mins of inactivity. +org.gradle.daemon.idletimeout=900000 + +# Maximum number of parallel gradle workers. +org.gradle.workers.max=@MAX_WORKERS@ + +# Maximum number of test JVMs forked per test task. +tests.jvms=@TEST_JVMS@ + +# Disable auto JVM provisioning (we don't use toolchains yet but want no surprises). +org.gradle.java.installations.auto-download=false diff --git a/gradle/validation/gradlew-scripts-tweaked.gradle b/gradle/validation/gradlew-scripts-tweaked.gradle index a37d4e52b13..2720e70617e 100644 --- a/gradle/validation/gradlew-scripts-tweaked.gradle +++ b/gradle/validation/gradlew-scripts-tweaked.gradle @@ -32,8 +32,8 @@ configure(rootProject) { doFirst { scripts.each { file -> def content = new String(file.readBytes(), StandardCharsets.US_ASCII) - if (content.indexOf("Don't fork a daemon mode on initial run that generates local defaults") < 0) { - throw new GradleException("Launch script ${file} does not have a manual daemon tweak (see LUCENE-9232).") + if (content.indexOf("GradlePropertiesGenerator") < 0) { + throw new GradleException("Launch script ${file} does not have a tweak to generate gradle.properties (see SOLR-16641).") } } } diff --git a/gradlew b/gradlew index 59dd3f7eb88..03f6fc89ee0 100755 --- a/gradlew +++ b/gradlew @@ -163,11 +163,16 @@ fi CLASSPATH=$GRADLE_WRAPPER_JAR -# Don't fork a daemon mode on initial run that generates local defaults. -GRADLE_DAEMON_CTRL= +# START OF SOLR CUSTOMIZATION +# Generate gradle.properties if they don't exist if [ ! -e "$APP_HOME/gradle.properties" ]; then - GRADLE_DAEMON_CTRL=--no-daemon + "$JAVACMD" $JAVA_OPTS --source 11 "$APP_HOME/buildSrc/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java" "$APP_HOME/gradle/template.gradle.properties" "$APP_HOME/gradle.properties" + GENERATOR_STATUS=$? + if [ "$GENERATOR_STATUS" -ne 0 ]; then + exit $GENERATOR_STATUS + fi fi +# END OF SOLR CUSTOMIZATION # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then @@ -237,7 +242,6 @@ set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ -classpath "$CLASSPATH" \ org.gradle.wrapper.GradleWrapperMain \ - $GRADLE_DAEMON_CTRL \ "$@" # Stop when "xargs" is not available. diff --git a/gradlew.bat b/gradlew.bat index 964ab6aced2..dc413985442 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -80,12 +80,19 @@ IF %ERRORLEVEL% NEQ 0 goto fail @rem Setup the command line set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar -@rem Don't fork a daemon mode on initial run that generates local defaults. -SET GRADLE_DAEMON_CTRL= -IF NOT EXIST "%DIRNAME%\gradle.properties" SET GRADLE_DAEMON_CTRL=--no-daemon +@rem START OF SOLR CUSTOMIZATION +@rem Generate gradle.properties if they don't exist +IF NOT EXIST "%APP_HOME%\gradle.properties" ( + @rem local expansion is needed to check ERRORLEVEL inside control blocks. + setlocal enableDelayedExpansion + "%JAVA_EXE%" --source 11 "%APP_HOME%/buildSrc/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java" "%APP_HOME%\gradle\template.gradle.properties" "%APP_HOME%\gradle.properties" + IF %ERRORLEVEL% NEQ 0 goto fail + endlocal +) +@rem END OF SOLR CUSTOMIZATION @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %GRADLE_DAEMON_CTRL% %* +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* :end @rem End local scope for the variables with windows NT shell diff --git a/help/localSettings.txt b/help/localSettings.txt index b4510563a7c..5089bb669ca 100644 --- a/help/localSettings.txt +++ b/help/localSettings.txt @@ -2,61 +2,7 @@ Local developer settings ======================== The first invocation of any task in Solr gradle build will generate -and save a project-local 'gradle.properties' file. This file contains -the defaults you may (but don't have to) tweak for your particular hardware -(or taste). Note there are certain settings in that file that may -be _required_ at runtime for certain plugins (an example is the spotless/ -google java format plugin, which requires adding custom exports to JVM modules). Gradle -build only generates this file if it's not already present (it never overwrites -the defaults) -- occasionally you may have to manually delete (or move) this -file and regenerate from scratch. - -This is an overview of some of these settings. - -Parallelism ------------ - -Gradle build can run tasks in parallel but by default it consumes all CPU cores which -is too optimistic a default for Solr tests. You can disable the parallelism -entirely or assign it a 'low' priority with these properties: - -org.gradle.parallel=[true, false] -org.gradle.priority=[normal, low] - -The default level of parallelism is computed based on the number of cores on -your machine (on the first run of gradle build). By default these are fairly conservative -settings (half the number of cores for workers, for example): - -org.gradle.workers.max=[X] -tests.jvms=[N <= X] - -The number of test JVMs can be lower than the number of workers: this just means -that two projects can run tests in parallel to saturate all the workers. The I/O and memory -bandwidth limits will kick in quickly so even if you have a very beefy machine bumping -it too high may not help. - -You can always override these settings locally using command line as well: -gradlew -Ptests.jvms=N --max-workers=X - -Test JVMS ---------- - -Test JVMs have their own set of arguments which can be customized. These are configured -separately from the gradle workers, for example: - -tests.jvms=3 -tests.heapsize=512m -tests.minheapsize=512m -tests.jvmargs=-XX:+UseParallelGC -XX:TieredStopAtLevel=1 -XX:ActiveProcessorCount=1 - -Gradle Daemon -------------- - -The gradle daemon is a background process that keeps an evaluated copy of the project -structure, some caches, etc. It speeds up repeated builds quite a bit but if you don't -like the idea of having a (sizeable) background process running in the background, -disable it. - -org.gradle.daemon=[true, false] -org.gradle.jvmargs=... +and save a project-local 'gradle.properties' file from a template, +with some default settings which you can modify. +See your 'gradle.properties' file for more details of those settings. diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index f38985d58af..ecc780988a9 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -207,6 +207,8 @@ Build * SOLR-16625: Upgrade OWASP dependency check to 8.0.1 (Kevin Risden) +* SOLR-16641: Generate gradle.properties from gradlew, if absent (Colvin Cowie) + Other Changes --------------------- * SOLR-16545: Upgrade Carrot2 to 4.5.0 (Dawid Weiss)