From 99dc8629891af948aed09d58705899ff176e1314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Wed, 2 Feb 2022 11:45:08 +0100 Subject: [PATCH 1/5] HSEARCH-4472 Move the release job to a separate Jenkinsfile --- Jenkinsfile | 87 +++++---------------------------------- jenkins/release.groovy | 92 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 78 deletions(-) create mode 100644 jenkins/release.groovy diff --git a/Jenkinsfile b/Jenkinsfile index 201316a0a14..7c54d6e1cf0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -56,14 +56,13 @@ import org.hibernate.jenkins.pipeline.helpers.version.Version * * #### Nexus deployment * - * This job includes two deployment modes: - * - * - A deployment of snapshot artifacts for every non-PR build on "primary" branches (main and maintenance branches). - * - A full release when starting the job with specific parameters. - * - * In the first case, the name of a Maven settings file must be provided in the job configuration file + * This job is only able to deploy snapshot artifacts, + * for every non-PR build on "primary" branches (main and maintenance branches), + * but the name of a Maven settings file must be provided in the job configuration file * (see below). * + * For actual releases, see jenkins/release.groovy. + * * #### AWS * * This job will trigger integration tests against an Elasticsearch service hosted on AWS. @@ -130,8 +129,7 @@ import org.hibernate.jenkins.pipeline.helpers.version.Version * deployment: * maven: * # String containing the ID of a Maven settings file registered using the config-file-provider Jenkins plugin. - * # The settings must provide credentials to the servers with ID - * # 'jboss-releases-repository' and 'jboss-snapshots-repository'. + * # The settings must provide credentials to the server with ID 'ossrh'. * settingsId: ... */ @@ -149,12 +147,8 @@ import org.hibernate.jenkins.pipeline.helpers.version.Version @Field boolean enableDefaultBuild = false @Field boolean enableDefaultBuildIT = false -@Field boolean performRelease = false @Field boolean deploySnapshot = false -@Field Version releaseVersion -@Field Version afterReleaseDevelopmentVersion - this.helper = new JobHelper(this) helper.runWithNotification { @@ -248,7 +242,7 @@ DEFAULT SUPPORTED ALL""", description: """A set of environments that must be checked. -'AUTOMATIC' picks a different set of environments based on the branch name and whether a release is being performed. +'AUTOMATIC' picks a different set of environments based on the branch name. 'DEFAULT' means a single build with the default environment expected by the Maven configuration, while other options will trigger multiple Maven executions in different environments.""" ), @@ -260,30 +254,11 @@ while other options will trigger multiple Maven executions in different environm If this parameter is non-empty, ENVIRONMENT_SET will be ignored and environments whose tag matches the given regex will be checked. Some useful filters: 'default', 'jdk', 'jdk-10', 'eclipse', 'postgresql', 'elasticsearch-local-[5.6'. """ - ), - string( - name: 'RELEASE_VERSION', - defaultValue: '', - description: 'The version to be released, e.g. 5.10.0.Final. Setting this triggers a release.', - trim: true - ), - string( - name: 'RELEASE_DEVELOPMENT_VERSION', - defaultValue: '', - description: 'The next version to be used after the release, e.g. 5.10.0-SNAPSHOT.', - trim: true - ), - booleanParam( - name: 'RELEASE_DRY_RUN', - defaultValue: false, - description: 'If true, just simulate the release, without pushing any commits or tags, and without uploading any artifacts or documentation.' ) ]) ]) - performRelease = (params.RELEASE_VERSION ? true : false) - - if (!performRelease && helper.scmSource.branch.primary && !helper.scmSource.pullRequest) { + if (helper.scmSource.branch.primary && !helper.scmSource.pullRequest) { if (helper.configuration.file?.deployment?.maven?.settingsId) { deploySnapshot = true } @@ -336,32 +311,8 @@ Resulting execution plan: enableDefaultBuild=$enableDefaultBuild enableDefaultBuildIT=$enableDefaultBuildIT environments=${environments.enabledAsString} - performRelease=$performRelease deploySnapshot=$deploySnapshot """ - - if (performRelease) { - releaseVersion = Version.parseReleaseVersion(params.RELEASE_VERSION) - echo "Inferred version family for the release to '$releaseVersion.family'" - - // Check that all the necessary parameters are set - if (!params.RELEASE_DEVELOPMENT_VERSION) { - throw new IllegalArgumentException( - "Missing value for parameter RELEASE_DEVELOPMENT_VERSION." + - " This parameter must be set when RELEASE_VERSION is set." - ) - } - if (!params.RELEASE_DRY_RUN && !helper.configuration.file?.deployment?.maven?.settingsId) { - throw new IllegalArgumentException( - "Missing deployment configuration in job configuration file." + - " Cannot deploy artifacts during the release." - ) - } - } - - if (params.RELEASE_DEVELOPMENT_VERSION) { - afterReleaseDevelopmentVersion = Version.parseDevelopmentVersion(params.RELEASE_DEVELOPMENT_VERSION) - } } stage('Default build') { @@ -517,24 +468,6 @@ stage('Deploy') { } } } - else if (performRelease) { - echo "Performing full release for version ${releaseVersion.toString()}" - runBuildOnNode { - helper.withMavenWorkspace(mavenSettingsConfig: params.RELEASE_DRY_RUN ? null : helper.configuration.file.deployment.maven.settingsId) { - configFileProvider([configFile(fileId: 'release.config.ssh', targetLocation: env.HOME + '/.ssh/config')]) { - withCredentials([file(credentialsId: 'release.gpg.private-key', variable: 'RELEASE_GPG_PRIVATE_KEY_PATH'), - string(credentialsId: 'release.gpg.passphrase', variable: 'RELEASE_GPG_PASSPHRASE')]) { - sshagent(['ed25519.Hibernate-CI.github.com', 'hibernate.filemgmt.jboss.org', 'hibernate-ci.frs.sourceforge.net']) { - sh 'cat $HOME/.ssh/config' - sh "git clone https://github.com/hibernate/hibernate-noorm-release-scripts.git" - env.RELEASE_GPG_HOMEDIR = env.WORKSPACE_TMP + '/.gpg' - sh "bash -xe hibernate-noorm-release-scripts/release.sh ${params.RELEASE_DRY_RUN ? '-d' : ''} search ${releaseVersion.toString()} ${afterReleaseDevelopmentVersion.toString()}" - } - } - } - } - } - } else { echo "Skipping deployment" helper.markStageSkipped() @@ -672,9 +605,7 @@ void keepOnlyEnvironmentsFromSet(String environmentSetName) { enableOptional = true break case 'AUTOMATIC': - if (params.RELEASE_VERSION) { - echo "Releasing version '$params.RELEASE_VERSION'." - } else if (helper.scmSource.pullRequest) { + if (helper.scmSource.pullRequest) { echo "Building pull request '$helper.scmSource.pullRequest.id'" enableDefaultEnv = true enableBeforeMergeEnvs = true diff --git a/jenkins/release.groovy b/jenkins/release.groovy new file mode 100644 index 00000000000..d1e2997dbbe --- /dev/null +++ b/jenkins/release.groovy @@ -0,0 +1,92 @@ +/* + * Hibernate Search, full-text search for your domain model + * + * License: GNU Lesser General Public License (LGPL), version 2.1 or later + * See the lgpl.txt file in the root directory or . + */ + +@Library('hibernate-jenkins-pipeline-helpers@1.5') _ + +import org.hibernate.jenkins.pipeline.helpers.version.Version + +pipeline { + agent { + label 'Worker&&Containers' + } + tools { + maven 'Apache Maven 3.8' + jdk 'OpenJDK 8 Latest' + } + options { + disableConcurrentBuilds(abortPrevious: false) + } + parameters { + string( + name: 'RELEASE_VERSION', + defaultValue: '', + description: 'The version to be released, e.g. 6.2.0.Final.', + trim: true + ) + string( + name: 'DEVELOPMENT_VERSION', + defaultValue: '', + description: 'The next version to be used after the release, e.g. 6.2.0-SNAPSHOT.', + trim: true + ) + booleanParam( + name: 'RELEASE_DRY_RUN', + defaultValue: false, + description: 'If true, just simulate the release, without pushing any commits or tags, and without uploading any artifacts or documentation.' + ) + } + stages { + stage('Release') { + when { + beforeAgent true + // Releases must be triggered explicitly + // This is just for safety; normally the Jenkins job for this pipeline + // should be configured to "Suppress automatic SCM triggering" + // See https://stackoverflow.com/questions/58259326/prevent-jenkins-multibranch-pipeline-from-triggering-builds-for-new-branches + triggeredBy cause: "UserIdCause" + } + steps { + script { + // Check that all the necessary parameters are set + if (!params.RELEASE_VERSION) { + throw new IllegalArgumentException("Missing value for parameter RELEASE_VERSION.") + } + if (!params.DEVELOPMENT_VERSION) { + throw new IllegalArgumentException("Missing value for parameter DEVELOPMENT_VERSION.") + } + + def releaseVersion = Version.parseReleaseVersion(params.RELEASE_VERSION) + def developmentVersion = Version.parseDevelopmentVersion(params.DEVELOPMENT_VERSION) + echo "Performing full release for version ${releaseVersion.toString()}" + + withMaven(mavenSettingsConfig: params.RELEASE_DRY_RUN ? null : 'ci-hibernate.deploy.settings.maven', + mavenLocalRepo: env.WORKSPACE_TMP + '.m2repository') { + configFileProvider([configFile(fileId: 'release.config.ssh', targetLocation: env.HOME + '/.ssh/config')]) { + withCredentials([file(credentialsId: 'release.gpg.private-key', variable: 'RELEASE_GPG_PRIVATE_KEY_PATH'), + string(credentialsId: 'release.gpg.passphrase', variable: 'RELEASE_GPG_PASSPHRASE')]) { + sshagent(['ed25519.Hibernate-CI.github.com', 'hibernate.filemgmt.jboss.org', 'hibernate-ci.frs.sourceforge.net']) { + sh 'cat $HOME/.ssh/config' + sh 'git clone https://github.com/hibernate/hibernate-noorm-release-scripts.git' + env.RELEASE_GPG_HOMEDIR = env.WORKSPACE_TMP + '/.gpg' + sh """ + bash -xe hibernate-noorm-release-scripts/release.sh ${params.RELEASE_DRY_RUN ? '-d' : ''} \ + search ${releaseVersion.toString()} ${developmentVersion.toString()} + """ + } + } + } + } + } + } + } + } + post { + always { + notifyBuildResult notifySuccessAfterSuccess: true, maintainers: 'yoann@hibernate.org' + } + } +} From 823885fa2d293fb544c1a8c248e94d0a1ea8fbe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Wed, 2 Feb 2022 11:47:11 +0100 Subject: [PATCH 2/5] HSEARCH-4472 Abort previous non-release builds on the same branch/PR when starting a new build --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 7c54d6e1cf0..33e966de416 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -226,6 +226,7 @@ stage('Configure') { buildDiscarder( logRotator(daysToKeepStr: '90') ), + disableConcurrentBuilds(abortPrevious: true), pipelineTriggers( // HSEARCH-3417: do not add snapshotDependencies() here, this was known to cause problems. [ From d2cebab1f08d5769ac5a0a9bd0d8160cac0b910c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Fri, 4 Feb 2022 14:07:46 +0100 Subject: [PATCH 3/5] HSEARCH-3244 Use name of the locked Jenkins resource as the Elasticsearch URI in AWS Elasticsearch tests So that we can get rid of environment variables defined globally in Jenkins. --- Jenkinsfile | 40 +++++++++------------------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 33e966de416..1394755c0a3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -198,13 +198,13 @@ stage('Configure') { ], esAws: [ new EsAwsBuildEnvironment(version: '2.3', mavenProfile: 'elasticsearch-2.2', - condition: TestCondition.AFTER_MERGE), + condition: TestCondition.ON_DEMAND), new EsAwsBuildEnvironment(version: '5.1', mavenProfile: 'elasticsearch-5.0', - condition: TestCondition.AFTER_MERGE), + condition: TestCondition.ON_DEMAND), new EsAwsBuildEnvironment(version: '5.3', mavenProfile: 'elasticsearch-5.2', - condition: TestCondition.AFTER_MERGE), + condition: TestCondition.ON_DEMAND), new EsAwsBuildEnvironment(version: '5.5', mavenProfile: 'elasticsearch-5.2', - condition: TestCondition.AFTER_MERGE), + condition: TestCondition.ON_DEMAND), new EsAwsBuildEnvironment(version: '5.6', mavenProfile: 'elasticsearch-5.2', condition: TestCondition.AFTER_MERGE) ] @@ -275,20 +275,6 @@ Some useful filters: 'default', 'jdk', 'jdk-10', 'eclipse', 'postgresql', 'elast keepOnlyEnvironmentsFromSet(params.ENVIRONMENT_SET) } - environments.content.esAws.enabled.removeAll { buildEnv -> - buildEnv.endpointUrl = env.getProperty(buildEnv.endpointVariableName) - if (!buildEnv.endpointUrl) { - echo "Skipping test ${buildEnv.tag} because environment variable '${buildEnv.endpointVariableName}' is not defined." - return true - } - buildEnv.awsRegion = env.ES_AWS_REGION - if (!buildEnv.awsRegion) { - echo "Skipping test ${buildEnv.tag} because environment variable 'ES_AWS_REGION' is not defined." - return true - } - return false // Environment is fully defined, do not remove - } - // Determine whether ITs need to be run in the default build enableDefaultBuildIT = environments.content.any { key, envSet -> return envSet.enabled.contains(envSet.default) @@ -408,18 +394,15 @@ stage('Non-default environments') { // Test Elasticsearch integration with multiple versions in an AWS instance environments.content.esAws.enabled.each { EsAwsBuildEnvironment buildEnv -> - if (!buildEnv.endpointUrl) { - throw new IllegalStateException("Unexpected empty endpoint URL") - } - if (!buildEnv.awsRegion) { - throw new IllegalStateException("Unexpected empty AWS region") + if (!env.ES_AWS_REGION) { + throw new IllegalStateException("Environment variable ES_AWS_REGION is not set") } def awsCredentialsId = helper.configuration.file?.aws?.credentials if (!awsCredentialsId) { throw new IllegalStateException("Missing AWS credentials") } executions.put(buildEnv.tag, { - lock(label: buildEnv.lockedResourcesLabel) { + lock(label: buildEnv.lockedResourcesLabel, variable: 'LOCKED_RESOURCE_URI') { runBuildOnNode(NODE_PATTERN_BASE + '&&AWS') { helper.withMavenWorkspace { // WARNING: Make sure credentials are evaluated by sh, not Groovy. @@ -437,11 +420,11 @@ stage('Non-default environments') { mavenNonDefaultBuild buildEnv, """ \ clean install -pl org.hibernate:hibernate-search-integrationtest-elasticsearch \ ${toElasticsearchVersionArgs(buildEnv.mavenProfile, buildEnv.version)} \ - -Dtest.elasticsearch.host.url=$buildEnv.endpointUrl \ + -Dtest.elasticsearch.host.url=$env.LOCKED_RESOURCE_URI \ -Dtest.elasticsearch.host.aws.signing.enabled=true \ -Dtest.elasticsearch.host.aws.access_key=\${AWS_ACCESS_KEY_ID} \ -Dtest.elasticsearch.host.aws.secret_key=\${AWS_SECRET_ACCESS_KEY} \ - -Dtest.elasticsearch.host.aws.region=$buildEnv.awsRegion \ + -Dtest.elasticsearch.host.aws.region=$env.ES_AWS_REGION \ """ } } @@ -551,8 +534,6 @@ class EsLocalBuildEnvironment extends BuildEnvironment { class EsAwsBuildEnvironment extends BuildEnvironment { String version String mavenProfile - String endpointUrl = null - String awsRegion = null @Override String getTag() { "elasticsearch-aws-$version" } @Override @@ -562,9 +543,6 @@ class EsAwsBuildEnvironment extends BuildEnvironment { String getNameEmbeddableVersion() { version.replaceAll('\\.', '') } - String getEndpointVariableName() { - "ES_AWS_${nameEmbeddableVersion}_ENDPOINT" - } String getLockedResourcesLabel() { "es-aws-${nameEmbeddableVersion}" } From 7e0558ccdc34fd0bd09f415163358839097306fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Fri, 4 Feb 2022 14:32:03 +0100 Subject: [PATCH 4/5] HSEARCH-3244 Use clearer labels for lockable AWS resources on Jenkins --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1394755c0a3..10a4323f916 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -541,7 +541,7 @@ class EsAwsBuildEnvironment extends BuildEnvironment { null // No JDK needed for Elasticsearch: the Elasticsearch instance is remote. } String getNameEmbeddableVersion() { - version.replaceAll('\\.', '') + version.replaceAll('\\.', '-') } String getLockedResourcesLabel() { "es-aws-${nameEmbeddableVersion}" From 6195437c6948f6a08acfa012b31c4bab4773090d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Mon, 7 Feb 2022 18:01:27 +0100 Subject: [PATCH 5/5] Keep fewer builds in the Jenkins history --- Jenkinsfile | 4 +--- jenkins/release.groovy | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 10a4323f916..31cf10617f0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,8 +13,6 @@ import groovy.transform.Field @Library('hibernate-jenkins-pipeline-helpers@1.3') import org.hibernate.jenkins.pipeline.helpers.job.JobHelper import org.hibernate.jenkins.pipeline.helpers.alternative.AlternativeMultiMap -import org.hibernate.jenkins.pipeline.helpers.version.Version - /* * WARNING: DO NOT IMPORT LOCAL LIBRARIES HERE. * @@ -224,7 +222,7 @@ stage('Configure') { properties([ buildDiscarder( - logRotator(daysToKeepStr: '90') + logRotator(daysToKeepStr: '30', numToKeepStr: '10') ), disableConcurrentBuilds(abortPrevious: true), pipelineTriggers( diff --git a/jenkins/release.groovy b/jenkins/release.groovy index d1e2997dbbe..127f36eaf44 100644 --- a/jenkins/release.groovy +++ b/jenkins/release.groovy @@ -18,6 +18,7 @@ pipeline { jdk 'OpenJDK 8 Latest' } options { + buildDiscarder logRotator(daysToKeepStr: '30', numToKeepStr: '10') disableConcurrentBuilds(abortPrevious: false) } parameters {