diff --git a/Jenkinsfile b/Jenkinsfile index 0d5bcd778bb..b68746d7765 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,8 +13,6 @@ import groovy.transform.Field @Library('hibernate-jenkins-pipeline-helpers@1.4') import org.hibernate.jenkins.pipeline.helpers.job.JobHelper import org.hibernate.jenkins.pipeline.helpers.alternative.AlternativeMultiMap -import org.hibernate.jenkins.pipeline.helpers.version.Version - /* * WARNING: DO NOT IMPORT LOCAL LIBRARIES HERE. * @@ -56,14 +54,13 @@ import org.hibernate.jenkins.pipeline.helpers.version.Version * * #### Nexus deployment * - * This job includes two deployment modes: - * - * - A deployment of snapshot artifacts for every non-PR build on "primary" branches (main and maintenance branches). - * - A full release when starting the job with specific parameters. - * - * In the first case, the name of a Maven settings file must be provided in the job configuration file + * This job is only able to deploy snapshot artifacts, + * for every non-PR build on "primary" branches (main and maintenance branches), + * but the name of a Maven settings file must be provided in the job configuration file * (see below). * + * For actual releases, see jenkins/release.groovy. + * * #### AWS * * This job will trigger integration tests against an Elasticsearch service hosted on AWS. @@ -143,8 +140,7 @@ import org.hibernate.jenkins.pipeline.helpers.version.Version * deployment: * maven: * # String containing the ID of a Maven settings file registered using the config-file-provider Jenkins plugin. - * # The settings must provide credentials to the servers with ID - * # 'jboss-releases-repository' and 'jboss-snapshots-repository'. + * # The settings must provide credentials to the server with ID 'ossrh'. * settingsId: ... */ @@ -163,12 +159,8 @@ import org.hibernate.jenkins.pipeline.helpers.version.Version @Field boolean enableDefaultBuild = false @Field boolean enableDefaultBuildIT = false -@Field boolean performRelease = false @Field boolean deploySnapshot = false -@Field Version releaseVersion -@Field Version afterReleaseDevelopmentVersion - this.helper = new JobHelper(this) helper.runWithNotification { @@ -244,13 +236,11 @@ stage('Configure') { condition: TestCondition.BEFORE_MERGE, isDefault: true) ], - // Note that each of these environments will only be tested if the appropriate - // environment variable with the AWS ES Service URL is defined in CI. esAws: [ new EsAwsBuildEnvironment(version: '5.6', mavenProfile: 'elasticsearch-5.6', condition: TestCondition.AFTER_MERGE), new EsAwsBuildEnvironment(version: '6.0', mavenProfile: 'elasticsearch-6.0', - condition: TestCondition.AFTER_MERGE), + condition: TestCondition.ON_DEMAND), // ES 6.2, 6.3.0, 6.3.1 and 6.3.2 and below have a bug that prevents double-nested // sorts from working: https://github.com/elastic/elasticsearch/issues/32130 new EsAwsBuildEnvironment(version: '6.2', mavenProfile: 'elasticsearch-6.0', @@ -260,21 +250,21 @@ stage('Configure') { new EsAwsBuildEnvironment(version: '6.3', mavenProfile: 'elasticsearch-6.3', condition: TestCondition.ON_DEMAND), new EsAwsBuildEnvironment(version: '6.4', mavenProfile: 'elasticsearch-6.4', - condition: TestCondition.AFTER_MERGE), + condition: TestCondition.ON_DEMAND), new EsAwsBuildEnvironment(version: '6.5', mavenProfile: 'elasticsearch-6.4', - condition: TestCondition.AFTER_MERGE), + condition: TestCondition.ON_DEMAND), new EsAwsBuildEnvironment(version: '6.7', mavenProfile: 'elasticsearch-6.7', - condition: TestCondition.AFTER_MERGE), + condition: TestCondition.ON_DEMAND), new EsAwsBuildEnvironment(version: '6.8', mavenProfile: 'elasticsearch-6.8', - condition: TestCondition.AFTER_MERGE), + condition: TestCondition.ON_DEMAND), new EsAwsBuildEnvironment(version: '7.1', mavenProfile: 'elasticsearch-7.0', - condition: TestCondition.AFTER_MERGE), + condition: TestCondition.ON_DEMAND), new EsAwsBuildEnvironment(version: '7.4', mavenProfile: 'elasticsearch-7.3', - condition: TestCondition.AFTER_MERGE), + condition: TestCondition.ON_DEMAND), new EsAwsBuildEnvironment(version: '7.7', mavenProfile: 'elasticsearch-7.7', - condition: TestCondition.AFTER_MERGE), + condition: TestCondition.ON_DEMAND), new EsAwsBuildEnvironment(version: '7.8', mavenProfile: 'elasticsearch-7.8', - condition: TestCondition.AFTER_MERGE), + condition: TestCondition.ON_DEMAND), new EsAwsBuildEnvironment(version: '7.10', mavenProfile: 'elasticsearch-7.10', condition: TestCondition.AFTER_MERGE), @@ -300,8 +290,9 @@ stage('Configure') { properties([ buildDiscarder( - logRotator(daysToKeepStr: '90') + logRotator(daysToKeepStr: '30', numToKeepStr: '10') ), + disableConcurrentBuilds(abortPrevious: true), pipelineTriggers( // HSEARCH-3417: do not add snapshotDependencies() here, this was known to cause problems. [ @@ -318,7 +309,7 @@ DEFAULT SUPPORTED ALL""", description: """A set of environments that must be checked. -'AUTOMATIC' picks a different set of environments based on the branch name and whether a release is being performed. +'AUTOMATIC' picks a different set of environments based on the branch name. 'DEFAULT' means a single build with the default environment expected by the Maven configuration, while other options will trigger multiple Maven executions in different environments.""" ), @@ -330,30 +321,11 @@ while other options will trigger multiple Maven executions in different environm If this parameter is non-empty, ENVIRONMENT_SET will be ignored and environments whose tag matches the given regex will be checked. Some useful filters: 'default', 'jdk', 'jdk-10', 'eclipse', 'postgresql', 'elasticsearch-local-[5.6'. """ - ), - string( - name: 'RELEASE_VERSION', - defaultValue: '', - description: 'The version to be released, e.g. 5.10.0.Final. Setting this triggers a release.', - trim: true - ), - string( - name: 'RELEASE_DEVELOPMENT_VERSION', - defaultValue: '', - description: 'The next version to be used after the release, e.g. 5.10.0-SNAPSHOT.', - trim: true - ), - booleanParam( - name: 'RELEASE_DRY_RUN', - defaultValue: false, - description: 'If true, just simulate the release, without pushing any commits or tags, and without uploading any artifacts or documentation.' ) ]) ]) - performRelease = (params.RELEASE_VERSION ? true : false) - - if (!performRelease && helper.scmSource.branch.primary && !helper.scmSource.pullRequest) { + if (helper.scmSource.branch.primary && !helper.scmSource.pullRequest) { if (helper.configuration.file?.deployment?.maven?.settingsId) { deploySnapshot = true } @@ -369,20 +341,6 @@ Some useful filters: 'default', 'jdk', 'jdk-10', 'eclipse', 'postgresql', 'elast keepOnlyEnvironmentsFromSet(params.ENVIRONMENT_SET) } - environments.content.esAws.enabled.removeAll { buildEnv -> - buildEnv.endpointUris = env.getProperty(buildEnv.endpointVariableName) - if (!buildEnv.endpointUris) { - echo "Skipping test ${buildEnv.tag} because environment variable '${buildEnv.endpointVariableName}' is not defined." - return true - } - buildEnv.awsRegion = env.ES_AWS_REGION - if (!buildEnv.awsRegion) { - echo "Skipping test ${buildEnv.tag} because environment variable 'ES_AWS_REGION' is not defined." - return true - } - return false // Environment is fully defined, do not remove - } - // Determine whether ITs need to be run in the default build enableDefaultBuildIT = environments.content.any { key, envSet -> return envSet.enabled.contains(envSet.default) @@ -406,32 +364,8 @@ Resulting execution plan: enableDefaultBuild=$enableDefaultBuild enableDefaultBuildIT=$enableDefaultBuildIT environments=${environments.enabledAsString} - performRelease=$performRelease deploySnapshot=$deploySnapshot """ - - if (performRelease) { - releaseVersion = Version.parseReleaseVersion(params.RELEASE_VERSION) - echo "Inferred version family for the release to '$releaseVersion.family'" - - // Check that all the necessary parameters are set - if (!params.RELEASE_DEVELOPMENT_VERSION) { - throw new IllegalArgumentException( - "Missing value for parameter RELEASE_DEVELOPMENT_VERSION." + - " This parameter must be set when RELEASE_VERSION is set." - ) - } - if (!params.RELEASE_DRY_RUN && !helper.configuration.file?.deployment?.maven?.settingsId) { - throw new IllegalArgumentException( - "Missing deployment configuration in job configuration file." + - " Cannot deploy artifacts during the release." - ) - } - } - - if (params.RELEASE_DEVELOPMENT_VERSION) { - afterReleaseDevelopmentVersion = Version.parseDevelopmentVersion(params.RELEASE_DEVELOPMENT_VERSION) - } } stage('Default build') { @@ -590,11 +524,8 @@ stage('Non-default environments') { // Test Elasticsearch integration with multiple versions in an AWS instance environments.content.esAws.enabled.each { EsAwsBuildEnvironment buildEnv -> - if (!buildEnv.endpointUris) { - throw new IllegalStateException("Unexpected empty endpoint URI") - } - if (!buildEnv.awsRegion) { - throw new IllegalStateException("Unexpected empty AWS region") + if (!env.ES_AWS_REGION) { + throw new IllegalStateException("Environment variable ES_AWS_REGION is not set") } def awsCredentialsId = null if (buildEnv.staticCredentials) { @@ -604,7 +535,7 @@ stage('Non-default environments') { } } executions.put(buildEnv.tag, { - lock(label: buildEnv.lockedResourcesLabel) { + lock(label: buildEnv.lockedResourcesLabel, variable: 'LOCKED_RESOURCE_URI') { runBuildOnNode(NODE_PATTERN_BASE + '&&AWS') { if (awsCredentialsId == null) { // By default, rely on credentials provided by the EC2 infrastructure @@ -627,9 +558,9 @@ stage('Non-default environments') { 'org.hibernate.search:hibernate-search-integrationtest-showcase-library' ].join(',')} \ ${toElasticsearchVersionArgs(buildEnv.mavenProfile, buildEnv.version)} \ - -Dtest.elasticsearch.connection.uris=$buildEnv.endpointUris \ + -Dtest.elasticsearch.connection.uris=$env.LOCKED_RESOURCE_URI \ -Dtest.elasticsearch.connection.aws.signing.enabled=true \ - -Dtest.elasticsearch.connection.aws.region=$buildEnv.awsRegion \ + -Dtest.elasticsearch.connection.aws.region=$env.ES_AWS_REGION \ """ } } @@ -662,9 +593,9 @@ stage('Non-default environments') { --fail-fast \ -pl org.hibernate.search:hibernate-search-integrationtest-backend-elasticsearch,org.hibernate.search:hibernate-search-integrationtest-showcase-library \ ${toElasticsearchVersionArgs(buildEnv.mavenProfile, buildEnv.version)} \ - -Dtest.elasticsearch.connection.uris=$buildEnv.endpointUris \ + -Dtest.elasticsearch.connection.uris=$env.LOCKED_RESOURCE_URI \ -Dtest.elasticsearch.connection.aws.signing.enabled=true \ - -Dtest.elasticsearch.connection.aws.region=$buildEnv.awsRegion \ + -Dtest.elasticsearch.connection.aws.region=$env.ES_AWS_REGION \ -Dtest.elasticsearch.connection.aws.credentials.type=static \ -Dtest.elasticsearch.connection.aws.credentials.access_key_id=\${AWS_ACCESS_KEY_ID} \ -Dtest.elasticsearch.connection.aws.credentials.secret_access_key=\${AWS_SECRET_ACCESS_KEY} \ @@ -687,36 +618,6 @@ stage('Non-default environments') { } } -stage('Deploy') { - if (deploySnapshot) { - // TODO delay the release to this stage? This would require using staging repositories for snapshots, not sure it's possible. - echo "Already deployed snapshot as part of the 'Default build' stage." - } - else if (performRelease) { - echo "Performing full release for version ${releaseVersion.toString()}" - runBuildOnNode { - withMavenWorkspace(mavenSettingsConfig: params.RELEASE_DRY_RUN ? null : helper.configuration.file.deployment.maven.settingsId) { - configFileProvider([configFile(fileId: 'release.config.ssh', targetLocation: env.HOME + '/.ssh/config')]) { - withCredentials([file(credentialsId: 'release.gpg.private-key', variable: 'RELEASE_GPG_PRIVATE_KEY_PATH'), - string(credentialsId: 'release.gpg.passphrase', variable: 'RELEASE_GPG_PASSPHRASE')]) { - sshagent(['ed25519.Hibernate-CI.github.com', 'hibernate.filemgmt.jboss.org', 'hibernate-ci.frs.sourceforge.net']) { - sh 'cat $HOME/.ssh/config' - sh "git clone https://github.com/hibernate/hibernate-noorm-release-scripts.git" - env.RELEASE_GPG_HOMEDIR = env.WORKSPACE_TMP + '/.gpg' - sh "bash -xe hibernate-noorm-release-scripts/release.sh ${params.RELEASE_DRY_RUN ? '-d' : ''} search ${releaseVersion.toString()} ${afterReleaseDevelopmentVersion.toString()}" - } - } - } - } - } - } - else { - echo "Skipping deployment" - helper.markStageSkipped() - return - } -} - } // End of helper.runWithNotification // Job-specific helpers @@ -782,16 +683,11 @@ class EsLocalBuildEnvironment extends BuildEnvironment { class EsAwsBuildEnvironment extends BuildEnvironment { String version String mavenProfile - String endpointUris = null - String awsRegion = null boolean staticCredentials = false @Override String getTag() { "elasticsearch-aws-$version" + (staticCredentials ? "-credentials-static" : "") } String getNameEmbeddableVersion() { - version.replaceAll('\\.', '') - } - String getEndpointVariableName() { - "ES_AWS_${nameEmbeddableVersion}_ENDPOINT" + version.replaceAll('\\.', '-') } String getLockedResourcesLabel() { "es-aws-${nameEmbeddableVersion}" @@ -831,9 +727,7 @@ void keepOnlyEnvironmentsFromSet(String environmentSetName) { enableOptional = true break case 'AUTOMATIC': - if (params.RELEASE_VERSION) { - echo "Releasing version '$params.RELEASE_VERSION'." - } else if (helper.scmSource.pullRequest) { + if (helper.scmSource.pullRequest) { echo "Building pull request '$helper.scmSource.pullRequest.id'" enableDefaultEnv = true enableBeforeMergeEnvs = true diff --git a/jenkins/performance-elasticsearch.groovy b/jenkins/performance-elasticsearch.groovy index f040ca5b9a7..db85d189f16 100644 --- a/jenkins/performance-elasticsearch.groovy +++ b/jenkins/performance-elasticsearch.groovy @@ -24,7 +24,7 @@ import org.hibernate.jenkins.pipeline.helpers.job.JobHelper @Field JobHelper helper -@Field EsAwsBuildEnvironment esAwsBuildEnv = new EsAwsBuildEnvironment(version: "7.8") +@Field EsAwsBuildEnvironment esAwsBuildEnv = new EsAwsBuildEnvironment(version: "7.10") this.helper = new JobHelper(this) @@ -52,22 +52,13 @@ stage('Configure') { helper.generateNotificationProperty() ]) - esAwsBuildEnv.endpointUris = env.getProperty(esAwsBuildEnv.endpointVariableName) - if (!esAwsBuildEnv.endpointUris) { - throw new IllegalStateException( - "Cannot run performance test because environment variable '$esAwsBuildEnv.endpointVariableName' is not defined." - ) - } - esAwsBuildEnv.awsRegion = env.ES_AWS_REGION - if (!esAwsBuildEnv.awsRegion) { - throw new IllegalStateException( - "Cannot run performance test because environment variable 'ES_AWS_REGION' is not defined." - ) + if (!env.ES_AWS_REGION) { + throw new IllegalStateException("Environment variable ES_AWS_REGION is not set") } } -lock(label: esAwsBuildEnv.lockedResourcesLabel) { - node ('Performance') { +lock(label: esAwsBuildEnv.lockedResourcesLabel, variable: 'LOCKED_RESOURCE_URI') { + node (PERFORMANCE_NODE_PATTERN) { stage ('Checkout') { checkout scm } @@ -101,9 +92,9 @@ lock(label: esAwsBuildEnv.lockedResourcesLabel) { sh """ \ java \ -jar benchmarks.jar \ - -jvmArgsAppend -Duris=$esAwsBuildEnv.endpointUris \ + -jvmArgsAppend -Duris=$env.LOCKED_RESOURCE_URI \ -jvmArgsAppend -Daws.signing.enabled=true \ - -jvmArgsAppend -Daws.region=$esAwsBuildEnv.awsRegion \ + -jvmArgsAppend -Daws.region=$env.ES_AWS_REGION \ -jvmArgsAppend -Daws.credentials.type=static \ -jvmArgsAppend -Daws.credentials.access_key_id=$AWS_ACCESS_KEY_ID \ -jvmArgsAppend -Daws.credentials.secret_access_key=$AWS_SECRET_ACCESS_KEY \ @@ -124,13 +115,8 @@ lock(label: esAwsBuildEnv.lockedResourcesLabel) { class EsAwsBuildEnvironment { String version - String endpointUris = null - String awsRegion = null String getNameEmbeddableVersion() { - version.replaceAll('\\.', '') - } - String getEndpointVariableName() { - "ES_AWS_${nameEmbeddableVersion}_ENDPOINT" + version.replaceAll('\\.', '-') } String getLockedResourcesLabel() { "es-aws-${nameEmbeddableVersion}" diff --git a/jenkins/release.groovy b/jenkins/release.groovy new file mode 100644 index 00000000000..67b0d9bdf17 --- /dev/null +++ b/jenkins/release.groovy @@ -0,0 +1,93 @@ +/* + * Hibernate Search, full-text search for your domain model + * + * License: GNU Lesser General Public License (LGPL), version 2.1 or later + * See the lgpl.txt file in the root directory or . + */ + +@Library('hibernate-jenkins-pipeline-helpers@1.5') _ + +import org.hibernate.jenkins.pipeline.helpers.version.Version + +pipeline { + agent { + label 'Worker&&Containers' + } + tools { + maven 'Apache Maven 3.8' + jdk 'OpenJDK 11 Latest' + } + options { + buildDiscarder logRotator(daysToKeepStr: '30', numToKeepStr: '10') + disableConcurrentBuilds(abortPrevious: false) + } + parameters { + string( + name: 'RELEASE_VERSION', + defaultValue: '', + description: 'The version to be released, e.g. 6.2.0.Final.', + trim: true + ) + string( + name: 'DEVELOPMENT_VERSION', + defaultValue: '', + description: 'The next version to be used after the release, e.g. 6.2.0-SNAPSHOT.', + trim: true + ) + booleanParam( + name: 'RELEASE_DRY_RUN', + defaultValue: false, + description: 'If true, just simulate the release, without pushing any commits or tags, and without uploading any artifacts or documentation.' + ) + } + stages { + stage('Release') { + when { + beforeAgent true + // Releases must be triggered explicitly + // This is just for safety; normally the Jenkins job for this pipeline + // should be configured to "Suppress automatic SCM triggering" + // See https://stackoverflow.com/questions/58259326/prevent-jenkins-multibranch-pipeline-from-triggering-builds-for-new-branches + triggeredBy cause: "UserIdCause" + } + steps { + script { + // Check that all the necessary parameters are set + if (!params.RELEASE_VERSION) { + throw new IllegalArgumentException("Missing value for parameter RELEASE_VERSION.") + } + if (!params.DEVELOPMENT_VERSION) { + throw new IllegalArgumentException("Missing value for parameter DEVELOPMENT_VERSION.") + } + + def releaseVersion = Version.parseReleaseVersion(params.RELEASE_VERSION) + def developmentVersion = Version.parseDevelopmentVersion(params.DEVELOPMENT_VERSION) + echo "Performing full release for version ${releaseVersion.toString()}" + + withMaven(mavenSettingsConfig: params.RELEASE_DRY_RUN ? null : 'ci-hibernate.deploy.settings.maven', + mavenLocalRepo: env.WORKSPACE_TMP + '.m2repository') { + configFileProvider([configFile(fileId: 'release.config.ssh', targetLocation: env.HOME + '/.ssh/config')]) { + withCredentials([file(credentialsId: 'release.gpg.private-key', variable: 'RELEASE_GPG_PRIVATE_KEY_PATH'), + string(credentialsId: 'release.gpg.passphrase', variable: 'RELEASE_GPG_PASSPHRASE')]) { + sshagent(['ed25519.Hibernate-CI.github.com', 'hibernate.filemgmt.jboss.org', 'hibernate-ci.frs.sourceforge.net']) { + sh 'cat $HOME/.ssh/config' + sh 'git clone https://github.com/hibernate/hibernate-noorm-release-scripts.git' + env.RELEASE_GPG_HOMEDIR = env.WORKSPACE_TMP + '/.gpg' + sh """ + bash -xe hibernate-noorm-release-scripts/release.sh ${params.RELEASE_DRY_RUN ? '-d' : ''} \ + search ${releaseVersion.toString()} ${developmentVersion.toString()} + """ + } + } + } + } + } + } + } + } + post { + always { + notifyBuildResult notifySuccessAfterSuccess: true, maintainers: 'yoann@hibernate.org' + } + } +}