Skip to content

Backport HSEARCH-3244 + HSEARCH-4472 to branch 5.10 - Jenkinsfile improvements #2883

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Feb 7, 2022
Merged

Backport HSEARCH-3244 + HSEARCH-4472 to branch 5.10 - Jenkinsfile improvements #2883

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
99dc862
HSEARCH-4472 Move the release job to a separate Jenkinsfile
yrodiere Feb 2, 2022
823885f
HSEARCH-4472 Abort previous non-release builds on the same branch/PR …
yrodiere Feb 2, 2022
d2cebab
HSEARCH-3244 Use name of the locked Jenkins resource as the Elasticse…
yrodiere Feb 4, 2022
7e0558c
HSEARCH-3244 Use clearer labels for lockable AWS resources on Jenkins
yrodiere Feb 4, 2022
6195437
Keep fewer builds in the Jenkins history
yrodiere Feb 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 21 additions & 113 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@ import groovy.transform.Field
@Library('hibernate-jenkins-pipeline-helpers@1.3')
import org.hibernate.jenkins.pipeline.helpers.job.JobHelper
import org.hibernate.jenkins.pipeline.helpers.alternative.AlternativeMultiMap
import org.hibernate.jenkins.pipeline.helpers.version.Version

/*
* WARNING: DO NOT IMPORT LOCAL LIBRARIES HERE.
*
Expand Down Expand Up @@ -56,14 +54,13 @@ import org.hibernate.jenkins.pipeline.helpers.version.Version
*
* #### Nexus deployment
*
* This job includes two deployment modes:
*
* - A deployment of snapshot artifacts for every non-PR build on "primary" branches (main and maintenance branches).
* - A full release when starting the job with specific parameters.
*
* In the first case, the name of a Maven settings file must be provided in the job configuration file
* This job is only able to deploy snapshot artifacts,
* for every non-PR build on "primary" branches (main and maintenance branches),
* but the name of a Maven settings file must be provided in the job configuration file
* (see below).
*
* For actual releases, see jenkins/release.groovy.
*
* #### AWS
*
* This job will trigger integration tests against an Elasticsearch service hosted on AWS.
Expand Down Expand Up @@ -130,8 +127,7 @@ import org.hibernate.jenkins.pipeline.helpers.version.Version
* deployment:
* maven:
* # String containing the ID of a Maven settings file registered using the config-file-provider Jenkins plugin.
* # The settings must provide credentials to the servers with ID
* # 'jboss-releases-repository' and 'jboss-snapshots-repository'.
* # The settings must provide credentials to the server with ID 'ossrh'.
* settingsId: ...
*/

Expand All @@ -149,12 +145,8 @@ import org.hibernate.jenkins.pipeline.helpers.version.Version

@Field boolean enableDefaultBuild = false
@Field boolean enableDefaultBuildIT = false
@Field boolean performRelease = false
@Field boolean deploySnapshot = false

@Field Version releaseVersion
@Field Version afterReleaseDevelopmentVersion

this.helper = new JobHelper(this)

helper.runWithNotification {
Expand Down Expand Up @@ -204,13 +196,13 @@ stage('Configure') {
],
esAws: [
new EsAwsBuildEnvironment(version: '2.3', mavenProfile: 'elasticsearch-2.2',
condition: TestCondition.AFTER_MERGE),
condition: TestCondition.ON_DEMAND),
new EsAwsBuildEnvironment(version: '5.1', mavenProfile: 'elasticsearch-5.0',
condition: TestCondition.AFTER_MERGE),
condition: TestCondition.ON_DEMAND),
new EsAwsBuildEnvironment(version: '5.3', mavenProfile: 'elasticsearch-5.2',
condition: TestCondition.AFTER_MERGE),
condition: TestCondition.ON_DEMAND),
new EsAwsBuildEnvironment(version: '5.5', mavenProfile: 'elasticsearch-5.2',
condition: TestCondition.AFTER_MERGE),
condition: TestCondition.ON_DEMAND),
new EsAwsBuildEnvironment(version: '5.6', mavenProfile: 'elasticsearch-5.2',
condition: TestCondition.AFTER_MERGE)
]
Expand All @@ -230,8 +222,9 @@ stage('Configure') {

properties([
buildDiscarder(
logRotator(daysToKeepStr: '90')
logRotator(daysToKeepStr: '30', numToKeepStr: '10')
),
disableConcurrentBuilds(abortPrevious: true),
pipelineTriggers(
// HSEARCH-3417: do not add snapshotDependencies() here, this was known to cause problems.
[
Expand All @@ -248,7 +241,7 @@ DEFAULT
SUPPORTED
ALL""",
description: """A set of environments that must be checked.
'AUTOMATIC' picks a different set of environments based on the branch name and whether a release is being performed.
'AUTOMATIC' picks a different set of environments based on the branch name.
'DEFAULT' means a single build with the default environment expected by the Maven configuration,
while other options will trigger multiple Maven executions in different environments."""
),
Expand All @@ -260,30 +253,11 @@ while other options will trigger multiple Maven executions in different environm
If this parameter is non-empty, ENVIRONMENT_SET will be ignored and environments whose tag matches the given regex will be checked.
Some useful filters: 'default', 'jdk', 'jdk-10', 'eclipse', 'postgresql', 'elasticsearch-local-[5.6'.
"""
),
string(
name: 'RELEASE_VERSION',
defaultValue: '',
description: 'The version to be released, e.g. 5.10.0.Final. Setting this triggers a release.',
trim: true
),
string(
name: 'RELEASE_DEVELOPMENT_VERSION',
defaultValue: '',
description: 'The next version to be used after the release, e.g. 5.10.0-SNAPSHOT.',
trim: true
),
booleanParam(
name: 'RELEASE_DRY_RUN',
defaultValue: false,
description: 'If true, just simulate the release, without pushing any commits or tags, and without uploading any artifacts or documentation.'
)
])
])

performRelease = (params.RELEASE_VERSION ? true : false)

if (!performRelease && helper.scmSource.branch.primary && !helper.scmSource.pullRequest) {
if (helper.scmSource.branch.primary && !helper.scmSource.pullRequest) {
if (helper.configuration.file?.deployment?.maven?.settingsId) {
deploySnapshot = true
}
Expand All @@ -299,20 +273,6 @@ Some useful filters: 'default', 'jdk', 'jdk-10', 'eclipse', 'postgresql', 'elast
keepOnlyEnvironmentsFromSet(params.ENVIRONMENT_SET)
}

environments.content.esAws.enabled.removeAll { buildEnv ->
buildEnv.endpointUrl = env.getProperty(buildEnv.endpointVariableName)
if (!buildEnv.endpointUrl) {
echo "Skipping test ${buildEnv.tag} because environment variable '${buildEnv.endpointVariableName}' is not defined."
return true
}
buildEnv.awsRegion = env.ES_AWS_REGION
if (!buildEnv.awsRegion) {
echo "Skipping test ${buildEnv.tag} because environment variable 'ES_AWS_REGION' is not defined."
return true
}
return false // Environment is fully defined, do not remove
}

// Determine whether ITs need to be run in the default build
enableDefaultBuildIT = environments.content.any { key, envSet ->
return envSet.enabled.contains(envSet.default)
Expand All @@ -336,32 +296,8 @@ Resulting execution plan:
enableDefaultBuild=$enableDefaultBuild
enableDefaultBuildIT=$enableDefaultBuildIT
environments=${environments.enabledAsString}
performRelease=$performRelease
deploySnapshot=$deploySnapshot
"""

if (performRelease) {
releaseVersion = Version.parseReleaseVersion(params.RELEASE_VERSION)
echo "Inferred version family for the release to '$releaseVersion.family'"

// Check that all the necessary parameters are set
if (!params.RELEASE_DEVELOPMENT_VERSION) {
throw new IllegalArgumentException(
"Missing value for parameter RELEASE_DEVELOPMENT_VERSION." +
" This parameter must be set when RELEASE_VERSION is set."
)
}
if (!params.RELEASE_DRY_RUN && !helper.configuration.file?.deployment?.maven?.settingsId) {
throw new IllegalArgumentException(
"Missing deployment configuration in job configuration file." +
" Cannot deploy artifacts during the release."
)
}
}

if (params.RELEASE_DEVELOPMENT_VERSION) {
afterReleaseDevelopmentVersion = Version.parseDevelopmentVersion(params.RELEASE_DEVELOPMENT_VERSION)
}
}

stage('Default build') {
Expand Down Expand Up @@ -456,18 +392,15 @@ stage('Non-default environments') {

// Test Elasticsearch integration with multiple versions in an AWS instance
environments.content.esAws.enabled.each { EsAwsBuildEnvironment buildEnv ->
if (!buildEnv.endpointUrl) {
throw new IllegalStateException("Unexpected empty endpoint URL")
}
if (!buildEnv.awsRegion) {
throw new IllegalStateException("Unexpected empty AWS region")
if (!env.ES_AWS_REGION) {
throw new IllegalStateException("Environment variable ES_AWS_REGION is not set")
}
def awsCredentialsId = helper.configuration.file?.aws?.credentials
if (!awsCredentialsId) {
throw new IllegalStateException("Missing AWS credentials")
}
executions.put(buildEnv.tag, {
lock(label: buildEnv.lockedResourcesLabel) {
lock(label: buildEnv.lockedResourcesLabel, variable: 'LOCKED_RESOURCE_URI') {
runBuildOnNode(NODE_PATTERN_BASE + '&&AWS') {
helper.withMavenWorkspace {
// WARNING: Make sure credentials are evaluated by sh, not Groovy.
Expand All @@ -485,11 +418,11 @@ stage('Non-default environments') {
mavenNonDefaultBuild buildEnv, """ \
clean install -pl org.hibernate:hibernate-search-integrationtest-elasticsearch \
${toElasticsearchVersionArgs(buildEnv.mavenProfile, buildEnv.version)} \
-Dtest.elasticsearch.host.url=$buildEnv.endpointUrl \
-Dtest.elasticsearch.host.url=$env.LOCKED_RESOURCE_URI \
-Dtest.elasticsearch.host.aws.signing.enabled=true \
-Dtest.elasticsearch.host.aws.access_key=\${AWS_ACCESS_KEY_ID} \
-Dtest.elasticsearch.host.aws.secret_key=\${AWS_SECRET_ACCESS_KEY} \
-Dtest.elasticsearch.host.aws.region=$buildEnv.awsRegion \
-Dtest.elasticsearch.host.aws.region=$env.ES_AWS_REGION \
"""
}
}
Expand Down Expand Up @@ -517,24 +450,6 @@ stage('Deploy') {
}
}
}
else if (performRelease) {
echo "Performing full release for version ${releaseVersion.toString()}"
runBuildOnNode {
helper.withMavenWorkspace(mavenSettingsConfig: params.RELEASE_DRY_RUN ? null : helper.configuration.file.deployment.maven.settingsId) {
configFileProvider([configFile(fileId: 'release.config.ssh', targetLocation: env.HOME + '/.ssh/config')]) {
withCredentials([file(credentialsId: 'release.gpg.private-key', variable: 'RELEASE_GPG_PRIVATE_KEY_PATH'),
string(credentialsId: 'release.gpg.passphrase', variable: 'RELEASE_GPG_PASSPHRASE')]) {
sshagent(['ed25519.Hibernate-CI.github.com', 'hibernate.filemgmt.jboss.org', 'hibernate-ci.frs.sourceforge.net']) {
sh 'cat $HOME/.ssh/config'
sh "git clone https://github.com/hibernate/hibernate-noorm-release-scripts.git"
env.RELEASE_GPG_HOMEDIR = env.WORKSPACE_TMP + '/.gpg'
sh "bash -xe hibernate-noorm-release-scripts/release.sh ${params.RELEASE_DRY_RUN ? '-d' : ''} search ${releaseVersion.toString()} ${afterReleaseDevelopmentVersion.toString()}"
}
}
}
}
}
}
else {
echo "Skipping deployment"
helper.markStageSkipped()
Expand Down Expand Up @@ -617,19 +532,14 @@ class EsLocalBuildEnvironment extends BuildEnvironment {
class EsAwsBuildEnvironment extends BuildEnvironment {
String version
String mavenProfile
String endpointUrl = null
String awsRegion = null
@Override
String getTag() { "elasticsearch-aws-$version" }
@Override
String getElasticsearchJdkTool(def allEnvironments) {
null // No JDK needed for Elasticsearch: the Elasticsearch instance is remote.
}
String getNameEmbeddableVersion() {
version.replaceAll('\\.', '')
}
String getEndpointVariableName() {
"ES_AWS_${nameEmbeddableVersion}_ENDPOINT"
version.replaceAll('\\.', '-')
}
String getLockedResourcesLabel() {
"es-aws-${nameEmbeddableVersion}"
Expand Down Expand Up @@ -672,9 +582,7 @@ void keepOnlyEnvironmentsFromSet(String environmentSetName) {
enableOptional = true
break
case 'AUTOMATIC':
if (params.RELEASE_VERSION) {
echo "Releasing version '$params.RELEASE_VERSION'."
} else if (helper.scmSource.pullRequest) {
if (helper.scmSource.pullRequest) {
echo "Building pull request '$helper.scmSource.pullRequest.id'"
enableDefaultEnv = true
enableBeforeMergeEnvs = true
Expand Down
93 changes: 93 additions & 0 deletions jenkins/release.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
* Hibernate Search, full-text search for your domain model
*
* License: GNU Lesser General Public License (LGPL), version 2.1 or later
* See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
*/

@Library('hibernate-jenkins-pipeline-helpers@1.5') _

import org.hibernate.jenkins.pipeline.helpers.version.Version

pipeline {
agent {
label 'Worker&&Containers'
}
tools {
maven 'Apache Maven 3.8'
jdk 'OpenJDK 8 Latest'
}
options {
buildDiscarder logRotator(daysToKeepStr: '30', numToKeepStr: '10')
disableConcurrentBuilds(abortPrevious: false)
}
parameters {
string(
name: 'RELEASE_VERSION',
defaultValue: '',
description: 'The version to be released, e.g. 6.2.0.Final.',
trim: true
)
string(
name: 'DEVELOPMENT_VERSION',
defaultValue: '',
description: 'The next version to be used after the release, e.g. 6.2.0-SNAPSHOT.',
trim: true
)
booleanParam(
name: 'RELEASE_DRY_RUN',
defaultValue: false,
description: 'If true, just simulate the release, without pushing any commits or tags, and without uploading any artifacts or documentation.'
)
}
stages {
stage('Release') {
when {
beforeAgent true
// Releases must be triggered explicitly
// This is just for safety; normally the Jenkins job for this pipeline
// should be configured to "Suppress automatic SCM triggering"
// See https://stackoverflow.com/questions/58259326/prevent-jenkins-multibranch-pipeline-from-triggering-builds-for-new-branches
triggeredBy cause: "UserIdCause"
}
steps {
script {
// Check that all the necessary parameters are set
if (!params.RELEASE_VERSION) {
throw new IllegalArgumentException("Missing value for parameter RELEASE_VERSION.")
}
if (!params.DEVELOPMENT_VERSION) {
throw new IllegalArgumentException("Missing value for parameter DEVELOPMENT_VERSION.")
}

def releaseVersion = Version.parseReleaseVersion(params.RELEASE_VERSION)
def developmentVersion = Version.parseDevelopmentVersion(params.DEVELOPMENT_VERSION)
echo "Performing full release for version ${releaseVersion.toString()}"

withMaven(mavenSettingsConfig: params.RELEASE_DRY_RUN ? null : 'ci-hibernate.deploy.settings.maven',
mavenLocalRepo: env.WORKSPACE_TMP + '.m2repository') {
configFileProvider([configFile(fileId: 'release.config.ssh', targetLocation: env.HOME + '/.ssh/config')]) {
withCredentials([file(credentialsId: 'release.gpg.private-key', variable: 'RELEASE_GPG_PRIVATE_KEY_PATH'),
string(credentialsId: 'release.gpg.passphrase', variable: 'RELEASE_GPG_PASSPHRASE')]) {
sshagent(['ed25519.Hibernate-CI.github.com', 'hibernate.filemgmt.jboss.org', 'hibernate-ci.frs.sourceforge.net']) {
sh 'cat $HOME/.ssh/config'
sh 'git clone https://github.com/hibernate/hibernate-noorm-release-scripts.git'
env.RELEASE_GPG_HOMEDIR = env.WORKSPACE_TMP + '/.gpg'
sh """
bash -xe hibernate-noorm-release-scripts/release.sh ${params.RELEASE_DRY_RUN ? '-d' : ''} \
search ${releaseVersion.toString()} ${developmentVersion.toString()}
"""
}
}
}
}
}
}
}
}
post {
always {
notifyBuildResult notifySuccessAfterSuccess: true, maintainers: 'yoann@hibernate.org'
}
}
}