From 0a7b2b7f90b160577ce6d5e842590c8468dd4cf2 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 16 Mar 2022 12:11:52 +0100 Subject: [PATCH 01/16] WIP --- .ci/package-storage-publish.groovy | 100 ++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 3 deletions(-) diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy index 82a69cacef..56a24b1d35 100644 --- a/.ci/package-storage-publish.groovy +++ b/.ci/package-storage-publish.groovy @@ -6,11 +6,23 @@ pipeline { agent { label 'ubuntu-20 && immutable' } environment { REPO = "elastic-package" + REPO_BUILD_TAG = "${env.REPO}/${env.BUILD_TAG}" BASE_DIR="src/github.com/elastic/elastic-package" JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba" GITHUB_TOKEN_CREDENTIALS = "2a9602aa-ab9f-4e52-baf3-b71ca88469c7" PIPELINE_LOG_LEVEL='INFO' + + // Signing + INFRA_SIGNING_BUCKET_NAME = 'internal-ci-artifacts' + INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER = "${env.REPO_BUILD_TAG}/signed-artifacts" + INFRA_SIGNING_BUCKET_ARTIFACTS_PATH = "gs://${env.INFRA_SIGNING_BUCKET_NAME}/${env.REPO_BUILD_TAG}" + INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH = "gs://${env.INFRA_SIGNING_BUCKET_NAME}/${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}" + + // Publishing + PACKAGE_STORAGE_UPLOADER_CREDENTIALS = 'upload-package-to-package-storage' + PACKAGE_STORAGE_UPLOADER_GCP_SERVICE_ACCOUNT = 'secret/gce/elastic-bekitzur/service-account/package-storage-uploader' + PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH = "gs://elastic-bekitzur-package-storage-internal/queue-publishing/${env.REPO_BUILD_TAG}" } options { timeout(time: 1, unit: 'HOURS') @@ -31,7 +43,30 @@ pipeline { pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ]) deleteDir() gitCheckout(basedir: "${BASE_DIR}") - stash allowEmpty: true, name: 'source', useDefaultExcludes: false + stash(allowEmpty: true, name: 'source', useDefaultExcludes: false) + } + } + stage('Build package') { + steps { + cleanup() + useElasticPackage() + dir("${BASE_DIR}/test/packages/package_storage_candidate") { + sh(label: 'Build package',script: "elastic-package build") + } + stash(allowEmpty: true, name: 'build-package', useDefaultExcludes: false) + } + } + stage('Sign package') { + steps { + cleanup(source: 'build-package') + signArtifactsWithElastic('build/integrations') + stash(allowEmpty: true, name: 'sign-package', useDefaultExcludes: false) + } + } + stage('Publish package') { + steps { + cleanup(source: 'sign-package') + publishToPackageStorage('build/integrations') } } } @@ -42,9 +77,68 @@ pipeline { } } -def cleanup(){ +def useElasticPackage() { + withGoEnv() { + dir("${BASE_DIR}") { + sh(label: 'Install elastic-package',script: "make install") + // sh(label: 'Install elastic-package', script: 'go build github.com/elastic/elastic-package') + } + } +} + +def signArtifactsWithElastic(artifactsPath) { + dir("${BASE_DIR}") { + googleStorageUpload(bucket: env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH, + credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS, + pathPrefix: artifactsPath + '/', + pattern: artifactsPath + '/*.zip', + sharedPublicly: false, + showInline: true) + withCredentials([string(credentialsId: env.JOB_SIGNING_CREDENTIALS, variable: 'TOKEN')]) { + triggerRemoteJob(auth: CredentialsAuth(credentials: 'local-readonly-api-token'), + job: 'https://internal-ci.elastic.co/job/elastic+unified-release+master+sign-artifacts-with-gpg', + token: TOKEN, + parameters: "gcs_input_path=${env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}", + useCrumbCache: false, + useJobInfoCache: false) + } + googleStorageDownload(bucketUri: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH}/*", + credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS, + localDirectory: signaturesDestinationPath + '/', + pathPrefix: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}") + sh(label: 'Rename .asc to .sig', script: 'for f in ' + artifactsPath + '/*.asc; do mv "$f" "${f%.asc}.sig"; done') + } +} + +def publishToPackageStorage(artifactsPath) { + dir("${BASE_DIR}/${artifactsPath}") { + withGCPEnv(secret: env.PACKAGE_STORAGE_UPLOADER_GCP_SERVICE_ACCOUNT) { + withCredentials([string(credentialsId: env.PACKAGE_STORAGE_UPLOADER_CREDENTIALS, variable: 'TOKEN')]) { + findFiles()?.findAll{ it.name.endsWith('.zip') }?.collect{ it.name }?.sort()?.each { + def packageZip = it + sh(label: 'Upload package .zip file', script: "gsutil cp ${packageZip} ${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/") + sh(label: 'Upload package .sig file', script: "gsutil cp ${packageZip}.sig ${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/") + + triggerRemoteJob(auth: CredentialsAuth(credentials: 'local-readonly-api-token'), + job: 'https://internal-ci.elastic.co/job/package_storage/job/publishing-job-remote', + token: TOKEN, + parameters: """ + dry_run=true + gs_package_build_zip_path=${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/${packageZip} + gs_package_signature_path=${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/${packageZip}.sig + """, + useCrumbCache: true, + useJobInfoCache: true) + } + } + } + } +} + +def cleanup(Map args = [:]) { + def source = args.containsKey('source') ? args.source : 'source' dir("${BASE_DIR}"){ deleteDir() } - unstash 'source' + unstash source } \ No newline at end of file From e6ec52cf9dbc84528639f0f7dcf74240ecd69ffd Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 16 Mar 2022 12:29:04 +0100 Subject: [PATCH 02/16] Call the minimal pipeline --- .ci/Jenkinsfile | 71 +----------------------------- .ci/package-storage-publish.groovy | 2 + 2 files changed, 4 insertions(+), 69 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index cac4082d03..8b9e972258 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -22,21 +22,6 @@ pipeline { JOB_GCS_EXT_CREDENTIALS = 'beats-ci-gcs-plugin-file-credentials' ELASTIC_PACKAGE_GCP_SECRET = 'secret/observability-team/ci/service-account/elastic-package-gcp' ELASTIC_OBSERVABILITY_PROJECT_ID = 'elastic-observability' - - JOB_SIGNING_CREDENTIALS = 'sign-artifacts-with-gpg-job' - INTERNAL_CI_JOB_GCS_CREDENTIALS = 'internal-ci-gcs-plugin' - - REPO_BUILD_TAG = "${env.REPO}/${env.BUILD_TAG}" - INFRA_SIGNING_BUCKET_NAME = 'internal-ci-artifacts' - INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER = "${env.REPO_BUILD_TAG}/signed-artifacts" - INFRA_SIGNING_BUCKET_ARTIFACTS_PATH = "gs://${env.INFRA_SIGNING_BUCKET_NAME}/${env.REPO_BUILD_TAG}" - INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH = "gs://${env.INFRA_SIGNING_BUCKET_NAME}/${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}" - - INTEGRATIONS_SIGNATURES_PATH = 'build/integrations-elastic-signatures' // different path not to override signatures archived in the "build-zip" step - - PACKAGE_STORAGE_UPLOADER_CREDENTIALS = 'upload-package-to-package-storage' - PACKAGE_STORAGE_UPLOADER_GCP_SERVICE_ACCOUNT = 'secret/gce/elastic-bekitzur/service-account/package-storage-uploader' - PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH = "gs://elastic-bekitzur-package-storage-internal/queue-publishing/${env.REPO_BUILD_TAG}" } options { timeout(time: 1, unit: 'HOURS') @@ -165,64 +150,12 @@ def cleanup(){ def generateTestPublishToPackageStorageStage() { return { - withNode(labels: "ubuntu-20 && immutable", sleepMax: 20, forceWorkspace: true) { - cleanup() - dir("${BASE_DIR}"){ - withMageEnv(){ - sh(label: 'Install elastic-package',script: "make install") - dir("test/packages/package-storage/package_storage_candidate") { - sh(label: 'Lint package',script: "elastic-package lint") - sh(label: 'Build zipped package',script: "elastic-package build --zip") - } - signArtifactsWithElastic('build/integrations', env.INTEGRATIONS_SIGNATURES_PATH) - - // Add the package candidate to the "queue-publishing" - withGCPEnv(secret: env.PACKAGE_STORAGE_UPLOADER_GCP_SERVICE_ACCOUNT) { - sh(label: 'Upload package .zip file', script: "gsutil cp ${env.INTEGRATIONS_SIGNATURES_PATH}/package_storage_candidate-0.0.1.zip ${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/") - sh(label: 'Upload package .sig file', script: "gsutil cp ${env.INTEGRATIONS_SIGNATURES_PATH}/package_storage_candidate-0.0.1.zip.sig ${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/") - } - - // Call the publishing job - withCredentials([string(credentialsId: env.PACKAGE_STORAGE_UPLOADER_CREDENTIALS, variable: 'TOKEN')]) { - triggerRemoteJob(auth: CredentialsAuth(credentials: 'local-readonly-api-token'), - job: 'https://internal-ci.elastic.co/job/package_storage/job/publishing-job-remote', - token: TOKEN, - parameters: """ - dry_run=true - gs_package_build_zip_path=${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/package_storage_candidate-0.0.1.zip - gs_package_signature_path=${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/package_storage_candidate-0.0.1.zip.sig - """, - useCrumbCache: false, - useJobInfoCache: false) - } - } - } + withNode(sleepMax: 20, forceWorkspace: true) { + build(wait: true, propagate: true, job: "/Ingest-manager/elastic-package-package-storage-publish/${BRANCH_NAME}") } } } -def signArtifactsWithElastic(artifactsSourcePath, signaturesDestinationPath) { - googleStorageUpload(bucket: env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH, - credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS, - pathPrefix: artifactsSourcePath + '/', - pattern: artifactsSourcePath + '/*.zip', - sharedPublicly: false, - showInline: true) - withCredentials([string(credentialsId: env.JOB_SIGNING_CREDENTIALS, variable: 'TOKEN')]) { - triggerRemoteJob(auth: CredentialsAuth(credentials: 'local-readonly-api-token'), - job: 'https://internal-ci.elastic.co/job/elastic+unified-release+master+sign-artifacts-with-gpg', - token: TOKEN, - parameters: "gcs_input_path=${env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}", - useCrumbCache: false, - useJobInfoCache: false) - } - googleStorageDownload(bucketUri: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH}/*", - credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS, - localDirectory: signaturesDestinationPath + '/', - pathPrefix: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}") - sh(label: 'Rename .asc to .sig', script: 'for f in ' + signaturesDestinationPath + '/*.asc; do mv "$f" "${f%.asc}.sig"; done') -} - def generateTestCheckSinglePackageStage(Map args = [:]) { def artifacts = ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'] diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy index 56a24b1d35..676285f203 100644 --- a/.ci/package-storage-publish.groovy +++ b/.ci/package-storage-publish.groovy @@ -14,12 +14,14 @@ pipeline { PIPELINE_LOG_LEVEL='INFO' // Signing + JOB_SIGNING_CREDENTIALS = 'sign-artifacts-with-gpg-job' INFRA_SIGNING_BUCKET_NAME = 'internal-ci-artifacts' INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER = "${env.REPO_BUILD_TAG}/signed-artifacts" INFRA_SIGNING_BUCKET_ARTIFACTS_PATH = "gs://${env.INFRA_SIGNING_BUCKET_NAME}/${env.REPO_BUILD_TAG}" INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH = "gs://${env.INFRA_SIGNING_BUCKET_NAME}/${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}" // Publishing + INTERNAL_CI_JOB_GCS_CREDENTIALS = 'internal-ci-gcs-plugin' PACKAGE_STORAGE_UPLOADER_CREDENTIALS = 'upload-package-to-package-storage' PACKAGE_STORAGE_UPLOADER_GCP_SERVICE_ACCOUNT = 'secret/gce/elastic-bekitzur/service-account/package-storage-uploader' PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH = "gs://elastic-bekitzur-package-storage-internal/queue-publishing/${env.REPO_BUILD_TAG}" From 3ff55387b6591aa2b91835275e0df53044764381 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 16 Mar 2022 12:42:24 +0100 Subject: [PATCH 03/16] Two pipelines starting --- .ci/Jenkinsfile | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 8b9e972258..caf1533913 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -102,8 +102,7 @@ pipeline { 'check-packages-with-kind': generateTestCommandStage(command: 'test-check-packages-with-kind', artifacts: ['build/test-results/*.xml', 'build/kubectl-dump.txt', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), 'check-packages-other': generateTestCommandStage(command: 'test-check-packages-other', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), 'build-zip': generateTestCommandStage(command: 'test-build-zip', artifacts: ['build/elastic-stack-dump/build-zip/logs/*.log', 'build/integrations/*.sig']), - 'profiles-command': generateTestCommandStage(command: 'test-profiles-command'), - 'publish-to-package-storage': generateTestPublishToPackageStorageStage() + 'profiles-command': generateTestCommandStage(command: 'test-profiles-command') ] def checkSinglePackageTasks = generateTestCheckSinglePackageStage() @@ -148,14 +147,6 @@ def cleanup(){ unstash 'source' } -def generateTestPublishToPackageStorageStage() { - return { - withNode(sleepMax: 20, forceWorkspace: true) { - build(wait: true, propagate: true, job: "/Ingest-manager/elastic-package-package-storage-publish/${BRANCH_NAME}") - } - } -} - def generateTestCheckSinglePackageStage(Map args = [:]) { def artifacts = ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'] From feb3f281a997c5b4a550cb1af5ec42adb857aa23 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 16 Mar 2022 12:45:58 +0100 Subject: [PATCH 04/16] Use ubuntu 20 --- .ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index caf1533913..44df0bc5fa 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -3,7 +3,7 @@ @Library('apm@current') _ pipeline { - agent { label 'ubuntu-18 && immutable' } + agent { label 'ubuntu-20 && immutable' } environment { REPO = "elastic-package" From 4e2e2e8b76670686a4126cf5188d99df6cd2dbf9 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 16 Mar 2022 12:55:19 +0100 Subject: [PATCH 05/16] Fix: paths --- .ci/package-storage-publish.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy index 676285f203..20ee0877b4 100644 --- a/.ci/package-storage-publish.groovy +++ b/.ci/package-storage-publish.groovy @@ -53,7 +53,7 @@ pipeline { cleanup() useElasticPackage() dir("${BASE_DIR}/test/packages/package_storage_candidate") { - sh(label: 'Build package',script: "elastic-package build") + sh(label: 'Build package',script: "../../../elastic-package build") } stash(allowEmpty: true, name: 'build-package', useDefaultExcludes: false) } From 6dccc8b794a976d4465a337edc1f439439f79429 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 16 Mar 2022 13:08:34 +0100 Subject: [PATCH 06/16] Fix: install elastic-package --- .ci/package-storage-publish.groovy | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy index 20ee0877b4..9243d4ad10 100644 --- a/.ci/package-storage-publish.groovy +++ b/.ci/package-storage-publish.groovy @@ -51,9 +51,14 @@ pipeline { stage('Build package') { steps { cleanup() - useElasticPackage() - dir("${BASE_DIR}/test/packages/package_storage_candidate") { - sh(label: 'Build package',script: "../../../elastic-package build") + withGoEnv() { + dir("${BASE_DIR}") { + sh(label: 'Install elastic-package',script: "make install") + // sh(label: 'Install elastic-package', script: 'go build github.com/elastic/elastic-package') + dir("test/packages/package_storage_candidate") { + sh(label: 'Build package',script: "elastic-package build") + } + } } stash(allowEmpty: true, name: 'build-package', useDefaultExcludes: false) } @@ -79,15 +84,6 @@ pipeline { } } -def useElasticPackage() { - withGoEnv() { - dir("${BASE_DIR}") { - sh(label: 'Install elastic-package',script: "make install") - // sh(label: 'Install elastic-package', script: 'go build github.com/elastic/elastic-package') - } - } -} - def signArtifactsWithElastic(artifactsPath) { dir("${BASE_DIR}") { googleStorageUpload(bucket: env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH, From 2579a96fdaaab811af5dfcc0f4b9b2e31f788fb8 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 16 Mar 2022 13:17:24 +0100 Subject: [PATCH 07/16] Fix --- .ci/package-storage-publish.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy index 9243d4ad10..d7d7957d67 100644 --- a/.ci/package-storage-publish.groovy +++ b/.ci/package-storage-publish.groovy @@ -55,7 +55,7 @@ pipeline { dir("${BASE_DIR}") { sh(label: 'Install elastic-package',script: "make install") // sh(label: 'Install elastic-package', script: 'go build github.com/elastic/elastic-package') - dir("test/packages/package_storage_candidate") { + dir("test/packages/package-storage/package_storage_candidate") { sh(label: 'Build package',script: "elastic-package build") } } From 2aa3c6986476c620c63d4b6b0c93d67b8b4f46ec Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 16 Mar 2022 13:53:33 +0100 Subject: [PATCH 08/16] Fix --- .ci/package-storage-publish.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy index d7d7957d67..b51d384f15 100644 --- a/.ci/package-storage-publish.groovy +++ b/.ci/package-storage-publish.groovy @@ -56,18 +56,18 @@ pipeline { sh(label: 'Install elastic-package',script: "make install") // sh(label: 'Install elastic-package', script: 'go build github.com/elastic/elastic-package') dir("test/packages/package-storage/package_storage_candidate") { - sh(label: 'Build package',script: "elastic-package build") + sh(label: 'Build package', script: "elastic-package build") } } } - stash(allowEmpty: true, name: 'build-package', useDefaultExcludes: false) + stash(allowEmpty: true, name: 'build-package', includes: 'build/integrations', useDefaultExcludes: false) } } stage('Sign package') { steps { cleanup(source: 'build-package') signArtifactsWithElastic('build/integrations') - stash(allowEmpty: true, name: 'sign-package', useDefaultExcludes: false) + stash(allowEmpty: true, name: 'sign-package', includes: 'build/integrations', useDefaultExcludes: false) } } stage('Publish package') { From db75562fc1ab79bdda35d8e0580bc02e634cfddf Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 16 Mar 2022 14:01:51 +0100 Subject: [PATCH 09/16] Fix --- .ci/package-storage-publish.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy index b51d384f15..8f1f0fed97 100644 --- a/.ci/package-storage-publish.groovy +++ b/.ci/package-storage-publish.groovy @@ -60,14 +60,14 @@ pipeline { } } } - stash(allowEmpty: true, name: 'build-package', includes: 'build/integrations', useDefaultExcludes: false) + stash(allowEmpty: true, name: 'build-package', includes: "${BASE_DIR}/build/integrations", useDefaultExcludes: false) } } stage('Sign package') { steps { cleanup(source: 'build-package') signArtifactsWithElastic('build/integrations') - stash(allowEmpty: true, name: 'sign-package', includes: 'build/integrations', useDefaultExcludes: false) + stash(allowEmpty: true, name: 'sign-package', includes: "${BASE_DIR}/build/integrations", useDefaultExcludes: false) } } stage('Publish package') { From 09eeb8a9967dd292d312dad439f11c9810193556 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 16 Mar 2022 14:09:34 +0100 Subject: [PATCH 10/16] Fix --- .ci/package-storage-publish.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy index 8f1f0fed97..3b5dd6611c 100644 --- a/.ci/package-storage-publish.groovy +++ b/.ci/package-storage-publish.groovy @@ -60,14 +60,14 @@ pipeline { } } } - stash(allowEmpty: true, name: 'build-package', includes: "${BASE_DIR}/build/integrations", useDefaultExcludes: false) + stash(allowEmpty: true, name: 'build-package', includes: "${BASE_DIR}/build/integrations/**", useDefaultExcludes: false) } } stage('Sign package') { steps { cleanup(source: 'build-package') signArtifactsWithElastic('build/integrations') - stash(allowEmpty: true, name: 'sign-package', includes: "${BASE_DIR}/build/integrations", useDefaultExcludes: false) + stash(allowEmpty: true, name: 'sign-package', includes: "${BASE_DIR}/build/integrations/**", useDefaultExcludes: false) } } stage('Publish package') { From 62611a37dfdc67d3e4c41cbfd3a662e07cabfbe6 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 16 Mar 2022 14:21:09 +0100 Subject: [PATCH 11/16] Fix: zip build --- .ci/package-storage-publish.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy index 3b5dd6611c..701732e837 100644 --- a/.ci/package-storage-publish.groovy +++ b/.ci/package-storage-publish.groovy @@ -56,11 +56,11 @@ pipeline { sh(label: 'Install elastic-package',script: "make install") // sh(label: 'Install elastic-package', script: 'go build github.com/elastic/elastic-package') dir("test/packages/package-storage/package_storage_candidate") { - sh(label: 'Build package', script: "elastic-package build") + sh(label: 'Build package', script: "elastic-package build -v --zip") } } } - stash(allowEmpty: true, name: 'build-package', includes: "${BASE_DIR}/build/integrations/**", useDefaultExcludes: false) + stash(allowEmpty: true, name: 'build-package', includes: "${BASE_DIR}/build/integrations/*.zip", useDefaultExcludes: false) } } stage('Sign package') { From 99bcec9090aaa36d4d834c6ee7ddb93fc8dc290a Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 16 Mar 2022 14:29:54 +0100 Subject: [PATCH 12/16] Fix: artifactsPath --- .ci/package-storage-publish.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy index 701732e837..9ec4186730 100644 --- a/.ci/package-storage-publish.groovy +++ b/.ci/package-storage-publish.groovy @@ -102,7 +102,7 @@ def signArtifactsWithElastic(artifactsPath) { } googleStorageDownload(bucketUri: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH}/*", credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS, - localDirectory: signaturesDestinationPath + '/', + localDirectory: artifactsPath + '/', pathPrefix: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}") sh(label: 'Rename .asc to .sig', script: 'for f in ' + artifactsPath + '/*.asc; do mv "$f" "${f%.asc}.sig"; done') } From 12c60ea3787ebaf48ecc3e59e8b2ea20340c83cc Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 16 Mar 2022 14:59:08 +0100 Subject: [PATCH 13/16] packageStoragePublish --- .ci/package-storage-publish.groovy | 72 ++++++++++++++++++------------ 1 file changed, 44 insertions(+), 28 deletions(-) diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy index 9ec4186730..0146ed137a 100644 --- a/.ci/package-storage-publish.groovy +++ b/.ci/package-storage-publish.groovy @@ -63,17 +63,12 @@ pipeline { stash(allowEmpty: true, name: 'build-package', includes: "${BASE_DIR}/build/integrations/*.zip", useDefaultExcludes: false) } } - stage('Sign package') { + stage('Sign and publish package') { steps { cleanup(source: 'build-package') - signArtifactsWithElastic('build/integrations') - stash(allowEmpty: true, name: 'sign-package', includes: "${BASE_DIR}/build/integrations/**", useDefaultExcludes: false) - } - } - stage('Publish package') { - steps { - cleanup(source: 'sign-package') - publishToPackageStorage('build/integrations') + dir("${BASE_DIR}") { + packageStoragePublish('build/integrations') + } } } } @@ -84,36 +79,50 @@ pipeline { } } -def signArtifactsWithElastic(artifactsPath) { - dir("${BASE_DIR}") { +def packageStoragePublish(builtPackagesPath) { + signUnpublishedArtifactsWithElastic(builtPackagesPath) + uploadUnpublishedToPackageStorage(builtPackagesPath) +} + +def signUnpublishedArtifactsWithElastic(builtPackagesPath) { + findFiles()?.findAll{ it.name.endsWith('.zip') }?.collect{ it.name }?.sort()?.each { + def packageZip = it + if (isAlreadyPublished(packageZip)) { + return + } + googleStorageUpload(bucket: env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH, credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS, - pathPrefix: artifactsPath + '/', - pattern: artifactsPath + '/*.zip', + pathPrefix: builtPackagesPath + '/', + pattern: builtPackagesPath + '/*.zip', sharedPublicly: false, showInline: true) - withCredentials([string(credentialsId: env.JOB_SIGNING_CREDENTIALS, variable: 'TOKEN')]) { - triggerRemoteJob(auth: CredentialsAuth(credentials: 'local-readonly-api-token'), - job: 'https://internal-ci.elastic.co/job/elastic+unified-release+master+sign-artifacts-with-gpg', - token: TOKEN, - parameters: "gcs_input_path=${env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}", - useCrumbCache: false, - useJobInfoCache: false) - } - googleStorageDownload(bucketUri: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH}/*", - credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS, - localDirectory: artifactsPath + '/', - pathPrefix: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}") - sh(label: 'Rename .asc to .sig', script: 'for f in ' + artifactsPath + '/*.asc; do mv "$f" "${f%.asc}.sig"; done') } + withCredentials([string(credentialsId: env.JOB_SIGNING_CREDENTIALS, variable: 'TOKEN')]) { + triggerRemoteJob(auth: CredentialsAuth(credentials: 'local-readonly-api-token'), + job: 'https://internal-ci.elastic.co/job/elastic+unified-release+master+sign-artifacts-with-gpg', + token: TOKEN, + parameters: "gcs_input_path=${env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}", + useCrumbCache: false, + useJobInfoCache: false) + } + googleStorageDownload(bucketUri: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH}/*", + credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS, + localDirectory: builtPackagesPath + '/', + pathPrefix: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}") + sh(label: 'Rename .asc to .sig', script: 'for f in ' + builtPackagesPath + '/*.asc; do mv "$f" "${f%.asc}.sig"; done') } -def publishToPackageStorage(artifactsPath) { - dir("${BASE_DIR}/${artifactsPath}") { +def uploadUnpublishedToPackageStorage(builtPackagesPath) { + dir("${BASE_DIR}/${builtPackagesPath}") { withGCPEnv(secret: env.PACKAGE_STORAGE_UPLOADER_GCP_SERVICE_ACCOUNT) { withCredentials([string(credentialsId: env.PACKAGE_STORAGE_UPLOADER_CREDENTIALS, variable: 'TOKEN')]) { findFiles()?.findAll{ it.name.endsWith('.zip') }?.collect{ it.name }?.sort()?.each { def packageZip = it + if (isAlreadyPublished(packageZip)) { + return + } + sh(label: 'Upload package .zip file', script: "gsutil cp ${packageZip} ${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/") sh(label: 'Upload package .sig file', script: "gsutil cp ${packageZip}.sig ${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/") @@ -133,6 +142,13 @@ def publishToPackageStorage(artifactsPath) { } } +def isAlreadyPublished(packageZip) { + def responseCode = httpRequest(method: "HEAD", + url: "https://package-storage.elastic.co/artifacts/packages/${it}", + response_code_only: true) + return responseCode == 200 +} + def cleanup(Map args = [:]) { def source = args.containsKey('source') ? args.source : 'source' dir("${BASE_DIR}"){ From 5563d19684394351b705f6fc5187d104346ae1c9 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 16 Mar 2022 15:52:37 +0100 Subject: [PATCH 14/16] Fix --- .ci/package-storage-publish.groovy | 31 +++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy index 0146ed137a..a025321146 100644 --- a/.ci/package-storage-publish.groovy +++ b/.ci/package-storage-publish.groovy @@ -85,19 +85,28 @@ def packageStoragePublish(builtPackagesPath) { } def signUnpublishedArtifactsWithElastic(builtPackagesPath) { - findFiles()?.findAll{ it.name.endsWith('.zip') }?.collect{ it.name }?.sort()?.each { - def packageZip = it - if (isAlreadyPublished(packageZip)) { - return + def unpublished = false + dir(builtPackagesPath) { + findFiles()?.findAll{ it.name.endsWith('.zip') }?.collect{ it.name }?.sort()?.each { + def packageZip = it + if (isAlreadyPublished(packageZip)) { + return + } + + unpublished = true + googleStorageUpload(bucket: env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH, + credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS, + pathPrefix: '/', + pattern: '/*.zip', + sharedPublicly: false, + showInline: true) } + } - googleStorageUpload(bucket: env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH, - credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS, - pathPrefix: builtPackagesPath + '/', - pattern: builtPackagesPath + '/*.zip', - sharedPublicly: false, - showInline: true) + if (!unpublished) { + return } + withCredentials([string(credentialsId: env.JOB_SIGNING_CREDENTIALS, variable: 'TOKEN')]) { triggerRemoteJob(auth: CredentialsAuth(credentials: 'local-readonly-api-token'), job: 'https://internal-ci.elastic.co/job/elastic+unified-release+master+sign-artifacts-with-gpg', @@ -114,7 +123,7 @@ def signUnpublishedArtifactsWithElastic(builtPackagesPath) { } def uploadUnpublishedToPackageStorage(builtPackagesPath) { - dir("${BASE_DIR}/${builtPackagesPath}") { + dir(builtPackagesPath) { withGCPEnv(secret: env.PACKAGE_STORAGE_UPLOADER_GCP_SERVICE_ACCOUNT) { withCredentials([string(credentialsId: env.PACKAGE_STORAGE_UPLOADER_CREDENTIALS, variable: 'TOKEN')]) { findFiles()?.findAll{ it.name.endsWith('.zip') }?.collect{ it.name }?.sort()?.each { From 947273c11e554b760b2c8ec706609a3a94e96b25 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 16 Mar 2022 16:10:26 +0100 Subject: [PATCH 15/16] Fix --- .ci/package-storage-publish.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy index a025321146..c056af80f6 100644 --- a/.ci/package-storage-publish.groovy +++ b/.ci/package-storage-publish.groovy @@ -153,7 +153,7 @@ def uploadUnpublishedToPackageStorage(builtPackagesPath) { def isAlreadyPublished(packageZip) { def responseCode = httpRequest(method: "HEAD", - url: "https://package-storage.elastic.co/artifacts/packages/${it}", + url: "https://package-storage.elastic.co/artifacts/packages/${packageZip}", response_code_only: true) return responseCode == 200 } From c89064d92fff30aeabfecb1ef89fe03215d6be75 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 16 Mar 2022 16:37:18 +0100 Subject: [PATCH 16/16] Fix --- .ci/package-storage-publish.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy index c056af80f6..ca32154e5d 100644 --- a/.ci/package-storage-publish.groovy +++ b/.ci/package-storage-publish.groovy @@ -96,8 +96,7 @@ def signUnpublishedArtifactsWithElastic(builtPackagesPath) { unpublished = true googleStorageUpload(bucket: env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH, credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS, - pathPrefix: '/', - pattern: '/*.zip', + pattern: '*.zip', sharedPublicly: false, showInline: true) }