diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index cac4082d03..44df0bc5fa 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -3,7 +3,7 @@ @Library('apm@current') _ pipeline { - agent { label 'ubuntu-18 && immutable' } + agent { label 'ubuntu-20 && immutable' } environment { REPO = "elastic-package" @@ -22,21 +22,6 @@ pipeline { JOB_GCS_EXT_CREDENTIALS = 'beats-ci-gcs-plugin-file-credentials' ELASTIC_PACKAGE_GCP_SECRET = 'secret/observability-team/ci/service-account/elastic-package-gcp' ELASTIC_OBSERVABILITY_PROJECT_ID = 'elastic-observability' - - JOB_SIGNING_CREDENTIALS = 'sign-artifacts-with-gpg-job' - INTERNAL_CI_JOB_GCS_CREDENTIALS = 'internal-ci-gcs-plugin' - - REPO_BUILD_TAG = "${env.REPO}/${env.BUILD_TAG}" - INFRA_SIGNING_BUCKET_NAME = 'internal-ci-artifacts' - INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER = "${env.REPO_BUILD_TAG}/signed-artifacts" - INFRA_SIGNING_BUCKET_ARTIFACTS_PATH = "gs://${env.INFRA_SIGNING_BUCKET_NAME}/${env.REPO_BUILD_TAG}" - INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH = "gs://${env.INFRA_SIGNING_BUCKET_NAME}/${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}" - - INTEGRATIONS_SIGNATURES_PATH = 'build/integrations-elastic-signatures' // different path not to override signatures archived in the "build-zip" step - - PACKAGE_STORAGE_UPLOADER_CREDENTIALS = 'upload-package-to-package-storage' - PACKAGE_STORAGE_UPLOADER_GCP_SERVICE_ACCOUNT = 'secret/gce/elastic-bekitzur/service-account/package-storage-uploader' - PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH = "gs://elastic-bekitzur-package-storage-internal/queue-publishing/${env.REPO_BUILD_TAG}" } options { timeout(time: 1, unit: 'HOURS') @@ -117,8 +102,7 @@ pipeline { 'check-packages-with-kind': generateTestCommandStage(command: 'test-check-packages-with-kind', artifacts: ['build/test-results/*.xml', 'build/kubectl-dump.txt', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), 'check-packages-other': generateTestCommandStage(command: 'test-check-packages-other', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), 'build-zip': generateTestCommandStage(command: 'test-build-zip', artifacts: ['build/elastic-stack-dump/build-zip/logs/*.log', 'build/integrations/*.sig']), - 'profiles-command': generateTestCommandStage(command: 'test-profiles-command'), - 'publish-to-package-storage': generateTestPublishToPackageStorageStage() + 'profiles-command': generateTestCommandStage(command: 'test-profiles-command') ] def checkSinglePackageTasks = generateTestCheckSinglePackageStage() @@ -163,66 +147,6 @@ def cleanup(){ unstash 'source' } -def generateTestPublishToPackageStorageStage() { - return { - withNode(labels: "ubuntu-20 && immutable", sleepMax: 20, forceWorkspace: true) { - cleanup() - dir("${BASE_DIR}"){ - withMageEnv(){ - sh(label: 'Install elastic-package',script: "make install") - dir("test/packages/package-storage/package_storage_candidate") { - sh(label: 'Lint package',script: "elastic-package lint") - sh(label: 'Build zipped package',script: "elastic-package build --zip") - } - signArtifactsWithElastic('build/integrations', env.INTEGRATIONS_SIGNATURES_PATH) - - // Add the package candidate to the "queue-publishing" - withGCPEnv(secret: env.PACKAGE_STORAGE_UPLOADER_GCP_SERVICE_ACCOUNT) { - sh(label: 'Upload package .zip file', script: "gsutil cp ${env.INTEGRATIONS_SIGNATURES_PATH}/package_storage_candidate-0.0.1.zip ${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/") - sh(label: 'Upload package .sig file', script: "gsutil cp ${env.INTEGRATIONS_SIGNATURES_PATH}/package_storage_candidate-0.0.1.zip.sig ${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/") - } - - // Call the publishing job - withCredentials([string(credentialsId: env.PACKAGE_STORAGE_UPLOADER_CREDENTIALS, variable: 'TOKEN')]) { - triggerRemoteJob(auth: CredentialsAuth(credentials: 'local-readonly-api-token'), - job: 'https://internal-ci.elastic.co/job/package_storage/job/publishing-job-remote', - token: TOKEN, - parameters: """ - dry_run=true - gs_package_build_zip_path=${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/package_storage_candidate-0.0.1.zip - gs_package_signature_path=${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/package_storage_candidate-0.0.1.zip.sig - """, - useCrumbCache: false, - useJobInfoCache: false) - } - } - } - } - } -} - -def signArtifactsWithElastic(artifactsSourcePath, signaturesDestinationPath) { - googleStorageUpload(bucket: env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH, - credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS, - pathPrefix: artifactsSourcePath + '/', - pattern: artifactsSourcePath + '/*.zip', - sharedPublicly: false, - showInline: true) - withCredentials([string(credentialsId: env.JOB_SIGNING_CREDENTIALS, variable: 'TOKEN')]) { - triggerRemoteJob(auth: CredentialsAuth(credentials: 'local-readonly-api-token'), - job: 'https://internal-ci.elastic.co/job/elastic+unified-release+master+sign-artifacts-with-gpg', - token: TOKEN, - parameters: "gcs_input_path=${env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}", - useCrumbCache: false, - useJobInfoCache: false) - } - googleStorageDownload(bucketUri: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH}/*", - credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS, - localDirectory: signaturesDestinationPath + '/', - pathPrefix: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}") - sh(label: 'Rename .asc to .sig', script: 'for f in ' + signaturesDestinationPath + '/*.asc; do mv "$f" "${f%.asc}.sig"; done') -} - def generateTestCheckSinglePackageStage(Map args = [:]) { def artifacts = ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'] diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy index 82a69cacef..ca32154e5d 100644 --- a/.ci/package-storage-publish.groovy +++ b/.ci/package-storage-publish.groovy @@ -6,11 +6,25 @@ pipeline { agent { label 'ubuntu-20 && immutable' } environment { REPO = "elastic-package" + REPO_BUILD_TAG = "${env.REPO}/${env.BUILD_TAG}" BASE_DIR="src/github.com/elastic/elastic-package" JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba" GITHUB_TOKEN_CREDENTIALS = "2a9602aa-ab9f-4e52-baf3-b71ca88469c7" PIPELINE_LOG_LEVEL='INFO' + + // Signing + JOB_SIGNING_CREDENTIALS = 'sign-artifacts-with-gpg-job' + INFRA_SIGNING_BUCKET_NAME = 'internal-ci-artifacts' + INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER = "${env.REPO_BUILD_TAG}/signed-artifacts" + INFRA_SIGNING_BUCKET_ARTIFACTS_PATH = "gs://${env.INFRA_SIGNING_BUCKET_NAME}/${env.REPO_BUILD_TAG}" + INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH = "gs://${env.INFRA_SIGNING_BUCKET_NAME}/${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}" + + // Publishing + INTERNAL_CI_JOB_GCS_CREDENTIALS = 'internal-ci-gcs-plugin' + PACKAGE_STORAGE_UPLOADER_CREDENTIALS = 'upload-package-to-package-storage' + PACKAGE_STORAGE_UPLOADER_GCP_SERVICE_ACCOUNT = 'secret/gce/elastic-bekitzur/service-account/package-storage-uploader' + PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH = "gs://elastic-bekitzur-package-storage-internal/queue-publishing/${env.REPO_BUILD_TAG}" } options { timeout(time: 1, unit: 'HOURS') @@ -31,7 +45,30 @@ pipeline { pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ]) deleteDir() gitCheckout(basedir: "${BASE_DIR}") - stash allowEmpty: true, name: 'source', useDefaultExcludes: false + stash(allowEmpty: true, name: 'source', useDefaultExcludes: false) + } + } + stage('Build package') { + steps { + cleanup() + withGoEnv() { + dir("${BASE_DIR}") { + sh(label: 'Install elastic-package',script: "make install") + // sh(label: 'Install elastic-package', script: 'go build github.com/elastic/elastic-package') + dir("test/packages/package-storage/package_storage_candidate") { + sh(label: 'Build package', script: "elastic-package build -v --zip") + } + } + } + stash(allowEmpty: true, name: 'build-package', includes: "${BASE_DIR}/build/integrations/*.zip", useDefaultExcludes: false) + } + } + stage('Sign and publish package') { + steps { + cleanup(source: 'build-package') + dir("${BASE_DIR}") { + packageStoragePublish('build/integrations') + } } } } @@ -42,9 +79,88 @@ pipeline { } } -def cleanup(){ +def packageStoragePublish(builtPackagesPath) { + signUnpublishedArtifactsWithElastic(builtPackagesPath) + uploadUnpublishedToPackageStorage(builtPackagesPath) +} + +def signUnpublishedArtifactsWithElastic(builtPackagesPath) { + def unpublished = false + dir(builtPackagesPath) { + findFiles()?.findAll{ it.name.endsWith('.zip') }?.collect{ it.name }?.sort()?.each { + def packageZip = it + if (isAlreadyPublished(packageZip)) { + return + } + + unpublished = true + googleStorageUpload(bucket: env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH, + credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS, + pattern: '*.zip', + sharedPublicly: false, + showInline: true) + } + } + + if (!unpublished) { + return + } + + withCredentials([string(credentialsId: env.JOB_SIGNING_CREDENTIALS, variable: 'TOKEN')]) { + triggerRemoteJob(auth: CredentialsAuth(credentials: 'local-readonly-api-token'), + job: 'https://internal-ci.elastic.co/job/elastic+unified-release+master+sign-artifacts-with-gpg', + token: TOKEN, + parameters: "gcs_input_path=${env.INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}", + useCrumbCache: false, + useJobInfoCache: false) + } + googleStorageDownload(bucketUri: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH}/*", + credentialsId: env.INTERNAL_CI_JOB_GCS_CREDENTIALS, + localDirectory: builtPackagesPath + '/', + pathPrefix: "${env.INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}") + sh(label: 'Rename .asc to .sig', script: 'for f in ' + builtPackagesPath + '/*.asc; do mv "$f" "${f%.asc}.sig"; done') +} + +def uploadUnpublishedToPackageStorage(builtPackagesPath) { + dir(builtPackagesPath) { + withGCPEnv(secret: env.PACKAGE_STORAGE_UPLOADER_GCP_SERVICE_ACCOUNT) { + withCredentials([string(credentialsId: env.PACKAGE_STORAGE_UPLOADER_CREDENTIALS, variable: 'TOKEN')]) { + findFiles()?.findAll{ it.name.endsWith('.zip') }?.collect{ it.name }?.sort()?.each { + def packageZip = it + if (isAlreadyPublished(packageZip)) { + return + } + + sh(label: 'Upload package .zip file', script: "gsutil cp ${packageZip} ${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/") + sh(label: 'Upload package .sig file', script: "gsutil cp ${packageZip}.sig ${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/") + + triggerRemoteJob(auth: CredentialsAuth(credentials: 'local-readonly-api-token'), + job: 'https://internal-ci.elastic.co/job/package_storage/job/publishing-job-remote', + token: TOKEN, + parameters: """ + dry_run=true + gs_package_build_zip_path=${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/${packageZip} + gs_package_signature_path=${env.PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/${packageZip}.sig + """, + useCrumbCache: true, + useJobInfoCache: true) + } + } + } + } +} + +def isAlreadyPublished(packageZip) { + def responseCode = httpRequest(method: "HEAD", + url: "https://package-storage.elastic.co/artifacts/packages/${packageZip}", + response_code_only: true) + return responseCode == 200 +} + +def cleanup(Map args = [:]) { + def source = args.containsKey('source') ? args.source : 'source' dir("${BASE_DIR}"){ deleteDir() } - unstash 'source' + unstash source } \ No newline at end of file