diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 6fdbea036c..44d9ebb2ff 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -26,6 +26,8 @@ GCP_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/gcp-service-ac AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_account_auth GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token JENKINS_API_TOKEN_PATH=kv/ci-shared/platform-ingest/jenkins_api_tokens +SIGNING_PACKAGES_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/signing_packages_gcs_artifacts_credentials +PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/package_storage_uploader # Secrets must be redacted # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables @@ -56,4 +58,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-package-storage-publish" && export JENKINS_USERNAME_SECRET=$(retry 5 vault kv get -field username ${JENKINS_API_TOKEN_PATH}) export JENKINS_HOST_SECRET=$(retry 5 vault kv get -field internal_ci_host ${JENKINS_API_TOKEN_PATH}) export JENKINS_TOKEN=$(retry 5 vault kv get -field internal_ci ${JENKINS_API_TOKEN_PATH}) + + # signing job + export SIGNING_PACKAGES_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field value ${SIGNING_PACKAGES_GCS_CREDENTIALS_PATH}) + + # publishing job + export PACKAGE_UPLOADER_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field value ${PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH}) fi diff --git a/.buildkite/pipeline.package-storage-publish.yml b/.buildkite/pipeline.package-storage-publish.yml index cc489cde81..360fba092b 100644 --- a/.buildkite/pipeline.package-storage-publish.yml +++ b/.buildkite/pipeline.package-storage-publish.yml @@ -1,7 +1,7 @@ -steps: - - label: "Example Test" - command: echo "Hello!" +env: + SETUP_GVM_VERSION: 'v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 +steps: - label: ":go: Build package" key: build-package command: @@ -11,10 +11,14 @@ steps: image: "golang:1.19.5" cpu: "8" memory: "4G" + artifact_paths: + - build/packages/*.zip - - label: "Test" + - label: "Sign and Publish package" key: sign-publish command: ".buildkite/scripts/signAndPublishPackage.sh" + depends_on: + - build-package agents: provider: "gcp" image: family/core-ubuntu-2004 diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh new file mode 100755 index 0000000000..20a1f8a9c2 --- /dev/null +++ b/.buildkite/scripts/install_deps.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -euo pipefail + +source .buildkite/scripts/tooling.sh + +with_kubernetes() { + mkdir -p ${WORKSPACE}/bin + retry 5 curl -sSLo ${WORKSPACE}/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" + chmod +x ${WORKSPACE}/bin/kind + kind version + which kind + + mkdir -p ${WORKSPACE}/bin + retry 5 curl -sSLo ${WORKSPACE}/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl" + chmod +x ${WORKSPACE}/bin/kubectl + kubectl version --client + which kubectl +} + +with_go() { + mkdir -p ${WORKSPACE}/bin + retry 5 curl -sL -o ${WORKSPACE}/bin/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-linux-amd64" + chmod +x ${WORKSPACE}/bin/gvm + eval "$(gvm $(cat .go-version))" + go version + which go +} + +with_docker_compose() { + mkdir -p ${WORKSPACE}/bin + retry 5 curl -SL -o ${WORKSPACE}/bin/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64" + chmod +x ${WORKSPACE}/bin/docker-compose + docker-compose version +} diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index 896927a07e..e94bba9307 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -13,38 +13,7 @@ usage() { echo -e "\t-h: Show this message" } -with_kubernetes() { - # FIXME add retry logic - mkdir -p ${WORKSPACE}/bin - curl -sSLo ${WORKSPACE}/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" - chmod +x ${WORKSPACE}/bin/kind - kind version - which kind - - mkdir -p ${WORKSPACE}/bin - curl -sSLo ${WORKSPACE}/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl" - chmod +x ${WORKSPACE}/bin/kubectl - kubectl version --client - which kubectl -} - -with_go() { - # FIXME add retry logic - mkdir -p ${WORKSPACE}/bin - curl -sL -o ${WORKSPACE}/bin/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-linux-amd64" - chmod +x ${WORKSPACE}/bin/gvm - eval "$(gvm $(cat .go-version))" - go version - which go -} - -with_docker_compose() { - # FIXME add retry logic - mkdir -p ${WORKSPACE}/bin - curl -SL -o ${WORKSPACE}/bin/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64" - chmod +x ${WORKSPACE}/bin/docker-compose - docker-compose version -} +source .buildkite/scripts/install_deps.sh TARGET="" PACKAGE="" diff --git a/.buildkite/scripts/signAndPublishPackage.sh b/.buildkite/scripts/signAndPublishPackage.sh index e81e3630d7..a7eccb81bb 100755 --- a/.buildkite/scripts/signAndPublishPackage.sh +++ b/.buildkite/scripts/signAndPublishPackage.sh @@ -1,6 +1,20 @@ #!/bin/bash set -euo pipefail +WORKSPACE="$(pwd)" +TMP_FOLDER_TEMPLATE_BASE="tmp.elastic-package" + +cleanup() { + echo "Deleting temporal files..." + cd ${WORKSPACE} + rm -rf ${TMP_FOLDER_TEMPLATE_BASE}.* + echo "Done." +} + +trap cleanup EXIT + +export PATH="${WORKSPACE}/bin:${PATH}" + echo "Checking gsutil command..." if ! command -v gsutil &> /dev/null ; then echo "⚠️ gsutil is not installed" @@ -9,4 +23,146 @@ else echo "✅ gsutil is installed" fi -gsutil help +source .buildkite/scripts/install_deps.sh +source .buildkite/scripts/tooling.sh + +isAlreadyPublished() { + local packageZip=$1 + + if curl -s --head https://package-storage.elastic.co/artifacts/packages/${packageZip} | grep -q "HTTP/2 200" ; then + echo "- Already published ${packageZip}" + return 0 + fi + echo "- Not published ${packageZip}" + return 1 +} + +REPO_NAME=$(repoName "${BUILDKITE_REPO}") +BUILD_TAG="buildkite-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_NUMBER}" + +REPO_BUILD_TAG="${REPO_NAME}/${BUILD_TAG}" + +BUILD_PACKAGES_PATH="build/packages" +TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX" +JENKINS_TRIGGER_PATH=".buildkite/scripts/triggerJenkinsJob" +GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json" + +## Signing +INFRA_SIGNING_BUCKET_NAME='internal-ci-artifacts' +INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER="${REPO_BUILD_TAG}/signed-artifacts" +INFRA_SIGNING_BUCKET_ARTIFACTS_PATH="gs://${INFRA_SIGNING_BUCKET_NAME}/${REPO_BUILD_TAG}" +INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH="gs://${INFRA_SIGNING_BUCKET_NAME}/${INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}" + +## Publishing +PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH="gs://elastic-bekitzur-package-storage-internal/queue-publishing/${REPO_BUILD_TAG}" + + +google_cloud_auth_signing() { + local gsUtilLocation=$(mktemp -d -p . -t ${TMP_FOLDER_TEMPLATE}) + + local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME} + echo "${SIGNING_PACKAGES_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation} + + google_cloud_auth "${secretFileLocation}" + + echo "${gsUtilLocation}" +} + +google_cloud_auth_publishing() { + local gsUtilLocation=$(mktemp -d -p . -t ${TMP_FOLDER_TEMPLATE}) + + local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME} + echo "${PACKAGE_UPLOADER_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation} + + google_cloud_auth "${secretFileLocation}" + + echo "${gsUtilLocation}" +} + +signPackage() { + local package=${1} + local packageZip=$(basename ${package}) + + local gsUtilLocation=$(google_cloud_auth_signing) + + # upload zip package (trailing forward slashes are required) + echo "Upload package .zip file for signing ${package} to ${INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}" + gsutil cp ${package} "${INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}/" + + echo "Trigger Jenkins job for signing package ${packageZip}" + pushd ${JENKINS_TRIGGER_PATH} > /dev/null + + go run main.go \ + --jenkins-job sign \ + --folder ${INFRA_SIGNING_BUCKET_ARTIFACTS_PATH} + + sleep 5 + popd > /dev/null + + echo "Download signatures" + gsutil cp "${INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH}/${packageZip}.asc" "${BUILD_PACKAGES_PATH}" + + echo "Rename asc to sig" + for f in $(ls ${BUILD_PACKAGES_PATH}/*.asc); do + mv "$f" "${f%.asc}.sig" + done + + ls -l "${BUILD_PACKAGES_PATH}" + + echo "Removing temporal location ${gsUtilLocation}" + rm -r "${gsUtilLocation}" +} + +publishPackage() { + local package=$1 + local packageZip=$(basename ${package}) + + # create file with credentials + local gsUtilLocation=$(google_cloud_auth_publishing) + + # upload files (trailing forward slashes are required) + echo "Upload package .zip file ${package} to ${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}" + gsutil cp ${package} "${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/" + echo "Upload package .sig file ${package}.sig to ${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}" + gsutil cp ${package}.sig "${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/" + + echo "Trigger Jenkins job for publishing package ${packageZip}" + pushd ${JENKINS_TRIGGER_PATH} > /dev/null + + go run main.go \ + --jenkins-job publish \ + --package "${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/${packageZip}" \ + --signature "${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/${packageZip}.sig" + + sleep 5 + + popd > /dev/null + + echo "Removing temporal location ${gsUtilLocation}" + rm -r "${gsUtilLocation}" +} + +# Required to trigger Jenkins job +with_go + +# download package artifact from previous step +mkdir -p "${BUILD_PACKAGES_PATH}" + +buildkite-agent artifact download "${BUILD_PACKAGES_PATH}/*.zip" --step build-package . +echo "Show artifacts downloaded from previous step ${BUILD_PACKAGES_PATH}" +ls -l "${BUILD_PACKAGES_PATH}" + +for package in $(ls ${BUILD_PACKAGES_PATH}/*.zip); do + echo "isAlreadyInstalled ${package}?" + packageZip=$(basename ${package}) + if isAlreadyPublished ${packageZip} ; then + echo "Skipping. ${packageZip} already published" + continue + fi + + echo "Signing package ${packageZip}" + signPackage "${package}" + + echo "Publishing package ${packageZip}" + publishPackage "${package}" +done diff --git a/.buildkite/scripts/tooling.sh b/.buildkite/scripts/tooling.sh new file mode 100755 index 0000000000..0ea9bfba66 --- /dev/null +++ b/.buildkite/scripts/tooling.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -euo pipefail + +repoName() { + # Example of URL: git@github.com:acme-inc/my-project.git + local repoUrl=$1 + + orgAndRepo=$(echo $repoUrl | cut -d':' -f 2) + echo "$(basename ${orgAndRepo} .git)" +} + +google_cloud_auth() { + local keyFile=$1 + + gcloud auth activate-service-account --key-file ${keyFile} 2> /dev/null + + export GOOGLE_APPLICATIONS_CREDENTIALS=${secretFileLocation} +} + +retry() { + local retries=$1 + shift + + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + diff --git a/.buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go b/.buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go index 58b27df1f8..b285990ea2 100644 --- a/.buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go +++ b/.buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go @@ -50,7 +50,11 @@ func (j *JenkinsClient) RunJob(ctx context.Context, jobName string, async bool, return fmt.Errorf("not finished job %s/%d: %w", jobName, build.GetBuildNumber(), err) } - log.Printf("Build %s finished with result: %v\n", build.GetUrl(), build.GetBuildNumber(), build.GetResult()) + log.Printf("Build %s finished with result: %s\n", build.GetUrl(), build.GetResult()) + + if build.GetResult() != gojenkins.STATUS_SUCCESS { + return fmt.Errorf("build %s finished with result %s", build.GetUrl(), build.GetResult()) + } return nil } @@ -94,16 +98,18 @@ func (j *JenkinsClient) getBuildFromQueueID(ctx context.Context, job *gojenkins. } func (j *JenkinsClient) waitForBuildFinished(ctx context.Context, build *gojenkins.Build) error { + const waitingPeriod = 10000 * time.Millisecond for build.IsRunning(ctx) { - log.Printf("Build still running, waiting for 5 secs...") + log.Printf("Build still running, waiting for %s...", waitingPeriod) select { - case <-time.After(5000 * time.Millisecond): + case <-time.After(waitingPeriod): case <-ctx.Done(): return ctx.Err() } - _, err = build.Poll(ctx) + _, err := build.Poll(ctx) if err != nil { return err } } + return nil } diff --git a/.buildkite/scripts/triggerJenkinsJob/main.go b/.buildkite/scripts/triggerJenkinsJob/main.go index ff6019f070..bfddaa2f6d 100644 --- a/.buildkite/scripts/triggerJenkinsJob/main.go +++ b/.buildkite/scripts/triggerJenkinsJob/main.go @@ -17,7 +17,7 @@ import ( const ( publishingRemoteJob = "package_storage/job/publishing-job-remote" - signingJob = "elastic+unified-release+master+sign-artifacts-wigh-gpg" + signingJob = "elastic+unified-release+master+sign-artifacts-with-gpg" publishJobKey = "publish" signJobKey = "sign" @@ -44,7 +44,8 @@ func jenkinsJobOptions() []string { func main() { jenkinsJob := flag.String("jenkins-job", "", fmt.Sprintf("Jenkins job to trigger. Allowed values: %s", strings.Join(jenkinsJobOptions(), " ,"))) - zipPackagePath := flag.String("package", "", "Path to zip package file (*.zip) ") + folderPath := flag.String("folder", "", "Path to artifacts folder") + zipPackagePath := flag.String("package", "", "Path to zip package file (*.zip)") sigPackagePath := flag.String("signature", "", "Path to the signature file of the package file (*.zip.sig)") async := flag.Bool("async", false, "Run async the Jenkins job") flag.Parse() @@ -65,24 +66,34 @@ func main() { case publishJobKey: err = runPublishingRemoteJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *zipPackagePath, *sigPackagePath) case signJobKey: - err = runSignPackageJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *zipPackagePath) + err = runSignPackageJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *folderPath) default: log.Fatal("unsupported jenkins job") } if err != nil { - log.Fatal("Error: %s", err) + log.Fatalf("Error: %s", err) } } -func runSignPackageJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, packagePath string) error { - params := map[string]string{} - // TODO set parameters for sign job +func runSignPackageJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, folderPath string) error { + if folderPath == "" { + return fmt.Errorf("missing parameter --gcs_input_path for") + } + params := map[string]string{ + "gcs_input_path": folderPath, + } return client.RunJob(ctx, jobName, async, params) } func runPublishingRemoteJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, packagePath, signaturePath string) error { + if packagePath == "" { + return fmt.Errorf("missing parameter --gs_package_build_zip_path") + } + if signaturePath == "" { + return fmt.Errorf("missing parameter --gs_package_signature_path") + } // Run the job with some parameters params := map[string]string{