Sign and publish pipeline in Buildkite

.buildkite/hooks/pre-command

@@ -26,6 +26,8 @@ GCP_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/gcp-service-ac
 AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_account_auth
 GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token
 JENKINS_API_TOKEN_PATH=kv/ci-shared/platform-ingest/jenkins_api_tokens
+SIGNING_PACKAGES_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/signing_packages_gcs_artifacts_credentials
+PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/package_storage_uploader
 
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
@@ -56,4 +58,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-package-storage-publish" &&
     export JENKINS_USERNAME_SECRET=$(retry 5 vault kv get -field username ${JENKINS_API_TOKEN_PATH})
     export JENKINS_HOST_SECRET=$(retry 5 vault kv get -field internal_ci_host ${JENKINS_API_TOKEN_PATH})
     export JENKINS_TOKEN=$(retry 5 vault kv get -field internal_ci ${JENKINS_API_TOKEN_PATH})
+
+    # signing job
+    export SIGNING_PACKAGES_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field value ${SIGNING_PACKAGES_GCS_CREDENTIALS_PATH})
+
+    # publishing job
+    export PACKAGE_UPLOADER_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field value ${PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH})
 fi

.buildkite/pipeline.package-storage-publish.yml

@@ -1,7 +1,7 @@
-steps:
-  - label: "Example Test"
-    command: echo "Hello!"
+env:
+  SETUP_GVM_VERSION: 'v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
 
+steps:
   - label: ":go: Build package"
     key: build-package
     command:
@@ -11,10 +11,14 @@ steps:
       image: "golang:1.19.5"
       cpu: "8"
       memory: "4G"
+    artifact_paths:
+      - build/packages/*.zip
 
-  - label: "Test"
+  - label: "Sign and Publish package"
     key: sign-publish
     command: ".buildkite/scripts/signAndPublishPackage.sh"
+    depends_on:
+      - build-package
     agents:
       provider: "gcp"
       image: family/core-ubuntu-2004

.buildkite/scripts/install_deps.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/tooling.sh
+
+with_kubernetes() {
+    mkdir -p ${WORKSPACE}/bin
+    retry 5 curl -sSLo ${WORKSPACE}/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
+    chmod +x ${WORKSPACE}/bin/kind
+    kind version
+    which kind
+
+    mkdir -p ${WORKSPACE}/bin
+    retry 5 curl -sSLo ${WORKSPACE}/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
+    chmod +x ${WORKSPACE}/bin/kubectl
+    kubectl version --client
+    which kubectl
+}
+
+with_go() {
+    mkdir -p ${WORKSPACE}/bin
+    retry 5 curl -sL -o ${WORKSPACE}/bin/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-linux-amd64"
+    chmod +x ${WORKSPACE}/bin/gvm
+    eval "$(gvm $(cat .go-version))"
+    go version
+    which go
+}
+
+with_docker_compose() {
+    mkdir -p ${WORKSPACE}/bin
+    retry 5 curl -SL -o ${WORKSPACE}/bin/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64"
+    chmod +x ${WORKSPACE}/bin/docker-compose
+    docker-compose version
+}

.buildkite/scripts/integration_tests.sh

@@ -13,38 +13,7 @@ usage() {
     echo -e "\t-h: Show this message"
 }
 
-with_kubernetes() {
-    # FIXME add retry logic
-    mkdir -p ${WORKSPACE}/bin
-    curl -sSLo ${WORKSPACE}/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
-    chmod +x ${WORKSPACE}/bin/kind
-    kind version
-    which kind
-
-    mkdir -p ${WORKSPACE}/bin
-    curl -sSLo ${WORKSPACE}/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
-    chmod +x ${WORKSPACE}/bin/kubectl
-    kubectl version --client
-    which kubectl
-}
-
-with_go() {
-    # FIXME add retry logic
-    mkdir -p ${WORKSPACE}/bin
-    curl -sL -o ${WORKSPACE}/bin/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-linux-amd64"
-    chmod +x ${WORKSPACE}/bin/gvm
-    eval "$(gvm $(cat .go-version))"
-    go version
-    which go
-}
-
-with_docker_compose() {
-    # FIXME add retry logic
-    mkdir -p ${WORKSPACE}/bin
-    curl -SL -o ${WORKSPACE}/bin/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64"
-    chmod +x ${WORKSPACE}/bin/docker-compose
-    docker-compose version
-}
+source .buildkite/scripts/install_deps.sh
 
 TARGET=""
 PACKAGE=""

.buildkite/scripts/signAndPublishPackage.sh

@@ -1,6 +1,20 @@
 #!/bin/bash
 set -euo pipefail
 
+WORKSPACE="$(pwd)"
+TMP_FOLDER_TEMPLATE_BASE="tmp.elastic-package"
+
+cleanup() {
+    echo "Deleting temporal files..."
+    cd ${WORKSPACE}
+    rm -rf ${TMP_FOLDER_TEMPLATE_BASE}.*
+    echo "Done."
+}
+
+trap cleanup EXIT
+
+export PATH="${WORKSPACE}/bin:${PATH}"
+
 echo "Checking gsutil command..."
 if ! command -v gsutil &> /dev/null ; then
     echo "⚠️  gsutil is not installed"
@@ -9,4 +23,146 @@ else
     echo "✅ gsutil is installed"
 fi
 
-gsutil help
+source .buildkite/scripts/install_deps.sh
+source .buildkite/scripts/tooling.sh
+
+isAlreadyPublished() {
+    local packageZip=$1
+
+    if curl -s --head https://package-storage.elastic.co/artifacts/packages/${packageZip} | grep -q "HTTP/2 200" ; then
+        echo "- Already published ${packageZip}"
+        return 0
+    fi
+    echo "- Not published ${packageZip}"
+    return 1
+}
+
+REPO_NAME=$(repoName "${BUILDKITE_REPO}")
+BUILD_TAG="buildkite-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_NUMBER}"
+
+REPO_BUILD_TAG="${REPO_NAME}/${BUILD_TAG}"
+
+BUILD_PACKAGES_PATH="build/packages"
+TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
+JENKINS_TRIGGER_PATH=".buildkite/scripts/triggerJenkinsJob"
+GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
+
+## Signing
+INFRA_SIGNING_BUCKET_NAME='internal-ci-artifacts'
+INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER="${REPO_BUILD_TAG}/signed-artifacts"
+INFRA_SIGNING_BUCKET_ARTIFACTS_PATH="gs://${INFRA_SIGNING_BUCKET_NAME}/${REPO_BUILD_TAG}"
+INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH="gs://${INFRA_SIGNING_BUCKET_NAME}/${INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_SUBFOLDER}"
+
+## Publishing
+PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH="gs://elastic-bekitzur-package-storage-internal/queue-publishing/${REPO_BUILD_TAG}"
+
+
+google_cloud_auth_signing() {
+    local gsUtilLocation=$(mktemp -d -p . -t ${TMP_FOLDER_TEMPLATE})
+
+    local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
+    echo "${SIGNING_PACKAGES_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
+
+    google_cloud_auth "${secretFileLocation}"
+
+    echo "${gsUtilLocation}"
+}
+
+google_cloud_auth_publishing() {
+    local gsUtilLocation=$(mktemp -d -p . -t ${TMP_FOLDER_TEMPLATE})
+
+    local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
+    echo "${PACKAGE_UPLOADER_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
+
+    google_cloud_auth "${secretFileLocation}"
+
+    echo "${gsUtilLocation}"
+}
+
+signPackage() {
+    local package=${1}
+    local packageZip=$(basename ${package})
+
+    local gsUtilLocation=$(google_cloud_auth_signing)
+
+    # upload zip package (trailing forward slashes are required)
+    echo "Upload package .zip file for signing ${package} to ${INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}"
+    gsutil cp ${package} "${INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}/"
+
+    echo "Trigger Jenkins job for signing package ${packageZip}"
+    pushd ${JENKINS_TRIGGER_PATH} > /dev/null
+
+    go run main.go \
+        --jenkins-job sign \
+        --folder ${INFRA_SIGNING_BUCKET_ARTIFACTS_PATH}
+
+    sleep 5
+    popd > /dev/null
+
+    echo "Download signatures"
+    gsutil cp "${INFRA_SIGNING_BUCKET_SIGNED_ARTIFACTS_PATH}/${packageZip}.asc" "${BUILD_PACKAGES_PATH}"
+
+    echo "Rename asc to sig"
+    for f in $(ls ${BUILD_PACKAGES_PATH}/*.asc); do
+        mv "$f" "${f%.asc}.sig"
+    done
+
+    ls -l "${BUILD_PACKAGES_PATH}"
+
+    echo "Removing temporal location ${gsUtilLocation}"
+    rm -r "${gsUtilLocation}"
+}
+
+publishPackage() {
+    local package=$1
+    local packageZip=$(basename ${package})
+
+    # create file with credentials
+    local gsUtilLocation=$(google_cloud_auth_publishing)
+
+    # upload files (trailing forward slashes are required)
+    echo "Upload package .zip file ${package} to ${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}"
+    gsutil cp ${package} "${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/"
+    echo "Upload package .sig file ${package}.sig to ${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}"
+    gsutil cp ${package}.sig "${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/"
+
+    echo "Trigger Jenkins job for publishing package ${packageZip}"
+    pushd ${JENKINS_TRIGGER_PATH} > /dev/null
+
+    go run main.go \
+        --jenkins-job publish \
+        --package "${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/${packageZip}" \
+        --signature "${PACKAGE_STORAGE_INTERNAL_BUCKET_QUEUE_PUBLISHING_PATH}/${packageZip}.sig"
+
+    sleep 5
+
+    popd > /dev/null
+
+    echo "Removing temporal location ${gsUtilLocation}"
+    rm -r "${gsUtilLocation}"
+}
+
+# Required to trigger Jenkins job
+with_go
+
+# download package artifact from previous step
+mkdir -p "${BUILD_PACKAGES_PATH}"
+
+buildkite-agent artifact download "${BUILD_PACKAGES_PATH}/*.zip" --step build-package .
+echo "Show artifacts downloaded from previous step ${BUILD_PACKAGES_PATH}"
+ls -l "${BUILD_PACKAGES_PATH}"
+
+for package in $(ls ${BUILD_PACKAGES_PATH}/*.zip); do
+    echo "isAlreadyInstalled ${package}?"
+    packageZip=$(basename ${package})
+    if isAlreadyPublished ${packageZip} ; then
+        echo "Skipping. ${packageZip} already published"
+        continue
+    fi
+
+    echo "Signing package ${packageZip}"
+    signPackage "${package}"
+
+    echo "Publishing package ${packageZip}"
+    publishPackage "${package}"
+done

.buildkite/scripts/tooling.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -euo pipefail
+
+repoName() {
+    # Example of URL: git@github.com:acme-inc/my-project.git
+    local repoUrl=$1
+
+    orgAndRepo=$(echo $repoUrl | cut -d':' -f 2)
+    echo "$(basename ${orgAndRepo} .git)"
+}
+
+google_cloud_auth() {
+    local keyFile=$1
+
+    gcloud auth activate-service-account --key-file ${keyFile} 2> /dev/null
+
+    export GOOGLE_APPLICATIONS_CREDENTIALS=${secretFileLocation}
+}
+
+retry() {
+    local retries=$1
+    shift
+
+    local count=0
+    until "$@"; do
+        exit=$?
+        wait=$((2 ** count))
+        count=$((count + 1))
+        if [ $count -lt "$retries" ]; then
+            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+            sleep $wait
+        else
+            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
+}
+

.buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go

@@ -50,7 +50,11 @@ func (j *JenkinsClient) RunJob(ctx context.Context, jobName string, async bool,
 		return fmt.Errorf("not finished job %s/%d: %w", jobName, build.GetBuildNumber(), err)
 	}
 
-	log.Printf("Build %s finished with result: %v\n", build.GetUrl(), build.GetBuildNumber(), build.GetResult())
+	log.Printf("Build %s finished with result: %s\n", build.GetUrl(), build.GetResult())
+
+	if build.GetResult() != gojenkins.STATUS_SUCCESS {
+		return fmt.Errorf("build %s finished with result %s", build.GetUrl(), build.GetResult())
+	}
 	return nil
 }
 
@@ -94,16 +98,18 @@ func (j *JenkinsClient) getBuildFromQueueID(ctx context.Context, job *gojenkins.
 }
 
 func (j *JenkinsClient) waitForBuildFinished(ctx context.Context, build *gojenkins.Build) error {
+	const waitingPeriod = 10000 * time.Millisecond
 	for build.IsRunning(ctx) {
-		log.Printf("Build still running, waiting for 5 secs...")
+		log.Printf("Build still running, waiting for %s...", waitingPeriod)
 		select {
-		case <-time.After(5000 * time.Millisecond):
+		case <-time.After(waitingPeriod):
 		case <-ctx.Done():
 			return ctx.Err()
 		}
-		_, err = build.Poll(ctx)
+		_, err := build.Poll(ctx)
 		if err != nil {
 			return err
 		}
 	}
+	return nil
 }