diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index ec76774033..6fdbea036c 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -25,6 +25,7 @@ function retry { GCP_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/gcp-service-account AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_account_auth GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token +JENKINS_API_TOKEN_PATH=kv/ci-shared/platform-ingest/jenkins_api_tokens # Secrets must be redacted # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables @@ -50,3 +51,9 @@ fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "release" ]]; then export GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH}) fi + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-package-storage-publish" && "$BUILDKITE_STEP_KEY" == "sign-publish" ]]; then + export JENKINS_USERNAME_SECRET=$(retry 5 vault kv get -field username ${JENKINS_API_TOKEN_PATH}) + export JENKINS_HOST_SECRET=$(retry 5 vault kv get -field internal_ci_host ${JENKINS_API_TOKEN_PATH}) + export JENKINS_TOKEN=$(retry 5 vault kv get -field internal_ci ${JENKINS_API_TOKEN_PATH}) +fi diff --git a/.buildkite/pipeline.package-storage-publish.yml b/.buildkite/pipeline.package-storage-publish.yml index 136b3056a3..cc489cde81 100644 --- a/.buildkite/pipeline.package-storage-publish.yml +++ b/.buildkite/pipeline.package-storage-publish.yml @@ -1,3 +1,20 @@ steps: - label: "Example Test" command: echo "Hello!" + + - label: ":go: Build package" + key: build-package + command: + - "make install" + - "cd test/packages/package-storage/package_storage_candidate; elastic-package build -v --zip" + agents: + image: "golang:1.19.5" + cpu: "8" + memory: "4G" + + - label: "Test" + key: sign-publish + command: ".buildkite/scripts/signAndPublishPackage.sh" + agents: + provider: "gcp" + image: family/core-ubuntu-2004 diff --git a/.buildkite/scripts/signAndPublishPackage.sh b/.buildkite/scripts/signAndPublishPackage.sh new file mode 100755 index 0000000000..e81e3630d7 --- /dev/null +++ b/.buildkite/scripts/signAndPublishPackage.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +echo "Checking gsutil command..." +if ! command -v gsutil &> /dev/null ; then + echo "⚠️ gsutil is not installed" + exit 1 +else + echo "✅ gsutil is installed" +fi + +gsutil help diff --git a/.buildkite/scripts/triggerJenkinsJob/go.mod b/.buildkite/scripts/triggerJenkinsJob/go.mod new file mode 100644 index 0000000000..60a7a4ceae --- /dev/null +++ b/.buildkite/scripts/triggerJenkinsJob/go.mod @@ -0,0 +1,7 @@ +module github.com/elastic/trigger-jenkins-buildkite-plugin + +go 1.19 + +require github.com/bndr/gojenkins v1.1.0 + +require golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa // indirect diff --git a/.buildkite/scripts/triggerJenkinsJob/go.sum b/.buildkite/scripts/triggerJenkinsJob/go.sum new file mode 100644 index 0000000000..5c00aafc03 --- /dev/null +++ b/.buildkite/scripts/triggerJenkinsJob/go.sum @@ -0,0 +1,17 @@ +github.com/bndr/gojenkins v1.1.0 h1:TWyJI6ST1qDAfH33DQb3G4mD8KkrBfyfSUoZBHQAvPI= +github.com/bndr/gojenkins v1.1.0/go.mod h1:QeskxN9F/Csz0XV/01IC8y37CapKKWvOHa0UHLLX1fM= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/.buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go b/.buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go new file mode 100644 index 0000000000..58b27df1f8 --- /dev/null +++ b/.buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go @@ -0,0 +1,109 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package jenkins + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/bndr/gojenkins" +) + +type JenkinsClient struct { + client *gojenkins.Jenkins +} + +func NewJenkinsClient(ctx context.Context, host, user, token string) (*JenkinsClient, error) { + jenkins, err := gojenkins.CreateJenkins(nil, host, user, token).Init(ctx) + if err != nil { + return nil, fmt.Errorf("client coult not be created: %w", err) + } + + return &JenkinsClient{ + client: jenkins, + }, nil +} + +func (j *JenkinsClient) RunJob(ctx context.Context, jobName string, async bool, params map[string]string) error { + queueId, err := j.client.BuildJob(ctx, jobName, params) + if err != nil { + fmt.Printf("error running job %s : %s\n", jobName, err) + return err + } + build, err := j.getBuildFromJobAndQueueID(ctx, jobName, queueId) + if err != nil { + return err + } + log.Printf("Job triggered %s/%d\n", jobName, build.GetBuildNumber()) + + if async { + return nil + } + + log.Printf("Waiting to be finished %s\n", build.GetUrl()) + err = j.waitForBuildFinished(ctx, build) + if err != nil { + return fmt.Errorf("not finished job %s/%d: %w", jobName, build.GetBuildNumber(), err) + } + + log.Printf("Build %s finished with result: %v\n", build.GetUrl(), build.GetBuildNumber(), build.GetResult()) + return nil +} + +func (j *JenkinsClient) getBuildFromJobAndQueueID(ctx context.Context, jobName string, queueId int64) (*gojenkins.Build, error) { + job, err := j.client.GetJob(ctx, jobName) + if err != nil { + return nil, fmt.Errorf("not able to get job %s: %w", jobName, err) + } + + build, err := j.getBuildFromQueueID(ctx, job, queueId) + if err != nil { + return nil, fmt.Errorf("not able to get build from %s: %w", jobName, err) + } + return build, nil +} + +// based on https://github.com/bndr/gojenkins/blob/master/jenkins.go#L282 +func (j *JenkinsClient) getBuildFromQueueID(ctx context.Context, job *gojenkins.Job, queueid int64) (*gojenkins.Build, error) { + task, err := j.client.GetQueueItem(ctx, queueid) + if err != nil { + return nil, err + } + // Jenkins queue API has about 4.7second quiet period + for task.Raw.Executable.Number == 0 { + select { + case <-time.After(1000 * time.Millisecond): + case <-ctx.Done(): + return nil, ctx.Err() + } + _, err = task.Poll(ctx) + if err != nil { + return nil, err + } + } + + build, err := job.GetBuild(ctx, task.Raw.Executable.Number) + if err != nil { + return nil, fmt.Errorf("not able to retrieve build %s", task.Raw.Executable.Number, err) + } + return build, nil +} + +func (j *JenkinsClient) waitForBuildFinished(ctx context.Context, build *gojenkins.Build) error { + for build.IsRunning(ctx) { + log.Printf("Build still running, waiting for 5 secs...") + select { + case <-time.After(5000 * time.Millisecond): + case <-ctx.Done(): + return ctx.Err() + } + _, err = build.Poll(ctx) + if err != nil { + return err + } + } +} diff --git a/.buildkite/scripts/triggerJenkinsJob/main.go b/.buildkite/scripts/triggerJenkinsJob/main.go new file mode 100644 index 0000000000..ff6019f070 --- /dev/null +++ b/.buildkite/scripts/triggerJenkinsJob/main.go @@ -0,0 +1,95 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "strings" + + "github.com/elastic/trigger-jenkins-buildkite-plugin/jenkins" +) + +const ( + publishingRemoteJob = "package_storage/job/publishing-job-remote" + signingJob = "elastic+unified-release+master+sign-artifacts-wigh-gpg" + + publishJobKey = "publish" + signJobKey = "sign" +) + +var allowedJenkinsJobs = map[string]string{ + publishJobKey: publishingRemoteJob, + signJobKey: signingJob, +} + +var ( + jenkinsHost = os.Getenv("JENKINS_HOST_SECRET") + jenkinsUser = os.Getenv("JENKINS_USERNAME_SECRET") + jenkinsToken = os.Getenv("JENKINS_TOKEN") +) + +func jenkinsJobOptions() []string { + keys := make([]string, 0, len(allowedJenkinsJobs)) + for k := range allowedJenkinsJobs { + keys = append(keys, k) + } + return keys +} + +func main() { + jenkinsJob := flag.String("jenkins-job", "", fmt.Sprintf("Jenkins job to trigger. Allowed values: %s", strings.Join(jenkinsJobOptions(), " ,"))) + zipPackagePath := flag.String("package", "", "Path to zip package file (*.zip) ") + sigPackagePath := flag.String("signature", "", "Path to the signature file of the package file (*.zip.sig)") + async := flag.Bool("async", false, "Run async the Jenkins job") + flag.Parse() + + if _, ok := allowedJenkinsJobs[*jenkinsJob]; !ok { + log.Fatal("Invalid jenkins job") + } + + log.Printf("Triggering job: %s", allowedJenkinsJobs[*jenkinsJob]) + + ctx := context.Background() + client, err := jenkins.NewJenkinsClient(ctx, jenkinsHost, jenkinsUser, jenkinsToken) + if err != nil { + log.Fatalf("error creating jenkins client") + } + + switch *jenkinsJob { + case publishJobKey: + err = runPublishingRemoteJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *zipPackagePath, *sigPackagePath) + case signJobKey: + err = runSignPackageJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *zipPackagePath) + default: + log.Fatal("unsupported jenkins job") + } + + if err != nil { + log.Fatal("Error: %s", err) + } +} + +func runSignPackageJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, packagePath string) error { + params := map[string]string{} + // TODO set parameters for sign job + + return client.RunJob(ctx, jobName, async, params) +} + +func runPublishingRemoteJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, packagePath, signaturePath string) error { + + // Run the job with some parameters + params := map[string]string{ + "dry_run": "true", + "gs_package_build_zip_path": packagePath, + "gs_package_signature_path": signaturePath, + } + + return client.RunJob(ctx, jobName, async, params) +} diff --git a/tools/tools.go b/tools/tools.go index 3cb45488e9..0e68450929 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -10,8 +10,9 @@ package tools // https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module import ( _ "github.com/boumenot/gocover-cobertura" - _ "github.com/elastic/go-licenser" _ "golang.org/x/tools/cmd/goimports" _ "gotest.tools/gotestsum" _ "honnef.co/go/tools/cmd/staticcheck" + + _ "github.com/elastic/go-licenser" )