Add script to trigger Jenkins jobs

.buildkite/hooks/pre-command

@@ -25,6 +25,7 @@ function retry {
 GCP_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/gcp-service-account
 AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_account_auth
 GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token
+JENKINS_API_TOKEN_PATH=kv/ci-shared/platform-ingest/jenkins_api_tokens
 
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
@@ -50,3 +51,9 @@ fi
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "release" ]]; then
     export GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
 fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-package-storage-publish" && "$BUILDKITE_STEP_KEY" == "sign-publish" ]]; then
+    export JENKINS_USERNAME_SECRET=$(retry 5 vault kv get -field username ${JENKINS_API_TOKEN_PATH})
+    export JENKINS_HOST_SECRET=$(retry 5 vault kv get -field internal_ci_host ${JENKINS_API_TOKEN_PATH})
+    export JENKINS_TOKEN=$(retry 5 vault kv get -field internal_ci ${JENKINS_API_TOKEN_PATH})
+fi

.buildkite/pipeline.package-storage-publish.yml

@@ -1,3 +1,20 @@
 steps:
   - label: "Example Test"
     command: echo "Hello!"
+
+  - label: ":go: Build package"
+    key: build-package
+    command:
+      - "make install"
+      - "cd test/packages/package-storage/package_storage_candidate; elastic-package build -v --zip"
+    agents:
+      image: "golang:1.19.5"
+      cpu: "8"
+      memory: "4G"
+
+  - label: "Test"
+    key: sign-publish
+    command: ".buildkite/scripts/signAndPublishPackage.sh"
+    agents:
+      provider: "gcp"
+      image: family/core-ubuntu-2004

.buildkite/scripts/signAndPublishPackage.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -euo pipefail
+
+echo "Checking gsutil command..."
+if ! command -v gsutil &> /dev/null ; then
+    echo "⚠️  gsutil is not installed"
+    exit 1
+else
+    echo "✅ gsutil is installed"
+fi
+
+gsutil help

.buildkite/scripts/triggerJenkinsJob/go.mod

@@ -0,0 +1,7 @@
+module github.com/elastic/trigger-jenkins-buildkite-plugin
+
+go 1.19
+
+require github.com/bndr/gojenkins v1.1.0
+
+require golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa // indirect

.buildkite/scripts/triggerJenkinsJob/go.sum

@@ -0,0 +1,17 @@
+github.com/bndr/gojenkins v1.1.0 h1:TWyJI6ST1qDAfH33DQb3G4mD8KkrBfyfSUoZBHQAvPI=
+github.com/bndr/gojenkins v1.1.0/go.mod h1:QeskxN9F/Csz0XV/01IC8y37CapKKWvOHa0UHLLX1fM=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

.buildkite/scripts/triggerJenkinsJob/jenkins/jenkins.go

@@ -0,0 +1,109 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package jenkins
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/bndr/gojenkins"
+)
+
+type JenkinsClient struct {
+	client *gojenkins.Jenkins
+}
+
+func NewJenkinsClient(ctx context.Context, host, user, token string) (*JenkinsClient, error) {
+	jenkins, err := gojenkins.CreateJenkins(nil, host, user, token).Init(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("client coult not be created: %w", err)
+	}
+
+	return &JenkinsClient{
+		client: jenkins,
+	}, nil
+}
+
+func (j *JenkinsClient) RunJob(ctx context.Context, jobName string, async bool, params map[string]string) error {
+	queueId, err := j.client.BuildJob(ctx, jobName, params)
+	if err != nil {
+		fmt.Printf("error running job %s : %s\n", jobName, err)
+		return err
+	}
+	build, err := j.getBuildFromJobAndQueueID(ctx, jobName, queueId)
+	if err != nil {
+		return err
+	}
+	log.Printf("Job triggered %s/%d\n", jobName, build.GetBuildNumber())
+
+	if async {
+		return nil
+	}
+
+	log.Printf("Waiting to be finished %s\n", build.GetUrl())
+	err = j.waitForBuildFinished(ctx, build)
+	if err != nil {
+		return fmt.Errorf("not finished job %s/%d: %w", jobName, build.GetBuildNumber(), err)
+	}
+
+	log.Printf("Build %s finished with result: %v\n", build.GetUrl(), build.GetBuildNumber(), build.GetResult())
+	return nil
+}
+
+func (j *JenkinsClient) getBuildFromJobAndQueueID(ctx context.Context, jobName string, queueId int64) (*gojenkins.Build, error) {
+	job, err := j.client.GetJob(ctx, jobName)
+	if err != nil {
+		return nil, fmt.Errorf("not able to get job %s: %w", jobName, err)
+	}
+
+	build, err := j.getBuildFromQueueID(ctx, job, queueId)
+	if err != nil {
+		return nil, fmt.Errorf("not able to get build from %s: %w", jobName, err)
+	}
+	return build, nil
+}
+
+// based on https://github.com/bndr/gojenkins/blob/master/jenkins.go#L282
+func (j *JenkinsClient) getBuildFromQueueID(ctx context.Context, job *gojenkins.Job, queueid int64) (*gojenkins.Build, error) {
+	task, err := j.client.GetQueueItem(ctx, queueid)
+	if err != nil {
+		return nil, err
+	}
+	// Jenkins queue API has about 4.7second quiet period
+	for task.Raw.Executable.Number == 0 {
+		select {
+		case <-time.After(1000 * time.Millisecond):
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+		_, err = task.Poll(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	build, err := job.GetBuild(ctx, task.Raw.Executable.Number)
+	if err != nil {
+		return nil, fmt.Errorf("not able to retrieve build %s", task.Raw.Executable.Number, err)
+	}
+	return build, nil
+}
+
+func (j *JenkinsClient) waitForBuildFinished(ctx context.Context, build *gojenkins.Build) error {
+	for build.IsRunning(ctx) {
+		log.Printf("Build still running, waiting for 5 secs...")
+		select {
+		case <-time.After(5000 * time.Millisecond):
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+		_, err = build.Poll(ctx)
+		if err != nil {
+			return err
+		}
+	}
+}

.buildkite/scripts/triggerJenkinsJob/main.go

@@ -0,0 +1,95 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/elastic/trigger-jenkins-buildkite-plugin/jenkins"
+)
+
+const (
+	publishingRemoteJob = "package_storage/job/publishing-job-remote"
+	signingJob          = "elastic+unified-release+master+sign-artifacts-wigh-gpg"
+
+	publishJobKey = "publish"
+	signJobKey    = "sign"
+)
+
+var allowedJenkinsJobs = map[string]string{
+	publishJobKey: publishingRemoteJob,
+	signJobKey:    signingJob,
+}
+
+var (
+	jenkinsHost  = os.Getenv("JENKINS_HOST_SECRET")
+	jenkinsUser  = os.Getenv("JENKINS_USERNAME_SECRET")
+	jenkinsToken = os.Getenv("JENKINS_TOKEN")
+)
+
+func jenkinsJobOptions() []string {
+	keys := make([]string, 0, len(allowedJenkinsJobs))
+	for k := range allowedJenkinsJobs {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
+func main() {
+	jenkinsJob := flag.String("jenkins-job", "", fmt.Sprintf("Jenkins job to trigger. Allowed values: %s", strings.Join(jenkinsJobOptions(), " ,")))
+	zipPackagePath := flag.String("package", "", "Path to zip package file (*.zip) ")
+	sigPackagePath := flag.String("signature", "", "Path to the signature file of the package file (*.zip.sig)")
+	async := flag.Bool("async", false, "Run async the Jenkins job")
+	flag.Parse()
+
+	if _, ok := allowedJenkinsJobs[*jenkinsJob]; !ok {
+		log.Fatal("Invalid jenkins job")
+	}
+
+	log.Printf("Triggering job: %s", allowedJenkinsJobs[*jenkinsJob])
+
+	ctx := context.Background()
+	client, err := jenkins.NewJenkinsClient(ctx, jenkinsHost, jenkinsUser, jenkinsToken)
+	if err != nil {
+		log.Fatalf("error creating jenkins client")
+	}
+
+	switch *jenkinsJob {
+	case publishJobKey:
+		err = runPublishingRemoteJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *zipPackagePath, *sigPackagePath)
+	case signJobKey:
+		err = runSignPackageJob(ctx, client, *async, allowedJenkinsJobs[*jenkinsJob], *zipPackagePath)
+	default:
+		log.Fatal("unsupported jenkins job")
+	}
+
+	if err != nil {
+		log.Fatal("Error: %s", err)
+	}
+}
+
+func runSignPackageJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, packagePath string) error {
+	params := map[string]string{}
+	// TODO set parameters for sign job
+
+	return client.RunJob(ctx, jobName, async, params)
+}
+
+func runPublishingRemoteJob(ctx context.Context, client *jenkins.JenkinsClient, async bool, jobName, packagePath, signaturePath string) error {
+
+	// Run the job with some parameters
+	params := map[string]string{
+		"dry_run":                   "true",
+		"gs_package_build_zip_path": packagePath,
+		"gs_package_signature_path": signaturePath,
+	}
+
+	return client.RunJob(ctx, jobName, async, params)
+}

tools/tools.go

@@ -10,8 +10,9 @@ package tools
 // https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module
 import (
 	_ "github.com/boumenot/gocover-cobertura"
-	_ "github.com/elastic/go-licenser"
 	_ "golang.org/x/tools/cmd/goimports"
 	_ "gotest.tools/gotestsum"
 	_ "honnef.co/go/tools/cmd/staticcheck"
+
+	_ "github.com/elastic/go-licenser"
 )