From 258637d7d569cb53e832bf412da5b9418c18c0aa Mon Sep 17 00:00:00 2001 From: Bird Date: Sat, 23 Jul 2022 17:21:12 +0800 Subject: [PATCH 1/5] feat: use `gojenkins` and complete `read`, `update`, `delete` interface Signed-off-by: Bird --- .../plugins/jenkins-pipeline-kubernetes.zh.md | 9 +- go.mod | 3 +- go.sum | 6 +- .../jenkinspipelinekubernetes/client.go | 153 ------------------ .../jenkinspipelinekubernetes/create.go | 59 ++++--- .../jenkinspipelinekubernetes/create_test.go | 54 +++++++ .../jenkinspipelinekubernetes/delete.go | 25 ++- .../jenkinspipelinekubernetes.go | 25 ++- .../jenkinspipelinekubernetes_suite_test.go | 2 +- .../job-template.xml | 2 +- .../jenkinspipelinekubernetes/options.go | 23 +-- .../plugin/jenkinspipelinekubernetes/read.go | 26 ++- .../jenkinspipelinekubernetes/update.go | 7 +- .../jenkinspipelinekubernetes/validate.go | 61 ++++--- .../plugins/jenkins-pipeline-kubernetes.yaml | 19 ++- pkg/util/jenkins/client.go | 106 ++++++++++++ pkg/util/jenkins/custom.go | 61 +++++++ pkg/util/jenkins/jenkins_test.go | 48 ++++++ 18 files changed, 458 insertions(+), 231 deletions(-) delete mode 100644 internal/pkg/plugin/jenkinspipelinekubernetes/client.go create mode 100644 internal/pkg/plugin/jenkinspipelinekubernetes/create_test.go create mode 100644 pkg/util/jenkins/client.go create mode 100644 pkg/util/jenkins/custom.go create mode 100644 pkg/util/jenkins/jenkins_test.go diff --git a/docs/plugins/jenkins-pipeline-kubernetes.zh.md b/docs/plugins/jenkins-pipeline-kubernetes.zh.md index 1b9d4a3d2..707f39ef0 100644 --- a/docs/plugins/jenkins-pipeline-kubernetes.zh.md +++ b/docs/plugins/jenkins-pipeline-kubernetes.zh.md @@ -4,11 +4,10 @@ 步骤： -1. 访问 Jenkins web UI，创建 token。步骤：People -> admin ->Configure -> API Token -> Add new Token。 -2. 按需修改配置项，其中 `githubRepoUrl` 为 GitHub 仓库地址，应预先建立一个 GitHub 仓库，并创建一个名为 "Jenkinsfile" 的文件放至仓库根目录。 -3. 设置环境变量 +1. 按需修改配置项，其中 `githubRepoUrl` 为 GitHub 仓库地址，应预先建立一个 GitHub 仓库，并创建一个名为 "Jenkinsfile" 的文件放至仓库根目录。 +2. 设置环境变量 - `GITHUB_TOKEN` - - `JENKINS_TOKEN` + - `JENKINS_PASSWORD` ## 用例 @@ -17,5 +16,3 @@ --8<-- "jenkins-pipeline-kubernetes.yaml" ``` - -目前，所有选项均为必填项。 diff --git a/go.mod b/go.mod index 157843c10..747bdd7bd 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.16.3 github.com/aws/aws-sdk-go-v2/config v1.15.5 github.com/aws/aws-sdk-go-v2/service/s3 v1.26.9 + github.com/bndr/gojenkins v1.1.0 github.com/cenkalti/backoff v2.2.1+incompatible github.com/cheggaaa/pb v1.0.29 github.com/deckarep/golang-set/v2 v2.1.0 @@ -27,7 +28,7 @@ require ( github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.7.0 github.com/tcnksm/go-input v0.0.0-20180404061846-548a7d7a8ee8 - github.com/withfig/autocomplete-tools/integrations/cobra v0.0.0-20220705165518-2761d7f4b8bc + github.com/withfig/autocomplete-tools/integrations/cobra v0.0.0-20220721102007-67b2515c5ea4 github.com/xanzy/go-gitlab v0.55.1 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 diff --git a/go.sum b/go.sum index 366888300..c5ca0029a 100644 --- a/go.sum +++ b/go.sum @@ -213,6 +213,8 @@ github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bndr/gojenkins v1.1.0 h1:TWyJI6ST1qDAfH33DQb3G4mD8KkrBfyfSUoZBHQAvPI= +github.com/bndr/gojenkins v1.1.0/go.mod h1:QeskxN9F/Csz0XV/01IC8y37CapKKWvOHa0UHLLX1fM= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bombsimon/logrusr v1.0.0 h1:CTCkURYAt5nhCCnKH9eLShYayj2/8Kn/4Qg3QfiU+Ro= github.com/bombsimon/logrusr v1.0.0/go.mod h1:Jq0nHtvxabKE5EMwAAdgTaz7dfWE8C4i11NOltxGQpc= @@ -1268,8 +1270,8 @@ github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= -github.com/withfig/autocomplete-tools/integrations/cobra v0.0.0-20220705165518-2761d7f4b8bc h1:2pGkMttK5jQ8+6YhdyeQIHyVa84HMdJhILozImSWX6c= -github.com/withfig/autocomplete-tools/integrations/cobra v0.0.0-20220705165518-2761d7f4b8bc/go.mod h1:nmuySobZb4kFgFy6BptpXp/BBw+xFSyvVPP6auoJB4k= +github.com/withfig/autocomplete-tools/integrations/cobra v0.0.0-20220721102007-67b2515c5ea4 h1:GoQuMyofxqcdyzWqVhsv5IsCL+wHoocrhUgzhtB2Nj4= +github.com/withfig/autocomplete-tools/integrations/cobra v0.0.0-20220721102007-67b2515c5ea4/go.mod h1:nmuySobZb4kFgFy6BptpXp/BBw+xFSyvVPP6auoJB4k= github.com/xanzy/go-gitlab v0.55.1 h1:IgX/DS9buV0AUz8fuJPQkdl0fQGfBiAsAHxpun8sNhg= github.com/xanzy/go-gitlab v0.55.1/go.mod h1:F0QEXwmqiBUxCgJm8fE9S+1veX4XC9Z4cfaAbqwk4YM= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= diff --git a/internal/pkg/plugin/jenkinspipelinekubernetes/client.go b/internal/pkg/plugin/jenkinspipelinekubernetes/client.go deleted file mode 100644 index 2a537517e..000000000 --- a/internal/pkg/plugin/jenkinspipelinekubernetes/client.go +++ /dev/null @@ -1,153 +0,0 @@ -package jenkinspipelinekubernetes - -import ( - _ "embed" - "fmt" - "os/exec" - "strings" - - "github.com/parnurzeal/gorequest" - - "github.com/devstream-io/devstream/pkg/util/log" -) - -// ClientInf represents the client abstraction for jenkins -type ClientInf interface { - CreateCredentialSecretText() error - CreateCredentialUsernamePassword() error - GetCrumb() (string, error) - GetCrumbHeader() (headerKey, headerValue string, err error) - CreateItem(jobXmlContent string) error -} - -// Client is the client for jenkins -type Client struct { - Opts *Options -} - -func NewClient(options *Options) *Client { - return &Client{ - Opts: options, - } -} - -//func (c *Client) ifCredentialExists() bool { -// // todo -// return false -//} - -// CreateCredentialSecretText creates a credential in the type of "Secret text" -func (c *Client) CreateCredentialSecretText() error { - - accessURL := c.Opts.GetJenkinsAccessURL() - crumb, err := c.GetCrumb() - if err != nil { - return fmt.Errorf("failed to create credential secret: %s", err) - } - - // TODO(aFlyBird0): use gorequest to do the request - cmdCreateCredential := fmt.Sprintf(` -curl -H %s -X POST '%s/credentials/store/system/domain/_/createCredentials' \ ---data-urlencode 'json={ - "": "0", - "credentials": { - "scope": "GLOBAL", - "id": "%s", - "secret": "%s", - "description": "%s", - "$class": "org.jenkinsci.plugins.plaincredentials.impl.StringCredentialsImpl" - } -}'`, crumb, accessURL, jenkinsCredentialID, c.Opts.GitHubToken, jenkinsCredentialDesc) - - cmd := exec.Command("sh", "-c", cmdCreateCredential) - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to create credential secret: %s", err) - } - - return nil -} - -// CreateCredentialUsernamePassword creates a credential in the type of "Username with password" -func (c *Client) CreateCredentialUsernamePassword() error { - - accessURL := c.Opts.GetJenkinsAccessURL() - crumb, err := c.GetCrumb() - if err != nil { - return fmt.Errorf("failed to create credential secret: %s", err) - } - - // TODO(aFlyBird0): use gorequest to do the request - cmdCreateCredential := fmt.Sprintf(` -curl -H %s -X POST '%s/credentials/store/system/domain/_/createCredentials' \ ---data-urlencode 'json={ - "": "0", - "credentials": { - "scope": "GLOBAL", - "id": "%s", - "username": "foo-useless-username", - "password": "%s", - "description": "%s", - "$class": "com.cloudbees.plugins.credentials.impl.UsernamePasswordCredentialsImpl" - } -}'`, crumb, accessURL, jenkinsCredentialID, c.Opts.GitHubToken, jenkinsCredentialDesc) - - cmd := exec.Command("sh", "-c", cmdCreateCredential) - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to create credential secret: %s", err) - } - - return nil -} - -// GetCrumb returns the crumb for jenkins, -// jenkins uses crumb to prevent CSRF(cross-site request forgery), -// format: "Jenkins-Crumb:a70290b6423777f0a4c771d4805637ac36d5fd78336a20d48d72167ef5f13b9a" -// ref: https://www.jenkins.io/doc/upgrade-guide/2.176/#upgrading-to-jenkins-lts-2-176-3 -// ref: https://stackoverflow.com/questions/44711696/jenkins-403-no-valid-crumb-was-included-in-the-request -func (c *Client) GetCrumb() (string, error) { - request := gorequest.New() - getCrumbURL := c.Opts.GetJenkinsAccessURL() + `/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)` - resp, body, errs := request.Get(getCrumbURL).End() - log.Debugf("GetCrumb url: %s", getCrumbURL) - if len(errs) != 0 { - return "", fmt.Errorf("failed to get crumb: %s", errs) - } - if resp.StatusCode != 200 { - return "", fmt.Errorf("failed to get crumb, here is response: %s", body) - } - - return strings.TrimSpace(body), nil -} - -// GetCrumbHeader behaves like GetCrumb, but it returns the header key and value -func (c *Client) GetCrumbHeader() (headerKey, headerValue string, err error) { - // crumb format: "Jenkins-Crumb:a70290b6423777f0a4c771d4805637ac36d5fd78336a20d48d72167ef5f13b9a" - crumb, err := c.GetCrumb() - if err != nil { - return "", "", err - } - crumbMap := strings.Split(crumb, ":") - if len(crumbMap) != 2 { - return "", "", fmt.Errorf("failed to get crumb, here is response: %s", crumb) - } - return crumbMap[0], crumbMap[1], nil -} - -//go:embed job-template.xml -var jobTemplate string - -// CreateItem creates a job in jenkins with the given job xml -func (c *Client) CreateItem(jobXmlContent string) error { - request := gorequest.New() - resp, body, errs := request.Post(c.Opts.GetJenkinsAccessURL()+"/createItem"). - Set("Content-Type", "application/xml"). - Query("name=" + c.Opts.JenkinsJobName).Send(jobXmlContent).End() - - if len(errs) != 0 { - return fmt.Errorf("failed to create job: %s", errs) - } else if resp.StatusCode != 200 { - return fmt.Errorf("failed to create job, here is response: %s", body) - } - - return nil -} diff --git a/internal/pkg/plugin/jenkinspipelinekubernetes/create.go b/internal/pkg/plugin/jenkinspipelinekubernetes/create.go index 505a0346e..2b9521097 100644 --- a/internal/pkg/plugin/jenkinspipelinekubernetes/create.go +++ b/internal/pkg/plugin/jenkinspipelinekubernetes/create.go @@ -1,6 +1,7 @@ package jenkinspipelinekubernetes import ( + "context" _ "embed" "fmt" "strings" @@ -21,26 +22,40 @@ func Create(options map[string]interface{}) (map[string]interface{}, error) { return nil, err } - if errs := validate(&opts); len(errs) != 0 { + if errs := validateAndHandleOptions(&opts); len(errs) != 0 { for _, e := range errs { log.Errorf("Options error: %s.", e) } return nil, fmt.Errorf("opts are illegal") } - client := NewClient(&opts) - - // always try to create credential, there will be no error if it already exists - if err := client.CreateCredentialUsernamePassword(); err != nil { + // get the jenkins client and test the connection + client, err := NewJenkinsFromOptions(&opts) + if err != nil { return nil, err } - jobXmlContent := renderJobXml(jobTemplate, &opts) - - // TODO(aFlyBird0): check if the job already exists + // create credential if not exists + if _, err := client.GetCredentialsUsername(jenkinsCredentialID); err != nil { + log.Infof("credential %s not found, creating...", jenkinsCredentialID) + if err := client.CreateCredentialsUsername(jenkinsCredentialID, jenkinsCredentialDesc); err != nil { + return nil, err + } + } - if err := client.CreateItem(jobXmlContent); err != nil { - return nil, fmt.Errorf("failed to create job: %s", err) + // create job if not exists + ctx := context.Background() + if _, err := client.GetJob(ctx, opts.J.JobName); err != nil { + log.Infof("job %s not found, creating...", opts.J.JobName) + jobXmlOpts := &JobXmlOptions{ + GitHubRepoURL: opts.GitHubRepoURL, + CredentialsID: jenkinsCredentialID, + PipelineScriptPath: opts.J.PipelineScriptPath, + } + jobXmlContent := renderJobXml(jobTemplate, jobXmlOpts) + if _, err := client.CreateJob(context.Background(), jobXmlContent, opts.J.JobName); err != nil { + return nil, fmt.Errorf("failed to create job: %s", err) + } } // TODO(aFlyBird0): use JCasC to configure job creation @@ -78,20 +93,28 @@ func Create(options map[string]interface{}) (map[string]interface{}, error) { // 4. add "read" functions to the ConfigMap to get the content of the ConfigMap and check if the resource is drifted // 5. maybe we also should consider to expose some config key in ConfigMap to the user - // TODO(aFlyBird0): build dtm resource - // TODO(aFlyBird0): what if user doesn't use helm to install jenkins? then the JCasC may not be automatically reloaded. - return (&resource{}).toMap(), nil + res := &resource{ + CredentialsCreated: true, + JobCreated: true, + } + + return res.toMap(), nil +} + +type JobXmlOptions struct { + GitHubRepoURL string + CredentialsID string + PipelineScriptPath string } // TODO(aFlyBird0): unit test -// TODO(aFlyBird0): now jenkins script path is hardcoded to "Jenkinsfile", it should be configurable -func renderJobXml(jobTemplate string, opts *Options) string { - // note: maybe it is better to use html/template to generate the job template, - // but that way is complex and this is the simplest way to do it +func renderJobXml(jobTemplate string, opts *JobXmlOptions) string { + // TODO(aFlyBird0): use html/template to generate the job template jobXml := strings.Replace(jobTemplate, "{{.GitHubRepoURL}}", opts.GitHubRepoURL, 1) - jobXml = strings.Replace(jobXml, "{{.CredentialsID}}", jenkinsCredentialID, 1) + jobXml = strings.Replace(jobXml, "{{.CredentialsID}}", opts.CredentialsID, 1) + jobXml = strings.Replace(jobXml, "{{.PipelineScriptPath}}", opts.PipelineScriptPath, 1) log.Debugf("job xml rendered: %s", jobXml) return jobXml diff --git a/internal/pkg/plugin/jenkinspipelinekubernetes/create_test.go b/internal/pkg/plugin/jenkinspipelinekubernetes/create_test.go new file mode 100644 index 000000000..9748fbdd5 --- /dev/null +++ b/internal/pkg/plugin/jenkinspipelinekubernetes/create_test.go @@ -0,0 +1,54 @@ +package jenkinspipelinekubernetes + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Create", func() { + Describe("renderJobXml function", func() { + var opts *JobXmlOptions + BeforeEach(func() { + opts = &JobXmlOptions{ + GitHubRepoURL: "https://github.com/xxx/jenkins-file-test.git", + CredentialsID: "credential-jenkins-pipeline-kubernetes-by-devstream", + PipelineScriptPath: "this-is-pipeline-script-path", + } + }) + + It("should return the correct xml", func() { + xml := renderJobXml(jobTemplate, opts) + expect := ` + + + false + + + + 2 + + + https://github.com/xxx/jenkins-file-test.git + credential-jenkins-pipeline-kubernetes-by-devstream + + + + + */main + + + false + + + + this-is-pipeline-script-path + false + + + false + +` + Expect(xml).To(Equal(expect)) + }) + }) +}) diff --git a/internal/pkg/plugin/jenkinspipelinekubernetes/delete.go b/internal/pkg/plugin/jenkinspipelinekubernetes/delete.go index 49d3a1910..48a9d885b 100644 --- a/internal/pkg/plugin/jenkinspipelinekubernetes/delete.go +++ b/internal/pkg/plugin/jenkinspipelinekubernetes/delete.go @@ -1,6 +1,7 @@ package jenkinspipelinekubernetes import ( + "context" "fmt" "github.com/mitchellh/mapstructure" @@ -14,16 +15,34 @@ func Delete(options map[string]interface{}) (bool, error) { return false, err } - if errs := validate(&opts); len(errs) != 0 { + if errs := validateAndHandleOptions(&opts); len(errs) != 0 { for _, e := range errs { log.Errorf("Options error: %s.", e) } return false, fmt.Errorf("opts are illegal") } - // TODO(aFlyBird0): use helm uninstall to delete the resource created by devstream // TODO(aFlyBird0): filter ConfigMaps Created by devstream in the "jenkins" namespace and delete them(filter by label) - // TODO(aFlyBird0): delete other resources created by devstream(if exists) + + // get the jenkins client and test the connection + client, err := NewJenkinsFromOptions(&opts) + if err != nil { + return false, err + } + + // delete the credentials created by devstream if exists + if _, err := client.GetCredentialsUsername(jenkinsCredentialID); err == nil { + if err := client.DeleteCredentialsUsername(jenkinsCredentialID); err != nil { + return false, err + } + } + + // delete the job created by devstream if exists + if _, err = client.GetJob(context.Background(), opts.J.JobName); err == nil { + if _, err := client.DeleteJob(context.Background(), opts.J.JobName); err != nil { + return false, err + } + } return true, nil } diff --git a/internal/pkg/plugin/jenkinspipelinekubernetes/jenkinspipelinekubernetes.go b/internal/pkg/plugin/jenkinspipelinekubernetes/jenkinspipelinekubernetes.go index 7dafaac88..c1fdc5bdc 100644 --- a/internal/pkg/plugin/jenkinspipelinekubernetes/jenkinspipelinekubernetes.go +++ b/internal/pkg/plugin/jenkinspipelinekubernetes/jenkinspipelinekubernetes.go @@ -1,9 +1,30 @@ package jenkinspipelinekubernetes -// TODO(aFlyBird0): specify the resource fields here. +import ( + _ "embed" + + "github.com/devstream-io/devstream/pkg/util/jenkins" +) + +//go:embed job-template.xml +var jobTemplate string + +// NewJenkinsFromOptions creates a Jenkins client from the given options and test the connection. +func NewJenkinsFromOptions(opts *Options) (*jenkins.Jenkins, error) { + return jenkins.NewJenkins(opts.J.URL, opts.J.User, opts.J.Password) +} + +// TODO(aFlyBird0): enhance the resource fields here to be read and the way to read it, such as: +// plugins install info(GitHub Pull Request Builder Plugin and OWASP Markup Formatter must be installed) +// should we keep an eye on job configuration && status changes? maybe not. type resource struct { + CredentialsCreated bool + JobCreated bool } func (res *resource) toMap() map[string]interface{} { - return map[string]interface{}{} + return map[string]interface{}{ + "credentialsCreated": res.CredentialsCreated, + "jobCreated": res.JobCreated, + } } diff --git a/internal/pkg/plugin/jenkinspipelinekubernetes/jenkinspipelinekubernetes_suite_test.go b/internal/pkg/plugin/jenkinspipelinekubernetes/jenkinspipelinekubernetes_suite_test.go index 0b0c314a0..ea0efe51c 100644 --- a/internal/pkg/plugin/jenkinspipelinekubernetes/jenkinspipelinekubernetes_suite_test.go +++ b/internal/pkg/plugin/jenkinspipelinekubernetes/jenkinspipelinekubernetes_suite_test.go @@ -1,4 +1,4 @@ -package jenkinspipelinekubernetes_test +package jenkinspipelinekubernetes import ( "testing" diff --git a/internal/pkg/plugin/jenkinspipelinekubernetes/job-template.xml b/internal/pkg/plugin/jenkinspipelinekubernetes/job-template.xml index 3fcf24ee2..271f8dc67 100644 --- a/internal/pkg/plugin/jenkinspipelinekubernetes/job-template.xml +++ b/internal/pkg/plugin/jenkinspipelinekubernetes/job-template.xml @@ -21,7 +21,7 @@ - Jenkinsfile + {{.PipelineScriptPath}} false diff --git a/internal/pkg/plugin/jenkinspipelinekubernetes/options.go b/internal/pkg/plugin/jenkinspipelinekubernetes/options.go index e61878fa0..48454d0cf 100644 --- a/internal/pkg/plugin/jenkinspipelinekubernetes/options.go +++ b/internal/pkg/plugin/jenkinspipelinekubernetes/options.go @@ -1,18 +1,21 @@ package jenkinspipelinekubernetes -import "fmt" +const ( + defaultJenkinsUser = "admin" + defaultJenkinsPipelineScriptPath = "Jenkinsfile" +) // Options is the struct for configurations of the jenkins-pipeline-kubernetes plugin. type Options struct { - JenkinsURL string `mapstructure:"jenkinsUrl" validate:"required,hostname_port"` - JenkinsUser string `mapstructure:"jenkinsUser" validate:"required"` - JenkinsToken string `mapstructure:"jenkinsToken"` - GitHubToken string `mapstructure:"githubToken"` - GitHubRepoURL string `mapstructure:"githubRepoUrl" validate:"required"` - JenkinsJobName string `mapstructure:"jenkinsJobName" validate:"required"` - // TODO(aFlyBird0): add options to configure the script path in GitHub repo, now it is hardcoded to "Jenkinsfile" + J *JenkinsOption `mapstructure:"jenkins"` + GitHubToken string `mapstructure:"githubToken"` + GitHubRepoURL string `mapstructure:"githubRepoUrl" validateAndHandleOptions:"required"` } -func (options *Options) GetJenkinsAccessURL() string { - return fmt.Sprintf("http://%s:%s@%s", options.JenkinsUser, options.JenkinsToken, options.JenkinsURL) +type JenkinsOption struct { + URL string `mapstructure:"url" validateAndHandleOptions:"required,hostname_port"` + User string `mapstructure:"user" validateAndHandleOptions:"required"` + Password string `mapstructure:"password"` + JobName string `mapstructure:"jobName"` + PipelineScriptPath string `mapstructure:"pipelineScriptPath"` } diff --git a/internal/pkg/plugin/jenkinspipelinekubernetes/read.go b/internal/pkg/plugin/jenkinspipelinekubernetes/read.go index 18ea8f638..855e73513 100644 --- a/internal/pkg/plugin/jenkinspipelinekubernetes/read.go +++ b/internal/pkg/plugin/jenkinspipelinekubernetes/read.go @@ -1,6 +1,7 @@ package jenkinspipelinekubernetes import ( + "context" "fmt" "github.com/mitchellh/mapstructure" @@ -14,19 +15,28 @@ func Read(options map[string]interface{}) (map[string]interface{}, error) { return nil, err } - if errs := validate(&opts); len(errs) != 0 { + if errs := validateAndHandleOptions(&opts); len(errs) != 0 { for _, e := range errs { log.Errorf("Options error: %s.", e) } return nil, fmt.Errorf("opts are illegal") } - // TODO(aFlyBird0): specify the resource to be read and the way to read it, such as: - // plugins install info(GitHub Pull Request Builder Plugin and OWASP Markup Formatter must be installed) - // job list(filter the jobs which are created by devstream) - // credential list(filter the credentials which are created by devstream)(if credential created by devstream exists) - // JCasC configuration - // job configuration && status + // get the jenkins client and test the connection + client, err := NewJenkinsFromOptions(&opts) + if err != nil { + return nil, err + } + + res := &resource{} + + if _, err = client.GetCredentialsUsername(jenkinsCredentialID); err == nil { + res.CredentialsCreated = true + } + + if _, err = client.GetJob(context.Background(), opts.J.JobName); err == nil { + res.JobCreated = true + } - return nil, nil + return res.toMap(), nil } diff --git a/internal/pkg/plugin/jenkinspipelinekubernetes/update.go b/internal/pkg/plugin/jenkinspipelinekubernetes/update.go index dfc7f07fd..2715770bf 100644 --- a/internal/pkg/plugin/jenkinspipelinekubernetes/update.go +++ b/internal/pkg/plugin/jenkinspipelinekubernetes/update.go @@ -14,7 +14,7 @@ func Update(options map[string]interface{}) (map[string]interface{}, error) { return nil, err } - if errs := validate(&opts); len(errs) != 0 { + if errs := validateAndHandleOptions(&opts); len(errs) != 0 { for _, e := range errs { log.Errorf("Options error: %s.", e) } @@ -26,5 +26,8 @@ func Update(options map[string]interface{}) (map[string]interface{}, error) { // some, we should only call some update function // others, we just ignore them - return (&resource{}).toMap(), nil + // now we just use the same way as create, + // because the logic is the same: "if not exists, create; if exists, do nothing" + // if it changes in the future, we should change the way to update + return Create(options) } diff --git a/internal/pkg/plugin/jenkinspipelinekubernetes/validate.go b/internal/pkg/plugin/jenkinspipelinekubernetes/validate.go index e0c291737..aaace0391 100644 --- a/internal/pkg/plugin/jenkinspipelinekubernetes/validate.go +++ b/internal/pkg/plugin/jenkinspipelinekubernetes/validate.go @@ -3,43 +3,64 @@ package jenkinspipelinekubernetes import ( "fmt" "os" - "strings" "github.com/devstream-io/devstream/pkg/util/validator" ) -// validate validates the options provided by the core. -func validate(options *Options) []error { +// validateAndHandleOptions validates and pre handle the options provided by the core. +func validateAndHandleOptions(options *Options) []error { + validateErrs := validate(options) + + defaults(options) + + envErrs := handleEnv(options) + + return append(validateErrs, envErrs...) +} - // pre-handle options to remove "http://" from JenkinsURL - preHandleOptions(options) +func validate(options *Options) []error { + return validator.Struct(options) +} - var retErrs []error +func defaults(options *Options) { + // pre handle the options + if options.J.PipelineScriptPath == "" { + options.J.PipelineScriptPath = defaultJenkinsPipelineScriptPath + } - if errs := validator.Struct(options); len(errs) != 0 { - retErrs = append(retErrs, errs...) + if options.J.User == "" { + options.J.User = defaultJenkinsUser } + options.J.URL = "http://" + options.J.URL +} + +func handleEnv(options *Options) []error { + var errs []error + options.GitHubToken = os.Getenv("GITHUB_TOKEN") + if options.GitHubToken == "" { + errs = append(errs, fmt.Errorf("env GITHUB_TOKEN is required")) + } // TODO(aFlyBird0): now jenkins token should be provided by the user, // so, user should install jenkins first and stop to set the token in env, then install this pipeline plugin. // could we generate a token automatically in "jenkins" plugin? // and put it into .outputs of "jenkins" plugin, // so that user could run "jenkins" and "jenkins-pipeline-kubernetes" in the same tool file.(using depends on). - options.JenkinsToken = os.Getenv("JENKINS_TOKEN") - if options.GitHubToken == "" { - retErrs = append(retErrs, fmt.Errorf("GITHUB_TOKEN is required")) - } - if options.JenkinsToken == "" { - retErrs = append(retErrs, fmt.Errorf("JENKINS_TOKEN is required")) - } + //options.J.Token = os.Getenv("JENKINS_TOKEN") + //if options.J.Token == "" { + // errs = append(errs, fmt.Errorf("env JENKINS_TOKEN is required")) + //} - // TODO(aFlyBird0): check if the jenkins url is valid (try to connect to jenkins) + // read the password from config file(including the outputs from last plugin) first, then from env + if options.J.Password == "" { + options.J.Password = os.Getenv("JENKINS_PASSWORD") + } - return retErrs -} + if options.J.Password == "" { + errs = append(errs, fmt.Errorf("env JENKINS_PASSWORD is required")) + } -func preHandleOptions(options *Options) { - options.JenkinsURL = strings.Replace(options.JenkinsURL, "http://", "", 1) + return errs } diff --git a/internal/pkg/show/config/plugins/jenkins-pipeline-kubernetes.yaml b/internal/pkg/show/config/plugins/jenkins-pipeline-kubernetes.yaml index df02d82bb..3382047e0 100644 --- a/internal/pkg/show/config/plugins/jenkins-pipeline-kubernetes.yaml +++ b/internal/pkg/show/config/plugins/jenkins-pipeline-kubernetes.yaml @@ -7,8 +7,19 @@ tools: dependsOn: [ ] # options for the plugin options: - # jenkinsUrl, format: hostname:port - jenkinsUrl: localhost:8080 - jenkinsUser: admin + jenkins: + # jenkinsUrl, format: hostname:port + url: localhost:8080 + # jenkins user name, default: admin + user: admin + # jenkins password, you have 2 options to set the password: + # 1. set the `JENKINS_PASSWORD` environment variable + # 2. fill in the password in this field(not recommended) + # if all set, devstream will read the password from the config file first. + password: + # jenkins job name, mandatory + jobName: + # path to the pipeline file, relative to the git repo root directory. default: Jenkinsfile + pipelineScriptPath: Jenkinsfile + # github repo url where the pipeline script is located. mandatory githubRepoUrl: https://github.com/xxx/xxx.git - jenkinsJobName: diff --git a/pkg/util/jenkins/client.go b/pkg/util/jenkins/client.go new file mode 100644 index 000000000..bb4909bcf --- /dev/null +++ b/pkg/util/jenkins/client.go @@ -0,0 +1,106 @@ +package jenkins + +import ( + "context" + "fmt" + + "github.com/bndr/gojenkins" +) + +const ( + domain = "_" + credentialScope = "GLOBAL" +) + +type ( + // Jenkins is a Jenkins client, + // it is a wrapper around gojenkins.Jenkins and implements additional methods + Jenkins struct { + *gojenkins.Jenkins + + *BasicAuth + JenkinsURL string + } + + BasicAuth struct { + Username string + Password string + } +) + +// NewJenkins creates a new Jenkins client and validates the connection +func NewJenkins(jenkinsURL, username, password string) (*Jenkins, error) { + // validate + if jenkinsURL == "" { + return nil, fmt.Errorf("jenkinsURL is required") + } + if username == "" || password == "" { + return nil, fmt.Errorf("username and password are required") + } + + // create gojenkins client + gojenkinsClient := gojenkins.CreateJenkins(nil, jenkinsURL, username, password) + ctx := context.Background() + // init and validate client + if _, err := gojenkinsClient.Init(ctx); err != nil { + return nil, err + } + + return &Jenkins{ + Jenkins: gojenkinsClient, + BasicAuth: &BasicAuth{ + Username: username, + Password: password, + }, + JenkinsURL: jenkinsURL, + }, nil +} + +func (j *Jenkins) GetCredentialManager() *gojenkins.CredentialsManager { + return &gojenkins.CredentialsManager{ + J: j.Jenkins, + } +} + +func (j *Jenkins) CreateCredentialsUsername(id, description string) error { + cred := gojenkins.UsernameCredentials{ + ID: id, + Scope: credentialScope, + Username: j.Username, + Password: j.Password, + Description: description, + } + + // create credential + ctx := context.Background() + cm := j.GetCredentialManager() + err := cm.Add(ctx, domain, cred) + if err != nil { + return fmt.Errorf("could not create credential: %v", err) + } + + // get credential to validate creation + getCred := gojenkins.UsernameCredentials{} + if err = cm.GetSingle(ctx, domain, cred.ID, &getCred); err != nil { + return fmt.Errorf("could not get credential: %v", err) + } + + return nil +} + +// GetCredentialsUsername returns the credentials of type username-password with the given id, +// it returns an error if the credential does not exist +func (j *Jenkins) GetCredentialsUsername(id string) (*gojenkins.UsernameCredentials, error) { + getCred := gojenkins.UsernameCredentials{} + ctx := context.Background() + cm := j.GetCredentialManager() + err := cm.GetSingle(ctx, domain, id, &getCred) + if err != nil { + return nil, fmt.Errorf("could not get credential: %v", err) + } + return &getCred, nil +} + +func (j *Jenkins) DeleteCredentialsUsername(id string) error { + return j.GetCredentialManager().Delete(context.Background(), domain, id) +} diff --git a/pkg/util/jenkins/custom.go b/pkg/util/jenkins/custom.go new file mode 100644 index 000000000..8ea8d7436 --- /dev/null +++ b/pkg/util/jenkins/custom.go @@ -0,0 +1,61 @@ +package jenkins + +import ( + _ "embed" + "fmt" + + "github.com/parnurzeal/gorequest" + + "github.com/devstream-io/devstream/pkg/util/log" +) + +// this file is used to write custom implementation of jenkins client + +// GetCrumb returns the crumb for jenkins, +// jenkins uses crumb to prevent CSRF(cross-site request forgery), +// ref: https://www.jenkins.io/doc/upgrade-guide/2.176/#upgrading-to-jenkins-lts-2-176-3 +// ref: https://stackoverflow.com/questions/44711696/jenkins-403-no-valid-crumb-was-included-in-the-request +func (j *Jenkins) GetCrumb() (crumbHeaderKey, crumbHeaderValue string, cookie string, err error) { + // crumb response format: + type CrumbResponse struct { + Crumb string `json:"crumb"` + CrumbRequestField string `json:"crumbRequestField"` + } + var crumbResp CrumbResponse + + // get crumb + request := gorequest.New() + getCrumbURL := j.JenkinsURL + `/crumbIssuer/api/json` + resp, body, errs := request.Get(getCrumbURL). + SetBasicAuth(j.BasicAuth.Username, j.BasicAuth.Password). + EndStruct(&crumbResp) + + // check error + if len(errs) != 0 { + return "", "", "", fmt.Errorf("failed to get jenkins crumb: %s", errs) + } + if resp.StatusCode != 200 { + return "", "", "", fmt.Errorf("failed to get jenkins crumb, here is response: %s", body) + } + + log.Debugf("crumb: %+v", crumbResp) + + return crumbResp.CrumbRequestField, crumbResp.Crumb, resp.Header.Get("set-cookie"), nil + +} + +// SetCrumb sets the jenkins crumb to the request +func (j *Jenkins) SetCrumb(req *gorequest.SuperAgent) error { + crumbHeaderKey, crumbHeaderValue, cookie, err := j.GetCrumb() + if err != nil { + return err + } + + // all these three should be set in the request, or it will cause 403 + req.Set(crumbHeaderKey, crumbHeaderValue) + req.Set("Cookie", cookie) + req.SetBasicAuth(j.BasicAuth.Username, j.BasicAuth.Password) + + return nil + +} diff --git a/pkg/util/jenkins/jenkins_test.go b/pkg/util/jenkins/jenkins_test.go new file mode 100644 index 000000000..1d0cd22d4 --- /dev/null +++ b/pkg/util/jenkins/jenkins_test.go @@ -0,0 +1,48 @@ +package jenkins + +import ( + "fmt" + "testing" + + "github.com/parnurzeal/gorequest" +) + +const jenkinsCredentialID = "credential-jenkins-pipeline-kubernetes-by-devstream" + +func initJenkins() *Jenkins { + jenkinsURL := "http://localhost:32001/" + username := "admin" + password := "B1OvhHMnPPxXz4kFwODYIh" + jenkins, err := NewJenkins(jenkinsURL, username, password) + if err != nil { + panic(err) + } + return jenkins +} + +func TestGetCredentialsUsername(t *testing.T) { + cd, err := initJenkins().GetCredentialsUsername(jenkinsCredentialID) + if err != nil { + t.Errorf("Error: %v", err) + } + fmt.Printf("%+v\n", cd) +} + +func TestDeleteJob(t *testing.T) { + req := gorequest.New().Post("http://localhost:32001/job/jenkins-plugin-test/doDelete") + j := initJenkins() + if err := j.SetCrumb(req); err != nil { + t.Errorf("Error: %v", err) + } + req.SetBasicAuth("admin", "B1OvhHMnPPxXz4kFwODYIh") + req.Debug = true + status, body, err := req.End() + if err != nil { + t.Errorf("Error: %v", err) + } + if status.StatusCode != 200 { + t.Errorf("Error: %v", status) + fmt.Printf("%+v\n", body) + } + +} From 93801b8af5312c5f1b87fe90eed53476e78a7e01 Mon Sep 17 00:00:00 2001 From: Bird Date: Sat, 23 Jul 2022 22:40:53 +0800 Subject: [PATCH 2/5] feat: connect jenkins and pipeline plugins Signed-off-by: Bird --- .../plugins/jenkins-pipeline-kubernetes.zh.md | 101 ++++++++++++++++ docs/plugins/jenkins.md | 7 ++ docs/plugins/jenkins.zh.md | 7 ++ internal/pkg/plugin/jenkins/create.go | 2 +- internal/pkg/plugin/jenkins/jenkins.go | 51 +++++++++ internal/pkg/plugin/jenkins/prompt.go | 108 ++++++++++++++++-- internal/pkg/plugin/jenkins/read.go | 2 +- internal/pkg/plugin/jenkins/update.go | 2 +- .../jenkinspipelinekubernetes/validate.go | 2 +- .../plugins/jenkins-pipeline-kubernetes.yaml | 5 +- pkg/util/jenkins/jenkins_test.go | 48 -------- 11 files changed, 273 insertions(+), 62 deletions(-) delete mode 100644 pkg/util/jenkins/jenkins_test.go diff --git a/docs/plugins/jenkins-pipeline-kubernetes.zh.md b/docs/plugins/jenkins-pipeline-kubernetes.zh.md index 707f39ef0..fb14d9b4d 100644 --- a/docs/plugins/jenkins-pipeline-kubernetes.zh.md +++ b/docs/plugins/jenkins-pipeline-kubernetes.zh.md @@ -16,3 +16,104 @@ --8<-- "jenkins-pipeline-kubernetes.yaml" ``` + +## 和 `jenkins` 插件一起使用 + +这个插件可以和 `jenkins` 插件一起使用，[`jenkins` 插件文档](./jenkins.zh.md)。 + +即在安装完 `Jenkins` 后，再建立 `Jenkins` job。 + +首先根据 `dependsOn` 设定插件依赖，再根据 `${{jenkins.default.outputs.jenkinsURL}}` 和 `${{jenkins.default.outputs.jenkinsPasswordOfAdmin}}` 设置 Jenkins 的 URL 和 admin 密码。 + +注意：如果你的 Kubernetes 集群是 K8s in docker 模式，请自行设置网络，确保 `jenkins` 插件中 `NodePort` 设置的端口在宿主机内能访问。 + +```yaml +--- +tools: + # name of the tool + - name: jenkins + # id of the tool instance + instanceID: default + # format: name.instanceID; If specified, dtm will make sure the dependency is applied first before handling this tool. + dependsOn: [ ] + # options for the plugin + options: + # if true, the plugin will use hostpath to create a pv named `jenkins-pv` + # and you should create the volumes directory manually, see plugin doc for details. + test_env: false + # need to create the namespace or not, default: false + create_namespace: false + # Helm repo information + repo: + # name of the Helm repo + name: jenkins + # url of the Helm repo + url: https://charts.jenkins.io + # Helm chart information + chart: + # name of the chart + chart_name: jenkins/jenkins + # release name of the chart + release_name: dev + # k8s namespace where jenkins will be installed + namespace: jenkins + # whether to wait for the release to be deployed or not + wait: true + # the time to wait for any individual Kubernetes operation (like Jobs for hooks). This defaults to 5m0s + timeout: 5m + # whether to perform a CRD upgrade during installation + upgradeCRDs: true + # custom configuration. You can refer to [Jenkins values.yaml](https://github.com/jenkinsci/helm-charts/blob/main/charts/jenkins/values.yaml) + values_yaml: | + persistence: + # for prod env: the existent storageClass, please change it + # for test env: just ignore it, but don't remove it + storageClass: jenkins-pv + serviceAccount: + create: false + name: jenkins + controller: + serviceType: NodePort + nodePort: 32000 + additionalPlugins: + # install "GitHub Pull Request Builder" plugin, see https://plugins.jenkins.io/ghprb/ for more details + - ghprb + # install "OWASP Markup Formatter" plugin, see https://plugins.jenkins.io/antisamy-markup-formatter/ for more details + - antisamy-markup-formatter + # Enable HTML parsing using OWASP Markup Formatter Plugin (antisamy-markup-formatter), useful with ghprb plugin. + enableRawHtmlMarkupFormatter: true + # Jenkins Configuraction as Code, refer to https://plugins.jenkins.io/configuration-as-code/ for more details + # notice: All configuration files that are discovered MUST be supplementary. They cannot overwrite each other's configuration values. This creates a conflict and raises a ConfiguratorException. + JCasC: + defaultConfig: true + # each key-value in configScripts will be added to the ${JENKINS_HOME}/casc_configs/ directory as a file. + configScripts: + # this will create a file named "safe_html.yaml" in the ${JENKINS_HOME}/casc_configs/ directory. + # it is used to configure the "Safe HTML" plugin. + # filename must meet RFC 1123, see https://tools.ietf.org/html/rfc1123 for more details + - name: jenkins-pipeline-kubernetes + # id of the tool instance + instanceID: default + # format: name.instanceID; If specified, dtm will make sure the dependency is applied first before handling this tool. + dependsOn: [ "jenkins.default" ] + # options for the plugin + options: + jenkins: + # jenkinsUrl, format: hostname:port + url: ${{jenkins.default.outputs.jenkinsURL}} + # jenkins user name, default: admin + user: admin + # jenkins password, you have 2 options to set the password: + # 1. use outputs of the `jenkins` plugin, see docs for more details + # 2. set the `JENKINS_PASSWORD` environment variable + # 3. fill in the password in this field(not recommended) + # if all set, devstream will read the password from the config file first. + password: ${{jenkins.default.outputs.jenkinsPasswordOfAdmin}} + # jenkins job name, mandatory + jobName: + # path to the pipeline file, relative to the git repo root directory. default: Jenkinsfile + pipelineScriptPath: Jenkinsfile + # github repo url where the pipeline script is located. mandatory + githubRepoUrl: https://github.com/xxx/xxx.git +``` + diff --git a/docs/plugins/jenkins.md b/docs/plugins/jenkins.md index 1de434ddf..b472f4ad5 100644 --- a/docs/plugins/jenkins.md +++ b/docs/plugins/jenkins.md @@ -47,3 +47,10 @@ chown -R 1000:1000 /data/jenkins-volume/ ``` Currently, all the parameters in the example above are mandatory. + +## Outputs + +This plugin has two outputs: + +- `jenkinsURL` (format: `hostname:port`, example: "localhost:8080") +- `jenkinsPasswordOfAdmin` diff --git a/docs/plugins/jenkins.zh.md b/docs/plugins/jenkins.zh.md index 6c685b514..0381699df 100644 --- a/docs/plugins/jenkins.zh.md +++ b/docs/plugins/jenkins.zh.md @@ -47,3 +47,10 @@ chown -R 1000:1000 /data/jenkins-volume/ ``` 当前，所有配置项均为必填。 + +## 输出 + +这个插件有两个输出： + +- `jenkinsURL` (格式: `hostname:port`, 例如: "localhost:8080") +- `jenkinsPasswordOfAdmin` diff --git a/internal/pkg/plugin/jenkins/create.go b/internal/pkg/plugin/jenkins/create.go index fcbb19396..005fb5f11 100644 --- a/internal/pkg/plugin/jenkins/create.go +++ b/internal/pkg/plugin/jenkins/create.go @@ -26,7 +26,7 @@ func Create(options map[string]interface{}) (map[string]interface{}, error) { TermateOperations: []plugininstaller.BaseOperation{ helm.DealWithNsWhenInterruption, }, - GetStatusOperation: helm.GetPluginStaticStateByReleaseNameWrapper(defaultStatefulsetTplList), + GetStatusOperation: wrapperHelmResourceAndCustomResource(options, helm.GetPluginStaticStateByReleaseNameWrapper(defaultStatefulsetTplList)), } // 2. execute installer get status and error diff --git a/internal/pkg/plugin/jenkins/jenkins.go b/internal/pkg/plugin/jenkins/jenkins.go index b11309f68..93663ce7e 100644 --- a/internal/pkg/plugin/jenkins/jenkins.go +++ b/internal/pkg/plugin/jenkins/jenkins.go @@ -1,6 +1,57 @@ package jenkins +import "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + var defaultStatefulsetTplList = []string{ // ${release-name}-jenkins "%s-jenkins", } + +// wrapperHelmResourceAndCustomResource wraps helm resource and custom resource, +// this is due to the limitation of `plugininstaller`, +// now `plugininstaller.GetStatusOperation` only support one resource get function, +// if we want to use both existing resource get function(such as helm's methods) and custom function, +// we have to wrap them into one function. +func wrapperHelmResourceAndCustomResource(options plugininstaller.RawOptions, helmResFunc plugininstaller.StatusOperation) plugininstaller.StatusOperation { + return func(options plugininstaller.RawOptions) (map[string]interface{}, error) { + opts, err := newOptions(options) + if err != nil { + return nil, err + } + + // 1. get helm resource + resource, err := helmResFunc(options) + if err != nil { + return nil, err + } + + // 2. get custom resource, and merge with helm resource + outputs := map[string]interface{}{} + // 2.1 get jenkins url + // TODO(aFlyBird0): TestEnv is not strictly as same as "K8s in docker" + if !opts.TestEnv { + jenkinsURL, err := getJenkinsURL(options) + if err != nil { + return nil, err + } + outputs["jenkinsURL"] = jenkinsURL + } else { + jenkinsURLForTestEnv, err := getJenkinsURLForTestEnv(options) + if err != nil { + return nil, err + } + outputs["jenkinsURL"] = jenkinsURLForTestEnv + } + + // 2.2 get jenkins password of admin + jenkinsPassword, err := getPasswdOfAdmin(options) + if err != nil { + return nil, err + } + outputs["jenkinsPasswordOfAdmin"] = jenkinsPassword + + resource["outputs"] = outputs + + return resource, nil + } +} diff --git a/internal/pkg/plugin/jenkins/prompt.go b/internal/pkg/plugin/jenkins/prompt.go index c05c00e95..f17bddd9b 100644 --- a/internal/pkg/plugin/jenkins/prompt.go +++ b/internal/pkg/plugin/jenkins/prompt.go @@ -9,38 +9,130 @@ import ( "github.com/devstream-io/devstream/pkg/util/log" ) +func buildPasswdOfAdminCommand(opts jenkinsOptions) string { + method := fmt.Sprintf("kubectl exec --namespace jenkins -it svc/%s-jenkins -c jenkins "+ + "-- /bin/cat /run/secrets/additional/chart-admin-password && echo", opts.Chart.ReleaseName) + + return method +} + func howToGetPasswdOfAdmin(options plugininstaller.RawOptions) error { opts, err := newOptions(options) if err != nil { return err } + log.Info("Here is how to get the password of the admin user:") - method := fmt.Sprintf("kubectl exec --namespace jenkins -it svc/%s-jenkins -c jenkins "+ - "-- /bin/cat /run/secrets/additional/chart-admin-password && echo", opts.Chart.ReleaseName) - log.Info(method) + command := buildPasswdOfAdminCommand(opts) + log.Info(command) + return nil } -func showJenkinsUrl(options plugininstaller.RawOptions) error { +func getPasswdOfAdmin(options plugininstaller.RawOptions) (string, error) { opts, err := newOptions(options) if err != nil { - return err + return "", err + } + + commandString := buildPasswdOfAdminCommand(opts) + command := exec.Command("sh", "-c", commandString) + + password, err := command.Output() + if err != nil { + return "", fmt.Errorf("failed to get password of admin user: %v", err) } + + return strings.TrimSpace(string(password)), nil + +} + +// getJenkinsURL returns the jenkins url of the jenkins, format: hostname:port +func getJenkinsURL(options plugininstaller.RawOptions) (string, error) { + opts, err := newOptions(options) + if err != nil { + return "", err + } + commands := []string{ `jsonpath="{.spec.ports[0].nodePort}"`, fmt.Sprintf(`NODE_PORT=$(kubectl get -n jenkins -o jsonpath=$jsonpath services %s-jenkins)`, opts.Chart.ReleaseName), `jsonpath="{.items[0].status.addresses[0].address}"`, `NODE_IP=$(kubectl get nodes -n jenkins -o jsonpath=$jsonpath)`, - `echo http://$NODE_IP:$NODE_PORT/login`, + `echo $NODE_IP:$NODE_PORT`, } cmd := exec.Command("sh", "-c", strings.Join(commands, " && ")) output, err := cmd.Output() if err != nil { - log.Errorf("Failed to get jenkins url: %v", err) + return "", fmt.Errorf("Failed to get jenkins url: %v", err) + } + + return strings.TrimSpace(string(output)), nil +} + +// getJenkinsURL behaves like getJenkinsURL, but the hostname will be always 127.0.0.1 +func getJenkinsURLForTestEnv(options plugininstaller.RawOptions) (string, error) { + opts, err := newOptions(options) + if err != nil { + return "", err + } + + commands := []string{ + `jsonpath="{.spec.ports[0].nodePort}"`, + fmt.Sprintf(`NODE_PORT=$(kubectl get -n jenkins -o jsonpath=$jsonpath services %s-jenkins)`, opts.Chart.ReleaseName), + `jsonpath="{.items[0].status.addresses[0].address}"`, + `NODE_IP=127.0.0.1`, + `echo $NODE_IP:$NODE_PORT`, + } + + cmd := exec.Command("sh", "-c", strings.Join(commands, " && ")) + + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("Failed to get jenkins url: %v", err) + } + + return strings.TrimSpace(string(output)), nil +} + +func showJenkinsUrl(options plugininstaller.RawOptions) error { + opts, err := newOptions(options) + if err != nil { + return err + } + + // prod env: just print Jenkins url + if !opts.TestEnv { + url, err := getJenkinsURL(options) + if err != nil { + log.Error(err) + return err + } + + log.Infof("Jenkins url: http://%s/login", url) + } + + // test env: print Jenkins url in host machine and Jenkins url in K8s cluster + if opts.TestEnv { + log.Info("You are in test env. Here are the Jenkins url in host machine and Jenkins url in K8s cluster.") + + urlForTestEnv, err := getJenkinsURLForTestEnv(options) + if err != nil { + log.Error(err) + return err + } + log.Infof("Jenkins url in host machine: http://%s/login", urlForTestEnv) + + urlInK8s, err := getJenkinsURL(options) + if err != nil { + log.Error(err) + return err + } + log.Info(", Jenkins url in K8s:", fmt.Sprintf("http://%s/login", urlInK8s)) + } - log.Info("Jenkins url:", string(output)) return nil } diff --git a/internal/pkg/plugin/jenkins/read.go b/internal/pkg/plugin/jenkins/read.go index 966235eae..b46164c56 100644 --- a/internal/pkg/plugin/jenkins/read.go +++ b/internal/pkg/plugin/jenkins/read.go @@ -13,7 +13,7 @@ func Read(options map[string]interface{}) (map[string]interface{}, error) { helm.Validate, replaceStroageClass, }, - GetStatusOperation: helm.GetPluginAllState, + GetStatusOperation: wrapperHelmResourceAndCustomResource(options, helm.GetPluginAllState), } status, err := runner.Execute(plugininstaller.RawOptions(options)) diff --git a/internal/pkg/plugin/jenkins/update.go b/internal/pkg/plugin/jenkins/update.go index d29dc5674..5b3f87692 100644 --- a/internal/pkg/plugin/jenkins/update.go +++ b/internal/pkg/plugin/jenkins/update.go @@ -20,7 +20,7 @@ func Update(options map[string]interface{}) (map[string]interface{}, error) { TermateOperations: []plugininstaller.BaseOperation{ helm.DealWithNsWhenInterruption, }, - GetStatusOperation: helm.GetPluginStaticStateByReleaseNameWrapper(defaultStatefulsetTplList), + GetStatusOperation: wrapperHelmResourceAndCustomResource(options, helm.GetPluginStaticStateByReleaseNameWrapper(defaultStatefulsetTplList)), } // 2. execute installer get status and error diff --git a/internal/pkg/plugin/jenkinspipelinekubernetes/validate.go b/internal/pkg/plugin/jenkinspipelinekubernetes/validate.go index aaace0391..ea884e43e 100644 --- a/internal/pkg/plugin/jenkinspipelinekubernetes/validate.go +++ b/internal/pkg/plugin/jenkinspipelinekubernetes/validate.go @@ -59,7 +59,7 @@ func handleEnv(options *Options) []error { } if options.J.Password == "" { - errs = append(errs, fmt.Errorf("env JENKINS_PASSWORD is required")) + errs = append(errs, fmt.Errorf("jenkins password is required")) } return errs diff --git a/internal/pkg/show/config/plugins/jenkins-pipeline-kubernetes.yaml b/internal/pkg/show/config/plugins/jenkins-pipeline-kubernetes.yaml index 3382047e0..9abeb67c1 100644 --- a/internal/pkg/show/config/plugins/jenkins-pipeline-kubernetes.yaml +++ b/internal/pkg/show/config/plugins/jenkins-pipeline-kubernetes.yaml @@ -13,8 +13,9 @@ tools: # jenkins user name, default: admin user: admin # jenkins password, you have 2 options to set the password: - # 1. set the `JENKINS_PASSWORD` environment variable - # 2. fill in the password in this field(not recommended) + # 1. use outputs of the `jenkins` plugin, see docs for more details + # 2. set the `JENKINS_PASSWORD` environment variable + # 3. fill in the password in this field(not recommended) # if all set, devstream will read the password from the config file first. password: # jenkins job name, mandatory diff --git a/pkg/util/jenkins/jenkins_test.go b/pkg/util/jenkins/jenkins_test.go deleted file mode 100644 index 1d0cd22d4..000000000 --- a/pkg/util/jenkins/jenkins_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package jenkins - -import ( - "fmt" - "testing" - - "github.com/parnurzeal/gorequest" -) - -const jenkinsCredentialID = "credential-jenkins-pipeline-kubernetes-by-devstream" - -func initJenkins() *Jenkins { - jenkinsURL := "http://localhost:32001/" - username := "admin" - password := "B1OvhHMnPPxXz4kFwODYIh" - jenkins, err := NewJenkins(jenkinsURL, username, password) - if err != nil { - panic(err) - } - return jenkins -} - -func TestGetCredentialsUsername(t *testing.T) { - cd, err := initJenkins().GetCredentialsUsername(jenkinsCredentialID) - if err != nil { - t.Errorf("Error: %v", err) - } - fmt.Printf("%+v\n", cd) -} - -func TestDeleteJob(t *testing.T) { - req := gorequest.New().Post("http://localhost:32001/job/jenkins-plugin-test/doDelete") - j := initJenkins() - if err := j.SetCrumb(req); err != nil { - t.Errorf("Error: %v", err) - } - req.SetBasicAuth("admin", "B1OvhHMnPPxXz4kFwODYIh") - req.Debug = true - status, body, err := req.End() - if err != nil { - t.Errorf("Error: %v", err) - } - if status.StatusCode != 200 { - t.Errorf("Error: %v", status) - fmt.Printf("%+v\n", body) - } - -} From 7e074fd2c21d646a66a41452ea7442ad67370873 Mon Sep 17 00:00:00 2001 From: Bird Date: Sat, 23 Jul 2022 23:29:12 +0800 Subject: [PATCH 3/5] fix: fix credential error in jenkins plugin Signed-off-by: Bird --- docs/plugins/jenkins-pipeline-kubernetes.zh.md | 4 ++-- internal/pkg/plugin/jenkins/prompt.go | 2 +- .../pkg/plugin/jenkinspipelinekubernetes/create.go | 10 +++++----- .../config/plugins/jenkins-pipeline-kubernetes.yaml | 8 ++++---- pkg/util/jenkins/client.go | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/plugins/jenkins-pipeline-kubernetes.zh.md b/docs/plugins/jenkins-pipeline-kubernetes.zh.md index fb14d9b4d..ccefb8840 100644 --- a/docs/plugins/jenkins-pipeline-kubernetes.zh.md +++ b/docs/plugins/jenkins-pipeline-kubernetes.zh.md @@ -103,11 +103,11 @@ tools: url: ${{jenkins.default.outputs.jenkinsURL}} # jenkins user name, default: admin user: admin - # jenkins password, you have 2 options to set the password: + # jenkins password, you have 3 options to set the password: # 1. use outputs of the `jenkins` plugin, see docs for more details # 2. set the `JENKINS_PASSWORD` environment variable # 3. fill in the password in this field(not recommended) - # if all set, devstream will read the password from the config file first. + # if all set, devstream will read the password from the config file or outputs from jenkins plugin first, then env var. password: ${{jenkins.default.outputs.jenkinsPasswordOfAdmin}} # jenkins job name, mandatory jobName: diff --git a/internal/pkg/plugin/jenkins/prompt.go b/internal/pkg/plugin/jenkins/prompt.go index f17bddd9b..74858dc1e 100644 --- a/internal/pkg/plugin/jenkins/prompt.go +++ b/internal/pkg/plugin/jenkins/prompt.go @@ -130,7 +130,7 @@ func showJenkinsUrl(options plugininstaller.RawOptions) error { log.Error(err) return err } - log.Info(", Jenkins url in K8s:", fmt.Sprintf("http://%s/login", urlInK8s)) + log.Info("Jenkins url in K8s:", fmt.Sprintf("http://%s/login", urlInK8s)) } diff --git a/internal/pkg/plugin/jenkinspipelinekubernetes/create.go b/internal/pkg/plugin/jenkinspipelinekubernetes/create.go index 2b9521097..e6b42c2d9 100644 --- a/internal/pkg/plugin/jenkinspipelinekubernetes/create.go +++ b/internal/pkg/plugin/jenkinspipelinekubernetes/create.go @@ -12,8 +12,9 @@ import ( ) const ( - jenkinsCredentialID = "credential-jenkins-pipeline-kubernetes-by-devstream" - jenkinsCredentialDesc = "Jenkins Pipeline secret, created by devstream/jenkins-pipeline-kubernetes" + jenkinsCredentialID = "credential-jenkins-pipeline-kubernetes-by-devstream" + jenkinsCredentialDesc = "Jenkins Pipeline secret, created by devstream/jenkins-pipeline-kubernetes" + jenkinsCredentialUsername = "foo-useless-username" ) func Create(options map[string]interface{}) (map[string]interface{}, error) { @@ -38,7 +39,7 @@ func Create(options map[string]interface{}) (map[string]interface{}, error) { // create credential if not exists if _, err := client.GetCredentialsUsername(jenkinsCredentialID); err != nil { log.Infof("credential %s not found, creating...", jenkinsCredentialID) - if err := client.CreateCredentialsUsername(jenkinsCredentialID, jenkinsCredentialDesc); err != nil { + if err := client.CreateCredentialsUsername(jenkinsCredentialUsername, opts.GitHubToken, jenkinsCredentialID, jenkinsCredentialDesc); err != nil { return nil, err } } @@ -109,9 +110,8 @@ type JobXmlOptions struct { PipelineScriptPath string } -// TODO(aFlyBird0): unit test func renderJobXml(jobTemplate string, opts *JobXmlOptions) string { - // TODO(aFlyBird0): use html/template to generate the job template + // TODO(aFlyBird0): use html/template to generate the job template. It's a good first issue. :) jobXml := strings.Replace(jobTemplate, "{{.GitHubRepoURL}}", opts.GitHubRepoURL, 1) jobXml = strings.Replace(jobXml, "{{.CredentialsID}}", opts.CredentialsID, 1) jobXml = strings.Replace(jobXml, "{{.PipelineScriptPath}}", opts.PipelineScriptPath, 1) diff --git a/internal/pkg/show/config/plugins/jenkins-pipeline-kubernetes.yaml b/internal/pkg/show/config/plugins/jenkins-pipeline-kubernetes.yaml index 9abeb67c1..d74ddbbc9 100644 --- a/internal/pkg/show/config/plugins/jenkins-pipeline-kubernetes.yaml +++ b/internal/pkg/show/config/plugins/jenkins-pipeline-kubernetes.yaml @@ -8,19 +8,19 @@ tools: # options for the plugin options: jenkins: - # jenkinsUrl, format: hostname:port + # jenkinsUrl, format: hostname:port, mandatory url: localhost:8080 # jenkins user name, default: admin user: admin - # jenkins password, you have 2 options to set the password: + # jenkins password, you have 3 options to set the password: # 1. use outputs of the `jenkins` plugin, see docs for more details # 2. set the `JENKINS_PASSWORD` environment variable # 3. fill in the password in this field(not recommended) - # if all set, devstream will read the password from the config file first. + # if all set, devstream will read the password from the config file or outputs from jenkins plugin first, then env var. password: # jenkins job name, mandatory jobName: - # path to the pipeline file, relative to the git repo root directory. default: Jenkinsfile + # path to the pipeline sript file path, relative to the git repo root directory. default: Jenkinsfile pipelineScriptPath: Jenkinsfile # github repo url where the pipeline script is located. mandatory githubRepoUrl: https://github.com/xxx/xxx.git diff --git a/pkg/util/jenkins/client.go b/pkg/util/jenkins/client.go index bb4909bcf..ef74bdb89 100644 --- a/pkg/util/jenkins/client.go +++ b/pkg/util/jenkins/client.go @@ -62,12 +62,12 @@ func (j *Jenkins) GetCredentialManager() *gojenkins.CredentialsManager { } } -func (j *Jenkins) CreateCredentialsUsername(id, description string) error { +func (j *Jenkins) CreateCredentialsUsername(username, password, id, description string) error { cred := gojenkins.UsernameCredentials{ ID: id, Scope: credentialScope, - Username: j.Username, - Password: j.Password, + Username: username, + Password: password, Description: description, } From 641fe689d9820c265dee3a285639e54b737228c4 Mon Sep 17 00:00:00 2001 From: Bird Date: Mon, 25 Jul 2022 09:42:31 +0800 Subject: [PATCH 4/5] refactor(jenkins): update state wrapper Signed-off-by: Bird --- internal/pkg/plugin/jenkins/create.go | 2 +- internal/pkg/plugin/jenkins/jenkins.go | 2 +- internal/pkg/plugin/jenkins/read.go | 2 +- internal/pkg/plugin/jenkins/update.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/pkg/plugin/jenkins/create.go b/internal/pkg/plugin/jenkins/create.go index 005fb5f11..360e06d45 100644 --- a/internal/pkg/plugin/jenkins/create.go +++ b/internal/pkg/plugin/jenkins/create.go @@ -26,7 +26,7 @@ func Create(options map[string]interface{}) (map[string]interface{}, error) { TermateOperations: []plugininstaller.BaseOperation{ helm.DealWithNsWhenInterruption, }, - GetStatusOperation: wrapperHelmResourceAndCustomResource(options, helm.GetPluginStaticStateByReleaseNameWrapper(defaultStatefulsetTplList)), + GetStatusOperation: wrapperHelmResourceAndCustomResource(helm.GetPluginStaticStateByReleaseNameWrapper(defaultStatefulsetTplList)), } // 2. execute installer get status and error diff --git a/internal/pkg/plugin/jenkins/jenkins.go b/internal/pkg/plugin/jenkins/jenkins.go index 93663ce7e..352a1e1a5 100644 --- a/internal/pkg/plugin/jenkins/jenkins.go +++ b/internal/pkg/plugin/jenkins/jenkins.go @@ -12,7 +12,7 @@ var defaultStatefulsetTplList = []string{ // now `plugininstaller.GetStatusOperation` only support one resource get function, // if we want to use both existing resource get function(such as helm's methods) and custom function, // we have to wrap them into one function. -func wrapperHelmResourceAndCustomResource(options plugininstaller.RawOptions, helmResFunc plugininstaller.StatusOperation) plugininstaller.StatusOperation { +func wrapperHelmResourceAndCustomResource(helmResFunc plugininstaller.StatusOperation) plugininstaller.StatusOperation { return func(options plugininstaller.RawOptions) (map[string]interface{}, error) { opts, err := newOptions(options) if err != nil { diff --git a/internal/pkg/plugin/jenkins/read.go b/internal/pkg/plugin/jenkins/read.go index b46164c56..be2e9b029 100644 --- a/internal/pkg/plugin/jenkins/read.go +++ b/internal/pkg/plugin/jenkins/read.go @@ -13,7 +13,7 @@ func Read(options map[string]interface{}) (map[string]interface{}, error) { helm.Validate, replaceStroageClass, }, - GetStatusOperation: wrapperHelmResourceAndCustomResource(options, helm.GetPluginAllState), + GetStatusOperation: wrapperHelmResourceAndCustomResource(helm.GetPluginAllState), } status, err := runner.Execute(plugininstaller.RawOptions(options)) diff --git a/internal/pkg/plugin/jenkins/update.go b/internal/pkg/plugin/jenkins/update.go index 5b3f87692..401036e7d 100644 --- a/internal/pkg/plugin/jenkins/update.go +++ b/internal/pkg/plugin/jenkins/update.go @@ -20,7 +20,7 @@ func Update(options map[string]interface{}) (map[string]interface{}, error) { TermateOperations: []plugininstaller.BaseOperation{ helm.DealWithNsWhenInterruption, }, - GetStatusOperation: wrapperHelmResourceAndCustomResource(options, helm.GetPluginStaticStateByReleaseNameWrapper(defaultStatefulsetTplList)), + GetStatusOperation: wrapperHelmResourceAndCustomResource(helm.GetPluginStaticStateByReleaseNameWrapper(defaultStatefulsetTplList)), } // 2. execute installer get status and error From 221258f20bf4113f5388bef67b05ed7710818f33 Mon Sep 17 00:00:00 2001 From: Bird Date: Mon, 25 Jul 2022 11:30:25 +0800 Subject: [PATCH 5/5] chore(jenkins): update the way of rendering job xml Signed-off-by: Bird --- .../pkg/plugin/jenkinspipelinekubernetes/create.go | 11 ++++++++--- pkg/util/jenkins/client.go | 3 ++- pkg/util/jenkins/custom.go | 2 -- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/internal/pkg/plugin/jenkinspipelinekubernetes/create.go b/internal/pkg/plugin/jenkinspipelinekubernetes/create.go index e6b42c2d9..284971155 100644 --- a/internal/pkg/plugin/jenkinspipelinekubernetes/create.go +++ b/internal/pkg/plugin/jenkinspipelinekubernetes/create.go @@ -112,10 +112,15 @@ type JobXmlOptions struct { func renderJobXml(jobTemplate string, opts *JobXmlOptions) string { // TODO(aFlyBird0): use html/template to generate the job template. It's a good first issue. :) - jobXml := strings.Replace(jobTemplate, "{{.GitHubRepoURL}}", opts.GitHubRepoURL, 1) - jobXml = strings.Replace(jobXml, "{{.CredentialsID}}", opts.CredentialsID, 1) - jobXml = strings.Replace(jobXml, "{{.PipelineScriptPath}}", opts.PipelineScriptPath, 1) + replacerSlice := []string{ + "{{.GitHubRepoURL}}", opts.GitHubRepoURL, + "{{.CredentialsID}}", opts.CredentialsID, + "{{.PipelineScriptPath}}", opts.PipelineScriptPath, + } + + jobXml := strings.NewReplacer(replacerSlice...).Replace(jobTemplate) log.Debugf("job xml rendered: %s", jobXml) + return jobXml } diff --git a/pkg/util/jenkins/client.go b/pkg/util/jenkins/client.go index ef74bdb89..8e34ca2ab 100644 --- a/pkg/util/jenkins/client.go +++ b/pkg/util/jenkins/client.go @@ -41,9 +41,10 @@ func NewJenkins(jenkinsURL, username, password string) (*Jenkins, error) { // create gojenkins client gojenkinsClient := gojenkins.CreateJenkins(nil, jenkinsURL, username, password) ctx := context.Background() + // init and validate client if _, err := gojenkinsClient.Init(ctx); err != nil { - return nil, err + return nil, fmt.Errorf("failed to init jenkins client: %v", err) } return &Jenkins{ diff --git a/pkg/util/jenkins/custom.go b/pkg/util/jenkins/custom.go index 8ea8d7436..b8401ce7a 100644 --- a/pkg/util/jenkins/custom.go +++ b/pkg/util/jenkins/custom.go @@ -41,7 +41,6 @@ func (j *Jenkins) GetCrumb() (crumbHeaderKey, crumbHeaderValue string, cookie st log.Debugf("crumb: %+v", crumbResp) return crumbResp.CrumbRequestField, crumbResp.Crumb, resp.Header.Get("set-cookie"), nil - } // SetCrumb sets the jenkins crumb to the request @@ -57,5 +56,4 @@ func (j *Jenkins) SetCrumb(req *gorequest.SuperAgent) error { req.SetBasicAuth(j.BasicAuth.Username, j.BasicAuth.Password) return nil - }