diff --git a/.gitignore b/.gitignore index 89a197e..5874b9b 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,6 @@ credentials-sync # Output of the go coverage tool, specifically when used with LiteIDE *.out coverage.txt + +# JetBrains products +.idea \ No newline at end of file diff --git a/README.md b/README.md index b40bfe2..16fce66 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,16 @@ [![codecov](https://codecov.io/gh/coveooss/credentials-sync/branch/master/graph/badge.svg)](https://codecov.io/gh/coveooss/credentials-sync) [![Go Report Card](https://goreportcard.com/badge/github.com/coveooss/credentials-sync)](https://goreportcard.com/report/github.com/coveooss/credentials-sync) -Sync credentials from various sources to various targets. It currently only supports Jenkins, but LastPass is planned because that is what we use. However, we are open to supporting more targets. +Sync credentials from various sources to various targets. Source credentials are defined and stored in standardized +formats, and are converted to the target's format upon sync. + +The supported sources and targets are listed below. We are open to supporting more targets. What's the point? -1. Easier credentials rotations. Rotating credentials manually is simply not an option when credentials rotations are done too often -2. Uses a push-model instead of a pull-model which means that you can put your credentials in a secure environment to which targets don't have access, targets may have varying degrees of security (prod vs dev) -3. Decouples your credentials and the systems which use these credentials. Standardized credentials format for all targets +1. Easier credentials rotations. Rotating credentials manually is simply not an option when credentials rotations are done too often. +2. Uses a push-model instead of a pull-model which means that you can put your credentials in a secure environment to + which targets don't have access, targets may have varying degrees of security (prod vs dev). +3. Decouples your credentials and the systems which use these credentials. Standardized credentials format for all targets. ## Installation @@ -25,27 +29,30 @@ What's the point? credentials-sync sync -c config.yml ``` +Run without any argument for the full list of available commands. + ## Logging The log level can be set with either: - The `--log-level` option - The `SYNC_LOG_LEVEL` env variable -Valid levels are `debug`, `info`, `warning` and `error` +Valid levels are `debug`, `info`, `warning` and `error`. ![example](https://raw.githubusercontent.com/coveooss/credentials-sync/master/example.png) ## Configuration file A configuration file must be given to the application. Its path can either be a local path or a S3 path -The path can either be passed as a parameter (`-c/--config`) or as an environment variable (`SYNC_CONFIG`) +The path can either be passed as a parameter (`-c/--config`) or as an environment variable (`SYNC_CONFIG`). -A configuration file contains [sources](#supported-sources) which contain [credentials](#supported-types-of-credentials). It also defines targets to which these credentials will be synced +A configuration file contains [sources](#supported-sources) which contain [credentials](#supported-types-of-source-credentials). +It also defines targets to which these credentials will be synced. Here is the accepted format: ```yaml sources: local: - - path: /home/jdoe/path/to/file.yaml + - file: /home/jdoe/path/to/file.yaml aws_s3: - bucket: name key: path/to/file.yaml @@ -66,9 +73,10 @@ targets: ## Supported sources Here are the supported sources: - - Local (Single file) - - AWS S3 (Single object) - - AWS SecretsManager (Single secret or a secret prefix) + + - **local**: Local (Single file) + - **aws_s3**: AWS S3 (Single object) + - **aws_secretsmanager**: AWS SecretsManager (Single secret or a secret prefix) The source's value must either be a list or a map in the following formats (JSON or YAML): ```yaml @@ -91,9 +99,14 @@ my_other_cred: ... ``` -## Supported types of credentials -Credentials are defined as JSON, here are the supported types of credentials with definition examples: +## Supported types of source credentials +Credentials are defined as JSON or YAML, here are the supported types of source credentials with definition examples: - Secret text + - Username/Password + - AWS IAM + - SSH Key + - [Github App](https://developer.github.com/apps/about-apps/#about-github-apps) + ```yaml secret_text: description: A secret text cred is only composed of a secret @@ -135,6 +148,37 @@ ssh_key: -----END RSA PRIVATE KEY----- ``` + - github App credentials +```yaml +github_app: + description: + type: github_app + app_id: The github app ID. It can be found on github in the app's settings, on the General page in the About section. + private_key: | + The private key with which to authenticate to github. It must be in PKCS#8 format. + Github gives it in PKCS#1 format. Convert it to PKCS#8 with: + `openssl pkcs8 -topk8 -inform PEM -outform PEM -in current-key.pem -out new-key.pem -nocrypt` + owner: The organisation or user that this app is to be used for. Only required if this app is installed to multiple + organisations. +``` + +## Supported targets + +Here are the supported targets: + +- **jenkins**: Jenkins global credentials + +### Jenkins target + +The jenkins target supports the following configuration parameters: + +```yaml + jenkins: + - name: Name of this target + url: URL to the Jenkins server + credentials_id: The ID of the global credential to modify in Jenkins +``` + ## Other features ### Unsynced credentials diff --git a/credentials/credentials.go b/credentials/credentials.go index 1706ab1..53008e1 100644 --- a/credentials/credentials.go +++ b/credentials/credentials.go @@ -1,7 +1,6 @@ package credentials import ( - "errors" "fmt" "github.com/hashicorp/go-multierror" @@ -52,10 +51,10 @@ func (credBase *Base) BaseToString() string { // BaseValidate verifies that the credentials fields common to all types of credentials contain valid values func (credBase *Base) BaseValidate() error { if credBase.ID == "" { - return fmt.Errorf("Credentials (%s) has no defined ID", credBase.BaseToString()) + return fmt.Errorf("credentials (%s) has no defined ID", credBase.BaseToString()) } if credBase.CredType == "" { - return fmt.Errorf("Credentials (%s) has no type. This is probably a bug in the software", credBase.ID) + return fmt.Errorf("credentials (%s) has no type. This is probably a bug in the software", credBase.ID) } return nil } @@ -120,7 +119,7 @@ func (credBase *Base) ShouldSync(targetName string, targetTags map[string]string // ParseCredentials transforms a list of maps into a list of Credentials // The credentials type is determined by the `type` attribute func ParseCredentials(credentialsMaps []map[string]interface{}) ([]Credentials, error) { - credentialsList := []Credentials{} + credentialsList := make([]Credentials, 0) for _, credentialsMap := range credentialsMaps { newCredentials, err := ParseSingleCredentials(credentialsMap) if err != nil { @@ -136,12 +135,13 @@ func ParseCredentials(credentialsMaps []map[string]interface{}) ([]Credentials, func ParseSingleCredentials(credentialsMap map[string]interface{}) (Credentials, error) { var credentialsType string var credentials Credentials + var id = credentialsMap["id"] if value, ok := credentialsMap["type"]; ok { if credentialsType, ok = value.(string); !ok { - return nil, errors.New("Credentials type is not a string") + return nil, fmt.Errorf("entry %s: credentials type '%v' is not a string", id, credentialsType) } } else { - return nil, errors.New("Unable to find the credentials type") + return nil, fmt.Errorf("entry %s: unable to find the credentials type %s", id, credentialsType) } switch credentialsType { @@ -153,10 +153,15 @@ func ParseSingleCredentials(credentialsMap map[string]interface{}) (Credentials, credentials = NewSecretText() case "ssh": credentials = NewSSHCredentials() + case "github_app": + credentials = NewGithubAppCredentials() default: - return nil, errors.New("Unknown credentials type") + return nil, fmt.Errorf("entry %s: unknown credentials type: %s", id, credentialsType) + } + err := mapstructure.Decode(credentialsMap, credentials) + if err != nil { + return nil, fmt.Errorf("entry %s: invalid credentials data: %v", id, err) } - mapstructure.Decode(credentialsMap, credentials) var validationErrors error if err := credentials.BaseValidate(); err != nil { validationErrors = multierror.Append(validationErrors, err) @@ -165,7 +170,7 @@ func ParseSingleCredentials(credentialsMap map[string]interface{}) (Credentials, validationErrors = multierror.Append(validationErrors, err) } if validationErrors != nil { - return nil, fmt.Errorf("The following credentials failed to validate: %v -> %v", credentials.ToString(false), validationErrors) + return nil, fmt.Errorf("the following credentials failed to validate: %v -> %v", credentials.ToString(false), validationErrors) } return credentials, nil } diff --git a/credentials/credentials_github_app.go b/credentials/credentials_github_app.go new file mode 100644 index 0000000..2a271b7 --- /dev/null +++ b/credentials/credentials_github_app.go @@ -0,0 +1,47 @@ +package credentials + +import ( + "fmt" +) + +// GithubAppCredentials represents credentials composed of an App ID, private key, and owner +type GithubAppCredentials struct { + Base `mapstructure:",squash"` + AppID int `mapstructure:"app_id"` + PrivateKey string `mapstructure:"private_key"` + Owner string `mapstructure:"owner"` +} + +// NewGithubAppCredentials instantiates a GithubAppCredentials struct +func NewGithubAppCredentials() *GithubAppCredentials { + cred := &GithubAppCredentials{} + cred.CredType = "Github App" + return cred +} + +// ToString prints out the content of a GithubAppCredentials struct. +func (cred *GithubAppCredentials) ToString(showSensitive bool) string { + privateKeyText := "********" + if showSensitive { + privateKeyText = cred.PrivateKey + } + + appIDOwner := fmt.Sprintf("%d", cred.AppID) + if len(cred.Owner) > 0 { + appIDOwner = fmt.Sprintf("%s(%s)", appIDOwner, cred.Owner) + } + return fmt.Sprintf("%s - %s:%s", cred.BaseToString(), appIDOwner, privateKeyText) +} + +// Validate verifies that the credentials is valid. +// A GithubAppCredentials must have an app id and a private key. Owner is optional. +func (cred *GithubAppCredentials) Validate() error { + switch { + case cred.AppID == 0: + return fmt.Errorf("the credentials with ID %s does not define an app ID", cred.ID) + case len(cred.PrivateKey) == 0: + return fmt.Errorf("the credentials with ID %s does not define a private key", cred.ID) + default: + return nil + } +} diff --git a/credentials/credentials_github_app_test.go b/credentials/credentials_github_app_test.go new file mode 100644 index 0000000..75f701f --- /dev/null +++ b/credentials/credentials_github_app_test.go @@ -0,0 +1,70 @@ +package credentials + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGithubAppCredentials(t *testing.T) { + tests := map[string]struct { + givenID string + givenAppID int + givenOwner string + givenPrivate string + givenShowSensitive bool + expectString string + }{ + "without owner": {givenID: "test", givenAppID: 1, givenOwner: "", givenPrivate: "private", givenShowSensitive: false, expectString: "test -> Type: Github App - 1:********"}, + "with owner": {givenID: "test", givenAppID: 2, givenOwner: "owner", givenPrivate: "private", givenShowSensitive: false, expectString: "test -> Type: Github App - 2(owner):********"}, + "without owner showSensitive": {givenID: "test", givenAppID: 1, givenOwner: "", givenPrivate: "private", givenShowSensitive: true, expectString: "test -> Type: Github App - 1:private"}, + "with owner showSensitive": {givenID: "test", givenAppID: 2, givenOwner: "owner", givenPrivate: "private", givenShowSensitive: true, expectString: "test -> Type: Github App - 2(owner):private"}, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + cred := NewGithubAppCredentials() + cred.ID = test.givenID + cred.AppID = test.givenAppID + cred.Owner = test.givenOwner + cred.PrivateKey = test.givenPrivate + assert.Equal(t, test.expectString, cred.ToString(test.givenShowSensitive)) + }) + } +} + +func TestGithubAppCredentialsValidation(t *testing.T) { + tests := map[string]struct { + givenCred GithubAppCredentials + expectError bool + }{ + "valid": {givenCred: GithubAppCredentials{ + AppID: 12345, + PrivateKey: "private", + Owner: "Me", + }, expectError: false}, + "valid no owner": {givenCred: GithubAppCredentials{ + AppID: 12345, + PrivateKey: "private", + }, expectError: false}, + "missing app id": {givenCred: GithubAppCredentials{ + PrivateKey: "private", + }, expectError: true}, + "missing private key": {givenCred: GithubAppCredentials{ + AppID: 12345, + }, expectError: true}, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + cred := NewGithubAppCredentials() + cred.AppID = test.givenCred.AppID + cred.PrivateKey = test.givenCred.PrivateKey + cred.Owner = test.givenCred.Owner + err := cred.Validate() + if test.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/credentials/credentials_test.go b/credentials/credentials_test.go index 0860c05..9524961 100644 --- a/credentials/credentials_test.go +++ b/credentials/credentials_test.go @@ -196,26 +196,24 @@ func TestBaseValidateCredentials(t *testing.T) { credWithoutType := &Base{ ID: "test", } - assert.EqualError(t, credWithoutType.BaseValidate(), "Credentials (test) has no type. This is probably a bug in the software") + assert.EqualError(t, credWithoutType.BaseValidate(), "credentials (test) has no type. This is probably a bug in the software") credWithoutID := &Base{ CredType: "test", Description: "test2", } - assert.EqualError(t, credWithoutID.BaseValidate(), "Credentials ( -> Type: test, Description: test2) has no defined ID") + assert.EqualError(t, credWithoutID.BaseValidate(), "credentials ( -> Type: test, Description: test2) has no defined ID") } func TestParseCredentials(t *testing.T) { t.Parallel() - cases := []struct { - name string + cases := map[string]struct { credMaps []map[string]interface{} result []Credentials wantErr bool }{ - { - name: "Invalid type", + "Invalid type": { credMaps: []map[string]interface{}{ { "id": "stuff", @@ -227,8 +225,7 @@ func TestParseCredentials(t *testing.T) { result: nil, wantErr: true, }, - { - name: "Invalid type (not a string)", + "Invalid type (not a string)": { credMaps: []map[string]interface{}{ { "id": "stuff", @@ -240,16 +237,123 @@ func TestParseCredentials(t *testing.T) { result: nil, wantErr: true, }, + "Valid aws": { + credMaps: []map[string]interface{}{ + { + "id": "stuff", + "type": "aws", + "access_key": "AKIAMYFAKEKEY", + "secret_key": "fdjVEsefk4kgjVsdjfew54", + "description": "test-desc", + }, + }, + result: []Credentials{&AmazonWebServicesCredentials{ + Base: Base{ + ID: "stuff", + Description: "test-desc", + CredType: "Amazon Web Services", + }, + AccessKey: "AKIAMYFAKEKEY", + SecretKey: "fdjVEsefk4kgjVsdjfew54", + }}, + wantErr: false, + }, + "Valid usernamepassword": { + credMaps: []map[string]interface{}{ + { + "id": "stuff", + "type": "usernamepassword", + "username": "username", + "password": "password", + "description": "test-desc", + }, + }, + result: []Credentials{&UsernamePasswordCredentials{ + Base: Base{ + ID: "stuff", + Description: "test-desc", + CredType: "Username/Password", + }, + Username: "username", + Password: "password", + }}, + wantErr: false, + }, + "Valid secret": { + credMaps: []map[string]interface{}{ + { + "id": "stuff", + "type": "secret", + "secret": "secret", + "description": "test-desc", + }, + }, + result: []Credentials{&SecretTextCredentials{ + Base: Base{ + ID: "stuff", + Description: "test-desc", + CredType: "Secret text", + }, + Secret: "secret", + }}, + wantErr: false, + }, + "Valid ssh": { + credMaps: []map[string]interface{}{ + { + "id": "stuff", + "type": "ssh", + "username": "user", + "passphrase": "pass", + "private_key": "private", + "description": "test-desc", + }, + }, + result: []Credentials{&SSHCredentials{ + Base: Base{ + ID: "stuff", + Description: "test-desc", + CredType: "SSH", + }, + Username: "user", + Passphrase: "pass", + PrivateKey: "private", + }}, + wantErr: false, + }, + "Valid github app": { + credMaps: []map[string]interface{}{ + { + "id": "stuff", + "type": "github_app", + "app_id": 12345, + "private_key": "private", + "owner": "owner", + "description": "test-desc", + }, + }, + result: []Credentials{&GithubAppCredentials{ + Base: Base{ + ID: "stuff", + Description: "test-desc", + CredType: "Github App", + }, + AppID: 12345, + PrivateKey: "private", + Owner: "owner", + }}, + wantErr: false, + }, } - for _, tt := range cases { - t.Run(tt.name, func(t *testing.T) { + for name, tt := range cases { + t.Run(name, func(t *testing.T) { gottenCreds, err := ParseCredentials(tt.credMaps) if tt.wantErr { assert.Error(t, err) } else { - assert.Nil(t, err) + assert.NoError(t, err) } assert.Equal(t, tt.result, gottenCreds) }) diff --git a/go.mod b/go.mod index 1fcfb33..b0ef4a2 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.12 require ( github.com/aws/aws-sdk-go v1.30.7 - github.com/bndr/gojenkins v0.0.0-00010101000000-000000000000 + github.com/bndr/gojenkins v1.0.1 github.com/golang/mock v1.4.3 github.com/hashicorp/go-multierror v1.1.0 github.com/mitchellh/mapstructure v1.2.2 diff --git a/targets/jenkins.go b/targets/jenkins.go index acbd6b1..fd1a2bd 100644 --- a/targets/jenkins.go +++ b/targets/jenkins.go @@ -1,6 +1,7 @@ package targets import ( + "encoding/xml" "errors" "fmt" "net/url" @@ -41,9 +42,9 @@ func (jenkins *JenkinsTarget) Initialize(allCredentials []credentials.Credential } } }() - for _, credentials := range allCredentials { - if jenkins.CredentialsID != nil && credentials.GetID() == *jenkins.CredentialsID { - jenkins.loginCredentials = credentials + for _, creds := range allCredentials { + if jenkins.CredentialsID != nil && creds.GetID() == *jenkins.CredentialsID { + jenkins.loginCredentials = creds } } options := &gojenkins.JenkinsOptions{SslVerify: aws.Bool(!jenkins.InsecureConnection)} @@ -54,7 +55,12 @@ func (jenkins *JenkinsTarget) Initialize(allCredentials []credentials.Credential } jenkins.client = gojenkins.CreateJenkinsWithOptions(jenkins.URL, options) - jenkins.client.Init() + _, err = jenkins.client.Init() + + if err != nil { + return err + } + jenkins.credentialsManager = &gojenkins.CredentialsManager{ J: jenkins.client, } @@ -83,7 +89,7 @@ func (jenkins *JenkinsTarget) DeleteCredentials(id string) error { func (jenkins *JenkinsTarget) UpdateCredentials(cred credentials.Credentials) error { jenkinsCred := toJenkinsCredential(cred) if jenkinsCred == nil { - return fmt.Errorf("Unable to create jenkins credentials from %s", cred.GetID()) + return fmt.Errorf("unable to create jenkins credentials from %s", cred.GetID()) } if HasCredential(jenkins, cred.GetTargetID()) { return jenkins.credentialsManager.Update(credentialsDomain, cred.GetTargetID(), jenkinsCred) @@ -94,15 +100,36 @@ func (jenkins *JenkinsTarget) UpdateCredentials(cred credentials.Credentials) er // ValidateConfiguration verifies that Jenkins configuration is valid func (jenkins *JenkinsTarget) ValidateConfiguration() error { if _, err := url.ParseRequestURI(jenkins.URL); err != nil { - return fmt.Errorf("The Jenkins target `%s` has an invalid URL: %s", jenkins.Name, jenkins.URL) + return fmt.Errorf("the Jenkins target `%s` has an invalid URL: %s", jenkins.Name, jenkins.URL) } return nil } +// JenkinsGithubAppCredentials is the Jenkins Github plugin's credentials configuration. +/* + It must be serializable to the following XML: + + github-app-dev + The GitHub app for Jenkins + 73157 + {some_private_key} + https://api.github.com + coveo + +*/ +type JenkinsGithubAppCredentials struct { + XMLName xml.Name `xml:"org.jenkinsci.plugins.github__branch__source.GitHubAppCredentials"` + ID string `xml:"id"` + Description string `xml:"description,omitempty"` + AppID int `xml:"appID"` + PrivateKey string `xml:"privateKey"` + APIURI string `xml:"apiUri,omitempty"` + Owner string `xml:"owner,omitempty"` +} + func toJenkinsCredential(creds credentials.Credentials) interface{} { - switch creds.(type) { + switch castCreds := creds.(type) { case *credentials.AmazonWebServicesCredentials: - castCreds := creds.(*credentials.AmazonWebServicesCredentials) return &gojenkins.AmazonWebServicesCredentials{ ID: creds.GetTargetID(), Description: castCreds.GetDescriptionOrID(), @@ -112,14 +139,12 @@ func toJenkinsCredential(creds credentials.Credentials) interface{} { IAMMFASerialNumber: castCreds.MFASerialNumber, } case *credentials.SecretTextCredentials: - castCreds := creds.(*credentials.SecretTextCredentials) return &gojenkins.StringCredentials{ ID: creds.GetTargetID(), Description: castCreds.GetDescriptionOrID(), Secret: castCreds.Secret, } case *credentials.UsernamePasswordCredentials: - castCreds := creds.(*credentials.UsernamePasswordCredentials) return &gojenkins.UsernameCredentials{ ID: castCreds.GetTargetID(), Description: castCreds.GetDescriptionOrID(), @@ -127,7 +152,6 @@ func toJenkinsCredential(creds credentials.Credentials) interface{} { Password: castCreds.Password, } case *credentials.SSHCredentials: - castCreds := creds.(*credentials.SSHCredentials) return &gojenkins.SSHCredentials{ ID: castCreds.GetTargetID(), Description: castCreds.GetDescriptionOrID(), @@ -138,6 +162,15 @@ func toJenkinsCredential(creds credentials.Credentials) interface{} { Value: castCreds.PrivateKey, }, } + case *credentials.GithubAppCredentials: + return &JenkinsGithubAppCredentials{ + ID: castCreds.GetTargetID(), + Description: castCreds.GetDescriptionOrID(), + AppID: castCreds.AppID, + PrivateKey: castCreds.PrivateKey, + APIURI: "https://api.github.com", + Owner: castCreds.Owner, + } } return nil } diff --git a/targets/jenkins_test.go b/targets/jenkins_test.go index 0803108..ddfa08f 100644 --- a/targets/jenkins_test.go +++ b/targets/jenkins_test.go @@ -83,3 +83,21 @@ func TestUsernamePasswordToJenkinsCred(t *testing.T) { assert.Equal(t, secret.Username, jenkinsSecret.Username) assert.Equal(t, secret.Password, jenkinsSecret.Password) } + +func TestGithubAppToJenkinsCred(t *testing.T) { + secret := credentials.NewGithubAppCredentials() + secret.ID = "test-id" + secret.Description = "a test description" + secret.Owner = "a-user" + secret.PrivateKey = "a-key" + secret.AppID = 12345 + + jenkinsSecretInterface := toJenkinsCredential(secret) + + jenkinsSecret := jenkinsSecretInterface.(*JenkinsGithubAppCredentials) + assert.Equal(t, secret.ID, jenkinsSecret.ID) + assert.Equal(t, secret.Description, jenkinsSecret.Description) + assert.Equal(t, secret.Owner, jenkinsSecret.Owner) + assert.Equal(t, secret.PrivateKey, jenkinsSecret.PrivateKey) + assert.Equal(t, secret.AppID, jenkinsSecret.AppID) +}