Skip to content

Commit

Permalink
Basic synchronization tests
Browse files Browse the repository at this point in the history
  • Loading branch information
Julien Duchesne committed Aug 6, 2019
1 parent 1b21474 commit d5defe1
Show file tree
Hide file tree
Showing 8 changed files with 175 additions and 57 deletions.
4 changes: 3 additions & 1 deletion cli/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ var syncCmd = &cobra.Command{
if err := configuration.Targets.ValidateConfiguration(); err != nil {
log.Fatalf("The targets section of the config file is invalid: %v", err)
}
configuration.Sync()
if err := configuration.Sync(); err != nil {
log.Fatalf("The synchronization process failed: %v", err)
}
},
}
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/bndr/gojenkins v0.0.0-00010101000000-000000000000
github.com/golang/mock v1.3.1
github.com/hashicorp/go-multierror v1.0.0
github.com/jinzhu/copier v0.0.0-20190625015134-976e0346caa8
github.com/mitchellh/mapstructure v1.1.2
github.com/sirupsen/logrus v1.4.2
github.com/spf13/cobra v0.0.5
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jinzhu/copier v0.0.0-20190625015134-976e0346caa8 h1:mGIXW/lubQ4B+3bXTLxcTMTjUNDqoF6T/HUW9LbFx9s=
github.com/jinzhu/copier v0.0.0-20190625015134-976e0346caa8/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
Expand Down
71 changes: 47 additions & 24 deletions sync/config.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package sync

import (
"fmt"

"github.com/coveooss/credentials-sync/credentials"
"github.com/coveooss/credentials-sync/targets"
log "github.com/sirupsen/logrus"
Expand Down Expand Up @@ -34,59 +36,76 @@ func (config *Configuration) SetTargets(targets targets.TargetCollection) {
}

// Sync syncs credentials from the configured sources to the configured targets
func (config *Configuration) Sync() {
func (config *Configuration) Sync() error {
// Start reading credentials
creds, err := config.Sources.Credentials()
if err != nil {
log.Fatalf("Caught an error while fetching credentials: %v", err)
return fmt.Errorf("Caught an error while fetching credentials: %v", err)
}

// Initialize targets
validTargets := []targets.Target{}
allTargets := config.Targets.AllTargets()
initChannel := make(chan targets.Target)
initChannel := make(chan interface{})
for _, target := range allTargets {
go config.initTarget(target, creds, initChannel)
}
for i := 0; i < len(allTargets); i++ {
initTarget := <-initChannel
if initTarget != nil {
validTargets = append(validTargets, initTarget)
if err, ok := initTarget.(error); ok {
if config.StopOnError {
return err
}
log.Error(err)
} else {
validTargets = append(validTargets, initTarget.(targets.Target))
}
}

syncChannel := make(chan bool, config.TargetParallelism)
// Sync credentials with as many targets as the config allows
parallelismChannel := make(chan bool, config.TargetParallelism)
errorChannel := make(chan error)
for _, target := range validTargets {
syncChannel <- true
go config.syncCredentials(target, creds, syncChannel)
parallelismChannel <- true
go config.syncCredentials(target, creds, parallelismChannel, errorChannel)

// Check for errors. Errors are only passed back if StopOnError is true so this should always return
err := <-errorChannel
if err != nil {
return err
}
}

for i := 0; i < cap(syncChannel); i++ {
syncChannel <- true
// Ensure that the sync method is completely done for all targets
for i := 0; i < cap(parallelismChannel); i++ {
parallelismChannel <- true
}

return nil
}

func (config *Configuration) logError(format string, args ...interface{}) {
if config.StopOnError {
log.Fatalf(format, args...)
}
log.Errorf(format, args...)
}
func (config *Configuration) initTarget(target targets.Target, creds []credentials.Credentials, channel chan interface{}) {
var channelValue interface{}

defer func() {
channel <- channelValue
}()

func (config *Configuration) initTarget(target targets.Target, creds []credentials.Credentials, channel chan targets.Target) {
err := target.Initialize(creds)
if err == nil {
log.Infof("Connected to %s", target.ToString())
channel <- target
channelValue = target
} else {
config.logError("Target `%s` has failed initialization: %v", target.GetName(), err)
channel <- nil
channelValue = fmt.Errorf("Target `%s` has failed initialization: %v", target.GetName(), err)
}
}

func (config *Configuration) syncCredentials(target targets.Target, credentialsList []credentials.Credentials, channel chan bool) {
defer func() { <-channel }()
func (config *Configuration) syncCredentials(target targets.Target, credentialsList []credentials.Credentials, parallelismChannel chan bool, errorChannel chan error) {
var err error
defer func() {
errorChannel <- err
<-parallelismChannel
}()

filteredCredentials := []credentials.Credentials{}
for _, cred := range credentialsList {
Expand All @@ -95,8 +114,12 @@ func (config *Configuration) syncCredentials(target targets.Target, credentialsL
}
}

config.UpdateListOfCredentials(target, filteredCredentials)
config.DeleteListOfCredentials(target)
if err = config.UpdateListOfCredentials(target, filteredCredentials); err != nil {
return
}
if err = config.DeleteListOfCredentials(target); err != nil {
return
}

log.Infof("Finished sync to %s", target.GetName())
}
46 changes: 46 additions & 0 deletions sync/config_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package sync

import (
"testing"

"github.com/coveooss/credentials-sync/credentials"
)

func TestSyncCredentials(t *testing.T) {
cred1, cred2 := credentials.NewSecretText(), credentials.NewSecretText()
cred1.ID = "test1"
cred2.ID = "test2"

config := &Configuration{StopOnError: true, TargetParallelism: 1}
targetController, target := setTargetMock(t, config, "target", []string{"test1"}, false)
sourceController, _ := setSourceMock(t, config, []credentials.Credentials{cred1, cred2})
defer targetController.Finish()
defer sourceController.Finish()

// Asserts that UpdateCredentials is called with `test1` and `test2`
target.EXPECT().UpdateCredentials(cred1).Times(1)
target.EXPECT().UpdateCredentials(cred2).Times(1)

config.Sync()
}

func TestSyncCredentialsAndDeleteUnsynced(t *testing.T) {
cred1, cred2 := credentials.NewSecretText(), credentials.NewSecretText()
cred1.ID = "test1"
cred2.ID = "test2"

config := &Configuration{StopOnError: true, TargetParallelism: 1}
targetController, target := setTargetMock(t, config, "target", []string{"test3"}, true)
sourceController, _ := setSourceMock(t, config, []credentials.Credentials{cred1, cred2})
defer targetController.Finish()
defer sourceController.Finish()

// Asserts that UpdateCredentials is called with `test1` and `test2`
target.EXPECT().UpdateCredentials(cred1).Times(1)
target.EXPECT().UpdateCredentials(cred2).Times(1)

// Asserts that DeleteCredentials is called with `unsynced`
target.EXPECT().DeleteCredentials("test3").Times(1)

config.Sync()
}
28 changes: 23 additions & 5 deletions sync/target.go
Original file line number Diff line number Diff line change
@@ -1,25 +1,33 @@
package sync

import (
"fmt"

"github.com/coveooss/credentials-sync/credentials"
"github.com/coveooss/credentials-sync/targets"
log "github.com/sirupsen/logrus"
)

// DeleteListOfCredentials deletes the configured list of credentials from the given target
func (config *Configuration) DeleteListOfCredentials(target targets.Target) {
func (config *Configuration) DeleteListOfCredentials(target targets.Target) error {
for _, id := range config.CredentialsToDelete {
if targets.HasCredential(target, id) {
log.Infof("[%s] Deleting %s", target.GetName(), id)
if err := target.DeleteCredentials(id); err != nil {
config.logError("Failed to delete credential %s from %s: %v", id, target.GetName(), err)
err = fmt.Errorf("Failed to delete credential %s from %s: %v", id, target.GetName(), err)
if config.StopOnError {
return err
}
log.Error(err)
}
}
}

return nil
}

// UpdateListOfCredentials syncs the given list of credentials to the given target
func (config *Configuration) UpdateListOfCredentials(target targets.Target, listOfCredentials []credentials.Credentials) {
func (config *Configuration) UpdateListOfCredentials(target targets.Target, listOfCredentials []credentials.Credentials) error {
isSynced := func(id string) bool {
for _, credentials := range listOfCredentials {
if credentials.GetID() == id {
Expand All @@ -32,7 +40,11 @@ func (config *Configuration) UpdateListOfCredentials(target targets.Target, list
for _, credentials := range listOfCredentials {
log.Infof("[%s] Syncing %s", target.GetName(), credentials.GetID())
if err := target.UpdateCredentials(credentials); err != nil {
config.logError("Failed to send credentials with ID %s to %s: %v", credentials.GetID(), target.GetName(), err)
err = fmt.Errorf("Failed to send credentials with ID %s to %s: %v", credentials.GetID(), target.GetName(), err)
if config.StopOnError {
return err
}
log.Error(err)
}
}

Expand All @@ -42,9 +54,15 @@ func (config *Configuration) UpdateListOfCredentials(target targets.Target, list
if !isSynced(existingID) {
log.Infof("[%s] Deleting %s", target.GetName(), existingID)
if err := target.DeleteCredentials(existingID); err != nil {
config.logError("Failed to delete credentials with ID %s to %s: %v", existingID, target.GetName(), err)
err = fmt.Errorf("Failed to delete credentials with ID %s to %s: %v", existingID, target.GetName(), err)
if config.StopOnError {
return err
}
log.Error(err)
}
}
}
}

return nil
}
29 changes: 2 additions & 27 deletions sync/target_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,34 +4,9 @@ import (
"testing"

"github.com/coveooss/credentials-sync/credentials"
"github.com/coveooss/credentials-sync/targets"
"github.com/golang/mock/gomock"
)

func setSourceMock(t *testing.T, config *Configuration) (*gomock.Controller, *credentials.MockSource) {
ctrl := gomock.NewController(t)
source := credentials.NewMockSource(ctrl)
sourceCollection := credentials.NewMockSourceCollection(ctrl)
sourceCollection.EXPECT().AllSources().Return([]credentials.Source{source}).AnyTimes()

config.SetSources(sourceCollection)
return ctrl, source
}

func setTargetMock(t *testing.T, config *Configuration, name string, credentials []string, shouldDeleteUnsynced bool) (*gomock.Controller, *targets.MockTarget) {
ctrl := gomock.NewController(t)
target := targets.NewMockTarget(ctrl)
targetCollection := targets.NewMockTargetCollection(ctrl)
targetCollection.EXPECT().AllTargets().Return([]targets.Target{target}).AnyTimes()

target.EXPECT().GetExistingCredentials().Return(credentials).AnyTimes()
target.EXPECT().GetName().Return(name).AnyTimes()
target.EXPECT().ShouldDeleteUnsynced().Return(shouldDeleteUnsynced).AnyTimes()

config.SetTargets(targetCollection)
return ctrl, target
}

func TestDeleteListOfCredentials(t *testing.T) {
config := &Configuration{
CredentialsToDelete: []string{"test1", "test-not-exist"},
Expand All @@ -46,7 +21,7 @@ func TestDeleteListOfCredentials(t *testing.T) {
}

func TestUpdateListOfCredentials(t *testing.T) {
config := &Configuration{}
config := NewConfiguration()
targetController, target := setTargetMock(t, config, "", []string{"test1"}, false)
defer targetController.Finish()

Expand All @@ -62,7 +37,7 @@ func TestUpdateListOfCredentials(t *testing.T) {
}

func TestDeleteUnsyncedCredentials(t *testing.T) {
config := &Configuration{}
config := NewConfiguration()
targetController, target := setTargetMock(t, config, "", []string{"unsynced"}, true)
defer targetController.Finish()

Expand Down
51 changes: 51 additions & 0 deletions sync/test_helpers_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package sync

import (
"testing"

"github.com/coveooss/credentials-sync/credentials"
"github.com/coveooss/credentials-sync/targets"
"github.com/golang/mock/gomock"
)

func setSourceMock(t *testing.T, config *Configuration, creds []credentials.Credentials) (*gomock.Controller, *credentials.MockSourceCollection) {
ctrl := gomock.NewController(t)
source := credentials.NewMockSource(ctrl)
sourceCollection := credentials.NewMockSourceCollection(ctrl)
sourceCollection.EXPECT().AllSources().Return([]credentials.Source{source}).AnyTimes()

if creds != nil {
sourceCollection.EXPECT().Credentials().Return(creds, nil).AnyTimes()
}

config.SetSources(sourceCollection)
return ctrl, sourceCollection
}

func setMultipleTargetMock(t *testing.T, config *Configuration, name string, credentials []string, shouldDeleteUnsynced bool, targetNumber int) (*gomock.Controller, []*targets.MockTarget) {
ctrl := gomock.NewController(t)
targetCollection := targets.NewMockTargetCollection(ctrl)

targetsToReturn := []*targets.MockTarget{}
targetsToReturnInterface := []targets.Target{}
for i := 1; i <= targetNumber; i++ {
target := targets.NewMockTarget(ctrl)
target.EXPECT().GetExistingCredentials().Return(credentials).AnyTimes()
target.EXPECT().GetName().Return(name).AnyTimes()
target.EXPECT().GetTags().Return(map[string]string{}).AnyTimes()
target.EXPECT().ToString().Return(name).AnyTimes()
target.EXPECT().ShouldDeleteUnsynced().Return(shouldDeleteUnsynced).AnyTimes()
target.EXPECT().Initialize(gomock.Any()).AnyTimes()
targetsToReturn = append(targetsToReturn, target)
targetsToReturnInterface = append(targetsToReturnInterface, target)
}
targetCollection.EXPECT().AllTargets().Return(targetsToReturnInterface).AnyTimes()

config.SetTargets(targetCollection)
return ctrl, targetsToReturn
}

func setTargetMock(t *testing.T, config *Configuration, name string, credentials []string, shouldDeleteUnsynced bool) (*gomock.Controller, *targets.MockTarget) {
ctrl, targetsToReturn := setMultipleTargetMock(t, config, name, credentials, shouldDeleteUnsynced, 1)
return ctrl, targetsToReturn[0]
}

0 comments on commit d5defe1

Please sign in to comment.