Merge pull request #641 from ystia/bugfix/GH-640-data-migration-concu…

…rrency-limit

Add concurrency limit during data migration for file storage

CHANGELOG.md

@@ -9,6 +9,7 @@
 ### BUG FIXES
 
 * Bootstrap may failed with a nil pointer error if download of a component fails ([GH-634](https://github.com/ystia/yorc/issues/634))
+* Missing concurrency limit during data migration for logs and events file storage ([GH-640](https://github.com/ystia/yorc/issues/640))
 
 ## 4.0.0 (April 17, 2020)
 

storage/internal/file/store.go

@@ -55,6 +55,7 @@ type fileStore struct {
 	cache             *ristretto.Cache
 	withEncryption    bool
 	encryptor         *encryption.Encryptor
+	concurrencyLimit  int
 }
 
 // NewStore returns a new File store
@@ -75,6 +76,7 @@ func NewStore(cfg config.Configuration, storeID string, properties config.Dynami
 		fileLocks:         make(map[string]*sync.RWMutex),
 		withCache:         withCache,
 		withEncryption:    withEncryption,
+		concurrencyLimit:  cfg.UpgradeConcurrencyLimit,
 	}
 
 	// Instantiate cache if necessary
@@ -208,9 +210,14 @@ func (s *fileStore) SetCollection(ctx context.Context, keyValues []store.KeyValu
 		return nil
 	}
 	errGroup, ctx := errgroup.WithContext(ctx)
+	sem := make(chan struct{}, s.concurrencyLimit)
 	for _, kv := range keyValues {
+		sem <- struct{}{}
 		kvItem := kv
 		errGroup.Go(func() error {
+			defer func() {
+				<-sem
+			}()
 			return s.Set(ctx, kvItem.Key, kvItem.Value)
 		})
 	}

storage/store/test.go

@@ -71,7 +71,8 @@ func SetupTestConfig(t testing.TB) config.Configuration {
 	assert.Nil(t, err)
 
 	return config.Configuration{
-		WorkingDirectory: workingDir,
+		WorkingDirectory:        workingDir,
+		UpgradeConcurrencyLimit: config.DefaultUpgradesConcurrencyLimit,
 	}
 }
 

testutil/helper.go

@@ -99,6 +99,7 @@ func SetupTestConfig(t testing.TB) config.Configuration {
 	assert.Nil(t, err)
 
 	return config.Configuration{
-		WorkingDirectory: workingDir,
+		WorkingDirectory:        workingDir,
+		UpgradeConcurrencyLimit: config.DefaultUpgradesConcurrencyLimit,
 	}
 }