Merge branch 'master' into feature/sort-records-by-date

cycles.yml

@@ -5,6 +5,14 @@ cycles:
       collection: methode
       throttle: 1m
       coolDown: 5m
+
+
+   -  name: wordpress-whole-archive
+      type: ThrottledWholeCollection
+      origin: wordpress
+      collection: wordpress
+      throttle: 1m
+      coolDown: 5m
 
   # -  name: methode-one-hour
   #    type: ScalingWindow

scheduler/scheduler.go

@@ -142,16 +142,46 @@ func (s *defaultScheduler) Start() error {
 	if s.IsRunning() {
 		return errors.New("Scheduler is already running")
 	}
-
 	s.setCurrentExecutionState(running)
 
+	startInterval := s.archiveCycleStartInterval()
+
 	for id, cycle := range s.cycles {
 		log.WithField("id", id).Info("Starting cycle.")
 		cycle.Start()
+		time.Sleep(startInterval)
+
 	}
 	return nil
 }
 
+func (s *defaultScheduler) archiveCycleStartInterval() time.Duration {
+
+	var temp time.Duration
+	var archiveCycles []*ThrottledWholeCollectionCycle
+
+	for _, cycle := range s.cycles {
+		if "ThrottledWholeCollection" == cycle.TransformToConfig().Type {
+			archiveCycles = append(archiveCycles, cycle.(*ThrottledWholeCollectionCycle))
+		}
+	}
+	numArchiveCycles := len(archiveCycles)
+
+	if numArchiveCycles > 1 {
+
+		minimumStartInterval := archiveCycles[0].throttle.Interval()
+
+		for _, cycle := range archiveCycles {
+			temp = cycle.throttle.Interval()
+			if temp < minimumStartInterval {
+				minimumStartInterval = temp
+			}
+		}
+		return minimumStartInterval / time.Duration(numArchiveCycles)
+	}
+	return temp
+}
+
 func (s *defaultScheduler) Shutdown() error {
 	s.cycleLock.RLock()
 	defer s.cycleLock.RUnlock()

scheduler/scheduler_test.go

@@ -28,6 +28,8 @@ func TestSchedulerShouldStartWhenEnabled(t *testing.T) {
 	c2.On("ID").Return("id2")
 	c1.On("Start").Return()
 	c2.On("Start").Return()
+	c1.On("TransformToConfig").Return(&CycleConfig{Type: "test"})
+	c2.On("TransformToConfig").Return(&CycleConfig{Type: "test"})
 
 	s.AddCycle(c1)
 	s.AddCycle(c2)
@@ -85,6 +87,8 @@ func TestSchedulerResumeAfterDisable(t *testing.T) {
 	c2.On("Start").Return()
 	c1.On("Stop").Return()
 	c2.On("Stop").Return()
+	c1.On("TransformToConfig").Return(&CycleConfig{Type: "test"})
+	c2.On("TransformToConfig").Return(&CycleConfig{Type: "test"})
 
 	s.AddCycle(c1)
 	s.AddCycle(c2)
@@ -162,6 +166,7 @@ func TestSaveCycleMetadata(t *testing.T) {
 	c1 := new(MockCycle)
 	c1.On("ID").Return(id1)
 	c1.On("Start").Return()
+	c1.On("TransformToConfig").Return(&CycleConfig{Type: "test"})
 
 	db := new(native.MockDB)
 	dbCollection := "testCollection"
@@ -183,3 +188,37 @@ func TestSaveCycleMetadata(t *testing.T) {
 
 	rw.AssertExpectations(t)
 }
+
+func TestCalculateArchiveCycleStartInterval(t *testing.T) {
+	assert := assert.New(t)
+	id1 := "id1"
+
+	c1 := new(MockCycle)
+	c1.On("ID").Return(id1)
+	c1.On("Start").Return()
+	c1.On("TransformToConfig").Return(&CycleConfig{Type: "test"})
+
+	db := new(native.MockDB)
+	dbCollection := "testCollection"
+	origin := "testOrigin"
+	coolDown := time.Minute
+	throttle, _ := NewThrottle(time.Second, 1)
+	throttle2, _ := NewThrottle(time.Second, 2)
+	c2 := NewThrottledWholeCollectionCycle("test", db, dbCollection, origin, coolDown, throttle, nil)
+	c3 := NewThrottledWholeCollectionCycle("test2", db, dbCollection, origin, coolDown, throttle2, nil)
+
+	rw := MockMetadataRW{}
+
+	s := NewScheduler(db, &tasks.MockTask{}, &rw, 1*time.Minute)
+
+	s.AddCycle(c1)
+	s.AddCycle(c2)
+	s.AddCycle(c3)
+
+	testIterval := s.(*defaultScheduler).archiveCycleStartInterval()
+
+	//we have 2 archive cycles out of 3, cycles and the shortest throttle is 1 sec
+	//therefore the startup interval is 500ms
+	expected, _ := time.ParseDuration("500ms")
+	assert.Equal(expected, testIterval, "test interval should be 500ms")
+}