Added parameters to control retention job

1. day of the week the retention job starts
2. if the retention job starts at the same hour each day

.settings/org.eclipse.jdt.ui.prefs

@@ -1,4 +1,4 @@
-#Tue Jan 13 14:27:58 PST 2009
+#Sat Sep 22 05:05:45 PDT 2012
 cleanup.add_default_serial_version_id=true
 cleanup.add_generated_serial_version_id=false
 cleanup.add_missing_annotations=true

src/java/voldemort/common/service/SchedulerService.java

@@ -164,10 +164,25 @@ public void schedule(String id, Runnable runnable, Date timeToRun) {
     }
 
     public void schedule(String id, Runnable runnable, Date nextRun, long periodMs) {
-        ScheduledFuture<?> future = scheduler.scheduleWithFixedDelay(runnable,
-                                                                     delayMs(nextRun),
-                                                                     periodMs,
-                                                                     TimeUnit.MILLISECONDS);
+        schedule(id, runnable, nextRun, periodMs, false);
+    }
+
+    public void schedule(String id,
+                         Runnable runnable,
+                         Date nextRun,
+                         long periodMs,
+                         boolean scheduleAtFixedRate) {
+        ScheduledFuture<?> future = null;
+        if(scheduleAtFixedRate)
+            future = scheduler.scheduleAtFixedRate(runnable,
+                                                   delayMs(nextRun),
+                                                   periodMs,
+                                                   TimeUnit.MILLISECONDS);
+        else
+            future = scheduler.scheduleWithFixedDelay(runnable,
+                                                      delayMs(nextRun),
+                                                      periodMs,
+                                                      TimeUnit.MILLISECONDS);
         if(!allJobs.containsKey(id)) {
             allJobs.put(id, new ScheduledRunnable(runnable, nextRun, periodMs));
         }

src/java/voldemort/server/VoldemortConfig.java

@@ -192,6 +192,8 @@ public class VoldemortConfig implements Serializable {
 
     private int retentionCleanupFirstStartTimeInHour;
     private int retentionCleanupScheduledPeriodInHour;
+    private int retentionCleanupFirstStartDayOfWeek;
+    private boolean retentionCleanupPinStartTime;
 
     private int maxRebalancingAttempt;
     private long rebalancingTimeoutSec;
@@ -395,9 +397,16 @@ public VoldemortConfig(Props props) {
         // start at midnight (0-23)
         this.retentionCleanupFirstStartTimeInHour = props.getInt("retention.cleanup.first.start.hour",
                                                                  0);
+        // start next day by default (1=SUN, 2=MON, 3=TUE, 4=WED, 5=THU, 6=FRI,
+        // 7=SAT)
+        this.retentionCleanupFirstStartDayOfWeek = props.getInt("retention.cleanup.first.start.day",
+                                                                Utils.getDayOfTheWeekFromNow(1));
         // repeat every 24 hours
         this.retentionCleanupScheduledPeriodInHour = props.getInt("retention.cleanup.period.hours",
                                                                   24);
+        // should the retention job always start at the 'start time' specified
+        this.retentionCleanupPinStartTime = props.getBoolean("retention.cleanup.pin.start.time",
+                                                             true);
 
         // save props for access from plugins
         this.allProps = props;
@@ -1723,6 +1732,14 @@ public void setRetentionCleanupFirstStartTimeInHour(int retentionCleanupFirstSta
         this.retentionCleanupFirstStartTimeInHour = retentionCleanupFirstStartTimeInHour;
     }
 
+    public int getRetentionCleanupFirstStartDayOfWeek() {
+        return retentionCleanupFirstStartDayOfWeek;
+    }
+
+    public void setRetentionCleanupFirstStartDayOfWeek(int retentionCleanupFirstStartDayOfWeek) {
+        this.retentionCleanupFirstStartDayOfWeek = retentionCleanupFirstStartDayOfWeek;
+    }
+
     public int getRetentionCleanupScheduledPeriodInHour() {
         return retentionCleanupScheduledPeriodInHour;
     }
@@ -1731,6 +1748,14 @@ public void setRetentionCleanupScheduledPeriodInHour(int retentionCleanupSchedul
         this.retentionCleanupScheduledPeriodInHour = retentionCleanupScheduledPeriodInHour;
     }
 
+    public boolean getRetentionCleanupPinStartTime() {
+        return retentionCleanupPinStartTime;
+    }
+
+    public void setRetentionCleanupPinStartTime(boolean retentionCleanupFixStartTime) {
+        this.retentionCleanupPinStartTime = retentionCleanupFixStartTime;
+    }
+
     public int getAdminSocketTimeout() {
         return adminSocketTimeout;
     }

src/java/voldemort/server/storage/StorageService.java

@@ -102,6 +102,7 @@
 import voldemort.utils.ReflectUtils;
 import voldemort.utils.SystemTime;
 import voldemort.utils.Time;
+import voldemort.utils.Utils;
 import voldemort.versioning.VectorClock;
 import voldemort.versioning.VectorClockInconsistencyResolver;
 import voldemort.versioning.Versioned;
@@ -857,13 +858,10 @@ private Store<ByteArray, byte[], byte[]> createNodeStore(String storeName, Node
      */
     private void scheduleCleanupJob(StoreDefinition storeDef,
                                     StorageEngine<ByteArray, byte[], byte[]> engine) {
-        // Schedule data retention cleanup job starting next day.
-        GregorianCalendar cal = new GregorianCalendar();
-        cal.add(Calendar.DAY_OF_YEAR, 1);
-        cal.set(Calendar.HOUR_OF_DAY, voldemortConfig.getRetentionCleanupFirstStartTimeInHour());
-        cal.set(Calendar.MINUTE, 0);
-        cal.set(Calendar.SECOND, 0);
-        cal.set(Calendar.MILLISECOND, 0);
+        // Compute the start time of the job, based on current time
+        GregorianCalendar cal = Utils.getCalendarForNextRun(new GregorianCalendar(),
+                                                            voldemortConfig.getRetentionCleanupFirstStartDayOfWeek(),
+                                                            voldemortConfig.getRetentionCleanupFirstStartTimeInHour());
 
         // allow only one cleanup job at a time
         Date startTime = cal.getTime();
@@ -893,7 +891,8 @@ private void scheduleCleanupJob(StoreDefinition storeDef,
         this.scheduler.schedule("cleanup-" + storeDef.getName(),
                                 cleanupJob,
                                 startTime,
-                                retentionFreqHours * Time.MS_PER_HOUR);
+                                retentionFreqHours * Time.MS_PER_HOUR,
+                                voldemortConfig.getRetentionCleanupPinStartTime());
     }
 
     @Override

src/java/voldemort/utils/Utils.java

@@ -22,8 +22,10 @@
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Calendar;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.GregorianCalendar;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -542,4 +544,46 @@ public static boolean isSymLink(File symlinkFile) {
         }
     }
 
+    /**
+     * Given a start time, computes the next time when the wallclock will reach
+     * a certain hour of the day, on a certain day of the week Eg: From today,
+     * when is the next Saturday, 12PM ?
+     * 
+     * @param startTime start time
+     * @param targetDay day of the week to choose
+     * @param targetHour hour of the day to choose
+     * @return calendar object representing the target time
+     */
+    public static GregorianCalendar getCalendarForNextRun(GregorianCalendar startTime,
+                                                          int targetDay,
+                                                          int targetHour) {
+        long startTimeMs = startTime.getTimeInMillis();
+        GregorianCalendar cal = new GregorianCalendar();
+        cal.setTimeInMillis(startTimeMs);
+
+        // adjust time to targetHour on startDay
+        cal.set(Calendar.HOUR_OF_DAY, targetHour);
+        cal.set(Calendar.MINUTE, 0);
+        cal.set(Calendar.SECOND, 0);
+        cal.set(Calendar.MILLISECOND, 0);
+
+        // check if we are past the targetHour for the current day
+        if(cal.get(Calendar.DAY_OF_WEEK) != targetDay || cal.getTimeInMillis() < startTimeMs) {
+            do {
+                cal.add(Calendar.DAY_OF_YEAR, 1);
+            } while(cal.get(Calendar.DAY_OF_WEEK) != targetDay);
+        }
+        return cal;
+    }
+
+    /**
+     * Returns the day of week, 'nDays' from today
+     * 
+     * @return Calendar constant representing the day of the week
+     */
+    public static int getDayOfTheWeekFromNow(int nDays) {
+        GregorianCalendar cal = new GregorianCalendar();
+        cal.add(Calendar.DAY_OF_YEAR, nDays);
+        return cal.get(Calendar.DAY_OF_WEEK);
+    }
 }

test/common/voldemort/TestUtils.java

@@ -22,7 +22,9 @@
 import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Calendar;
 import java.util.Collections;
+import java.util.GregorianCalendar;
 import java.util.List;
 import java.util.Random;
 import java.util.SortedSet;
@@ -421,4 +423,24 @@ public static RoutingStrategy makeSingleNodeRoutingStrategy() {
         List<StoreDefinition> storeDefs = mapper.readStoreList(new StringReader(VoldemortTestConstants.getSingleStoreDefinitionsXml()));
         return new RoutingStrategyFactory().updateRoutingStrategy(storeDefs.get(0), cluster);
     }
+
+    /**
+     * Constructs a calendar object representing the given time
+     */
+    public static GregorianCalendar getCalendar(int year,
+                                                int month,
+                                                int day,
+                                                int hour,
+                                                int mins,
+                                                int secs) {
+        GregorianCalendar cal = new GregorianCalendar();
+        cal.set(Calendar.YEAR, year);
+        cal.set(Calendar.MONTH, month);
+        cal.set(Calendar.DATE, day);
+        cal.set(Calendar.HOUR_OF_DAY, hour);
+        cal.set(Calendar.MINUTE, mins);
+        cal.set(Calendar.SECOND, secs);
+        cal.set(Calendar.MILLISECOND, 0);
+        return cal;
+    }
 }

test/unit/voldemort/scheduled/DataCleanupJobTest.java

@@ -16,14 +16,18 @@
 
 package voldemort.scheduled;
 
+import static junit.framework.Assert.assertEquals;
 import static junit.framework.Assert.assertTrue;
 import static junit.framework.Assert.fail;
 
 import java.io.File;
 import java.util.Arrays;
+import java.util.Calendar;
 import java.util.Collection;
 import java.util.Date;
+import java.util.GregorianCalendar;
 import java.util.List;
+import java.util.Random;
 
 import org.apache.commons.io.FileDeleteStrategy;
 import org.junit.After;
@@ -47,6 +51,7 @@
 import voldemort.utils.Props;
 import voldemort.utils.SystemTime;
 import voldemort.utils.Time;
+import voldemort.utils.Utils;
 import voldemort.versioning.VectorClock;
 import voldemort.versioning.Versioned;
 
@@ -190,6 +195,63 @@ public void testCleanupCleansUp() {
         assertContains("a", "d", "e", "f");
     }
 
+    public void testCleanupStartTime() {
+        // Make sure the default is always the next day.
+        GregorianCalendar cal = new GregorianCalendar();
+        assertEquals("Default is not tomorrow",
+                     Utils.getDayOfTheWeekFromNow(1),
+                     (cal.get(Calendar.DAY_OF_WEEK) + 1) % 7);
+
+        // When starting the server any day in the week from SUN to FRI and
+        // targeting a saturday, should always start on the next saturday
+        GregorianCalendar expectedStart = TestUtils.getCalendar(2012,
+                                                                Calendar.SEPTEMBER,
+                                                                29,
+                                                                0,
+                                                                0,
+                                                                0);
+        Random rand = new Random();
+        for(int day = Calendar.SUNDAY; day <= Calendar.FRIDAY; day++) {
+            GregorianCalendar serverStartTime = TestUtils.getCalendar(2012,
+                                                                      Calendar.SEPTEMBER,
+                                                                      22 + day,
+                                                                      rand.nextInt(24),
+                                                                      rand.nextInt(60),
+                                                                      rand.nextInt(60));
+            GregorianCalendar computedStart = Utils.getCalendarForNextRun(serverStartTime,
+                                                                          Calendar.SATURDAY,
+                                                                          0);
+            assertEquals("Expected :" + expectedStart.getTimeInMillis() + " Computed: "
+                                 + computedStart.getTimeInMillis(),
+                         expectedStart.getTimeInMillis(),
+                         computedStart.getTimeInMillis());
+        }
+
+        // Targeting saturday, 00:00 and starting on a friday 23:59:59 should
+        // start the next saturday
+        GregorianCalendar serverStartTime = TestUtils.getCalendar(2012,
+                                                                  Calendar.SEPTEMBER,
+                                                                  28,
+                                                                  23,
+                                                                  59,
+                                                                  59);
+        GregorianCalendar computedStart = Utils.getCalendarForNextRun(serverStartTime,
+                                                                      Calendar.SATURDAY,
+                                                                      0);
+        assertEquals("Expected :" + expectedStart.getTimeInMillis() + " Computed: "
+                             + computedStart.getTimeInMillis(),
+                     expectedStart.getTimeInMillis(),
+                     computedStart.getTimeInMillis());
+
+        // If we start past the start hour on the target day, it should start
+        // the next week
+        serverStartTime = TestUtils.getCalendar(2012, Calendar.SEPTEMBER, 29, 1, 0, 1);
+        computedStart = Utils.getCalendarForNextRun(serverStartTime, Calendar.SATURDAY, 0);
+        assertEquals(Calendar.SATURDAY, computedStart.get(Calendar.DAY_OF_WEEK));
+        assertEquals(serverStartTime.get(Calendar.DAY_OF_YEAR) + 7,
+                     computedStart.get(Calendar.DAY_OF_YEAR));
+    }
+
     private void put(String... items) {
         for(String item: items) {
             VectorClock clock = null;