From 08e52584190686f88a7f28aeab4e5e51f0a749c9 Mon Sep 17 00:00:00 2001 From: Vinoth Chandar Date: Mon, 1 Oct 2012 10:50:03 -0700 Subject: [PATCH] Added parameters to control retention job 1. day of the week the retention job starts 2. if the retention job starts at the same hour each day --- .settings/org.eclipse.jdt.ui.prefs | 2 +- .../common/service/SchedulerService.java | 23 +++++-- .../voldemort/server/VoldemortConfig.java | 25 ++++++++ .../server/storage/StorageService.java | 15 +++-- src/java/voldemort/utils/Utils.java | 44 +++++++++++++ test/common/voldemort/TestUtils.java | 22 +++++++ .../scheduled/DataCleanupJobTest.java | 62 +++++++++++++++++++ 7 files changed, 180 insertions(+), 13 deletions(-) diff --git a/.settings/org.eclipse.jdt.ui.prefs b/.settings/org.eclipse.jdt.ui.prefs index 038d4674f0..b5553d65f5 100644 --- a/.settings/org.eclipse.jdt.ui.prefs +++ b/.settings/org.eclipse.jdt.ui.prefs @@ -1,4 +1,4 @@ -#Tue Jan 13 14:27:58 PST 2009 +#Sat Sep 22 05:05:45 PDT 2012 cleanup.add_default_serial_version_id=true cleanup.add_generated_serial_version_id=false cleanup.add_missing_annotations=true diff --git a/src/java/voldemort/common/service/SchedulerService.java b/src/java/voldemort/common/service/SchedulerService.java index 645be4015d..2cde4dd98a 100644 --- a/src/java/voldemort/common/service/SchedulerService.java +++ b/src/java/voldemort/common/service/SchedulerService.java @@ -164,10 +164,25 @@ public void schedule(String id, Runnable runnable, Date timeToRun) { } public void schedule(String id, Runnable runnable, Date nextRun, long periodMs) { - ScheduledFuture future = scheduler.scheduleWithFixedDelay(runnable, - delayMs(nextRun), - periodMs, - TimeUnit.MILLISECONDS); + schedule(id, runnable, nextRun, periodMs, false); + } + + public void schedule(String id, + Runnable runnable, + Date nextRun, + long periodMs, + boolean scheduleAtFixedRate) { + ScheduledFuture future = null; + if(scheduleAtFixedRate) + future = scheduler.scheduleAtFixedRate(runnable, + delayMs(nextRun), + periodMs, + TimeUnit.MILLISECONDS); + else + future = scheduler.scheduleWithFixedDelay(runnable, + delayMs(nextRun), + periodMs, + TimeUnit.MILLISECONDS); if(!allJobs.containsKey(id)) { allJobs.put(id, new ScheduledRunnable(runnable, nextRun, periodMs)); } diff --git a/src/java/voldemort/server/VoldemortConfig.java b/src/java/voldemort/server/VoldemortConfig.java index d5b8dafe24..b8ea5e4089 100644 --- a/src/java/voldemort/server/VoldemortConfig.java +++ b/src/java/voldemort/server/VoldemortConfig.java @@ -192,6 +192,8 @@ public class VoldemortConfig implements Serializable { private int retentionCleanupFirstStartTimeInHour; private int retentionCleanupScheduledPeriodInHour; + private int retentionCleanupFirstStartDayOfWeek; + private boolean retentionCleanupPinStartTime; private int maxRebalancingAttempt; private long rebalancingTimeoutSec; @@ -395,9 +397,16 @@ public VoldemortConfig(Props props) { // start at midnight (0-23) this.retentionCleanupFirstStartTimeInHour = props.getInt("retention.cleanup.first.start.hour", 0); + // start next day by default (1=SUN, 2=MON, 3=TUE, 4=WED, 5=THU, 6=FRI, + // 7=SAT) + this.retentionCleanupFirstStartDayOfWeek = props.getInt("retention.cleanup.first.start.day", + Utils.getDayOfTheWeekFromNow(1)); // repeat every 24 hours this.retentionCleanupScheduledPeriodInHour = props.getInt("retention.cleanup.period.hours", 24); + // should the retention job always start at the 'start time' specified + this.retentionCleanupPinStartTime = props.getBoolean("retention.cleanup.pin.start.time", + true); // save props for access from plugins this.allProps = props; @@ -1723,6 +1732,14 @@ public void setRetentionCleanupFirstStartTimeInHour(int retentionCleanupFirstSta this.retentionCleanupFirstStartTimeInHour = retentionCleanupFirstStartTimeInHour; } + public int getRetentionCleanupFirstStartDayOfWeek() { + return retentionCleanupFirstStartDayOfWeek; + } + + public void setRetentionCleanupFirstStartDayOfWeek(int retentionCleanupFirstStartDayOfWeek) { + this.retentionCleanupFirstStartDayOfWeek = retentionCleanupFirstStartDayOfWeek; + } + public int getRetentionCleanupScheduledPeriodInHour() { return retentionCleanupScheduledPeriodInHour; } @@ -1731,6 +1748,14 @@ public void setRetentionCleanupScheduledPeriodInHour(int retentionCleanupSchedul this.retentionCleanupScheduledPeriodInHour = retentionCleanupScheduledPeriodInHour; } + public boolean getRetentionCleanupPinStartTime() { + return retentionCleanupPinStartTime; + } + + public void setRetentionCleanupPinStartTime(boolean retentionCleanupFixStartTime) { + this.retentionCleanupPinStartTime = retentionCleanupFixStartTime; + } + public int getAdminSocketTimeout() { return adminSocketTimeout; } diff --git a/src/java/voldemort/server/storage/StorageService.java b/src/java/voldemort/server/storage/StorageService.java index f683f757f3..7eaa495419 100644 --- a/src/java/voldemort/server/storage/StorageService.java +++ b/src/java/voldemort/server/storage/StorageService.java @@ -102,6 +102,7 @@ import voldemort.utils.ReflectUtils; import voldemort.utils.SystemTime; import voldemort.utils.Time; +import voldemort.utils.Utils; import voldemort.versioning.VectorClock; import voldemort.versioning.VectorClockInconsistencyResolver; import voldemort.versioning.Versioned; @@ -857,13 +858,10 @@ private Store createNodeStore(String storeName, Node */ private void scheduleCleanupJob(StoreDefinition storeDef, StorageEngine engine) { - // Schedule data retention cleanup job starting next day. - GregorianCalendar cal = new GregorianCalendar(); - cal.add(Calendar.DAY_OF_YEAR, 1); - cal.set(Calendar.HOUR_OF_DAY, voldemortConfig.getRetentionCleanupFirstStartTimeInHour()); - cal.set(Calendar.MINUTE, 0); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); + // Compute the start time of the job, based on current time + GregorianCalendar cal = Utils.getCalendarForNextRun(new GregorianCalendar(), + voldemortConfig.getRetentionCleanupFirstStartDayOfWeek(), + voldemortConfig.getRetentionCleanupFirstStartTimeInHour()); // allow only one cleanup job at a time Date startTime = cal.getTime(); @@ -893,7 +891,8 @@ private void scheduleCleanupJob(StoreDefinition storeDef, this.scheduler.schedule("cleanup-" + storeDef.getName(), cleanupJob, startTime, - retentionFreqHours * Time.MS_PER_HOUR); + retentionFreqHours * Time.MS_PER_HOUR, + voldemortConfig.getRetentionCleanupPinStartTime()); } @Override diff --git a/src/java/voldemort/utils/Utils.java b/src/java/voldemort/utils/Utils.java index 59e0da14a7..8289cfa49f 100644 --- a/src/java/voldemort/utils/Utils.java +++ b/src/java/voldemort/utils/Utils.java @@ -22,8 +22,10 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Calendar; import java.util.Collections; import java.util.Comparator; +import java.util.GregorianCalendar; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -542,4 +544,46 @@ public static boolean isSymLink(File symlinkFile) { } } + /** + * Given a start time, computes the next time when the wallclock will reach + * a certain hour of the day, on a certain day of the week Eg: From today, + * when is the next Saturday, 12PM ? + * + * @param startTime start time + * @param targetDay day of the week to choose + * @param targetHour hour of the day to choose + * @return calendar object representing the target time + */ + public static GregorianCalendar getCalendarForNextRun(GregorianCalendar startTime, + int targetDay, + int targetHour) { + long startTimeMs = startTime.getTimeInMillis(); + GregorianCalendar cal = new GregorianCalendar(); + cal.setTimeInMillis(startTimeMs); + + // adjust time to targetHour on startDay + cal.set(Calendar.HOUR_OF_DAY, targetHour); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + + // check if we are past the targetHour for the current day + if(cal.get(Calendar.DAY_OF_WEEK) != targetDay || cal.getTimeInMillis() < startTimeMs) { + do { + cal.add(Calendar.DAY_OF_YEAR, 1); + } while(cal.get(Calendar.DAY_OF_WEEK) != targetDay); + } + return cal; + } + + /** + * Returns the day of week, 'nDays' from today + * + * @return Calendar constant representing the day of the week + */ + public static int getDayOfTheWeekFromNow(int nDays) { + GregorianCalendar cal = new GregorianCalendar(); + cal.add(Calendar.DAY_OF_YEAR, nDays); + return cal.get(Calendar.DAY_OF_WEEK); + } } diff --git a/test/common/voldemort/TestUtils.java b/test/common/voldemort/TestUtils.java index 107c07d475..901216a312 100644 --- a/test/common/voldemort/TestUtils.java +++ b/test/common/voldemort/TestUtils.java @@ -22,7 +22,9 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; +import java.util.Calendar; import java.util.Collections; +import java.util.GregorianCalendar; import java.util.List; import java.util.Random; import java.util.SortedSet; @@ -421,4 +423,24 @@ public static RoutingStrategy makeSingleNodeRoutingStrategy() { List storeDefs = mapper.readStoreList(new StringReader(VoldemortTestConstants.getSingleStoreDefinitionsXml())); return new RoutingStrategyFactory().updateRoutingStrategy(storeDefs.get(0), cluster); } + + /** + * Constructs a calendar object representing the given time + */ + public static GregorianCalendar getCalendar(int year, + int month, + int day, + int hour, + int mins, + int secs) { + GregorianCalendar cal = new GregorianCalendar(); + cal.set(Calendar.YEAR, year); + cal.set(Calendar.MONTH, month); + cal.set(Calendar.DATE, day); + cal.set(Calendar.HOUR_OF_DAY, hour); + cal.set(Calendar.MINUTE, mins); + cal.set(Calendar.SECOND, secs); + cal.set(Calendar.MILLISECOND, 0); + return cal; + } } diff --git a/test/unit/voldemort/scheduled/DataCleanupJobTest.java b/test/unit/voldemort/scheduled/DataCleanupJobTest.java index d7b694e8e7..1438ee7bc0 100644 --- a/test/unit/voldemort/scheduled/DataCleanupJobTest.java +++ b/test/unit/voldemort/scheduled/DataCleanupJobTest.java @@ -16,14 +16,18 @@ package voldemort.scheduled; +import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertTrue; import static junit.framework.Assert.fail; import java.io.File; import java.util.Arrays; +import java.util.Calendar; import java.util.Collection; import java.util.Date; +import java.util.GregorianCalendar; import java.util.List; +import java.util.Random; import org.apache.commons.io.FileDeleteStrategy; import org.junit.After; @@ -47,6 +51,7 @@ import voldemort.utils.Props; import voldemort.utils.SystemTime; import voldemort.utils.Time; +import voldemort.utils.Utils; import voldemort.versioning.VectorClock; import voldemort.versioning.Versioned; @@ -190,6 +195,63 @@ public void testCleanupCleansUp() { assertContains("a", "d", "e", "f"); } + public void testCleanupStartTime() { + // Make sure the default is always the next day. + GregorianCalendar cal = new GregorianCalendar(); + assertEquals("Default is not tomorrow", + Utils.getDayOfTheWeekFromNow(1), + (cal.get(Calendar.DAY_OF_WEEK) + 1) % 7); + + // When starting the server any day in the week from SUN to FRI and + // targeting a saturday, should always start on the next saturday + GregorianCalendar expectedStart = TestUtils.getCalendar(2012, + Calendar.SEPTEMBER, + 29, + 0, + 0, + 0); + Random rand = new Random(); + for(int day = Calendar.SUNDAY; day <= Calendar.FRIDAY; day++) { + GregorianCalendar serverStartTime = TestUtils.getCalendar(2012, + Calendar.SEPTEMBER, + 22 + day, + rand.nextInt(24), + rand.nextInt(60), + rand.nextInt(60)); + GregorianCalendar computedStart = Utils.getCalendarForNextRun(serverStartTime, + Calendar.SATURDAY, + 0); + assertEquals("Expected :" + expectedStart.getTimeInMillis() + " Computed: " + + computedStart.getTimeInMillis(), + expectedStart.getTimeInMillis(), + computedStart.getTimeInMillis()); + } + + // Targeting saturday, 00:00 and starting on a friday 23:59:59 should + // start the next saturday + GregorianCalendar serverStartTime = TestUtils.getCalendar(2012, + Calendar.SEPTEMBER, + 28, + 23, + 59, + 59); + GregorianCalendar computedStart = Utils.getCalendarForNextRun(serverStartTime, + Calendar.SATURDAY, + 0); + assertEquals("Expected :" + expectedStart.getTimeInMillis() + " Computed: " + + computedStart.getTimeInMillis(), + expectedStart.getTimeInMillis(), + computedStart.getTimeInMillis()); + + // If we start past the start hour on the target day, it should start + // the next week + serverStartTime = TestUtils.getCalendar(2012, Calendar.SEPTEMBER, 29, 1, 0, 1); + computedStart = Utils.getCalendarForNextRun(serverStartTime, Calendar.SATURDAY, 0); + assertEquals(Calendar.SATURDAY, computedStart.get(Calendar.DAY_OF_WEEK)); + assertEquals(serverStartTime.get(Calendar.DAY_OF_YEAR) + 7, + computedStart.get(Calendar.DAY_OF_YEAR)); + } + private void put(String... items) { for(String item: items) { VectorClock clock = null;