Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Incomplete version of making push version generic. No more do we requ…

…ire continuous ids
  • Loading branch information...
commit 5a3ed4675ad04f63b1cf57a3b93deafbf2b73144 1 parent 49ae13c
Roshan Sumbaly rsumbaly authored
6 src/java/voldemort/server/http/gui/ReadOnlyStoreManagementServlet.java
View
@@ -178,13 +178,13 @@ private void doFetch(HttpServletRequest req, HttpServletResponse resp) throws IO
long pushVersion;
if(pushVersionString == null) {
- pushVersion = store.getMaxVersion() + 1;
+ pushVersion = store.getMaxVersionId() + 1;
} else {
pushVersion = Long.parseLong(pushVersionString);
- if(pushVersion <= store.getMaxVersion()) {
+ if(pushVersion <= store.getMaxVersionId()) {
throw new ServletException("Version of push specified (" + pushVersion
+ ") should be greater than current version "
- + store.getMaxVersion());
+ + store.getMaxVersionId());
}
}
6 src/java/voldemort/server/protocol/admin/AdminServiceRequestHandler.java
View
@@ -360,13 +360,13 @@ public StreamRequestHandler handleUpdatePartitionEntries(VAdminProto.UpdateParti
final long pushVersion;
if(request.hasPushVersion()) {
pushVersion = request.getPushVersion();
- if(pushVersion <= store.getMaxVersion()) {
+ if(pushVersion <= store.getMaxVersionId()) {
throw new VoldemortException("Version of push specified (" + pushVersion
+ ") should be greater than current version "
- + store.getMaxVersion());
+ + store.getMaxVersionId());
}
} else {
- pushVersion = store.getMaxVersion() + 1;
+ pushVersion = store.getMaxVersionId() + 1;
}
asyncService.submitOperation(requestId, new AsyncOperation(requestId, "Fetch store") {
109 src/java/voldemort/store/readonly/ReadOnlyStorageEngine.java
View
@@ -65,7 +65,7 @@
private final String name;
private final int numBackups;
- private int maxVersion;
+ private long maxVersionId;
private final File storeDir;
private final ReadWriteLock fileModificationLock;
private final SearchStrategy searchStrategy;
@@ -89,7 +89,7 @@ public ReadOnlyStorageEngine(String name,
this.name = Utils.notNull(name);
this.searchStrategy = searchStrategy;
this.fileSet = null;
- this.maxVersion = 0;
+ this.maxVersionId = 0L;
/*
* A lock that blocks reads during swap(), open(), and close()
* operations
@@ -100,7 +100,8 @@ public ReadOnlyStorageEngine(String name,
}
/**
- * Open the store with the version directory specified
+ * Open the store with the version directory specified. If null is specified
+ * we open the max versioned directory
*
* @param versionDir Version Directory to use
*/
@@ -118,7 +119,7 @@ public void open(File versionDir) {
} else {
setMaxVersion(versionDir);
}
- versionDir.mkdirs();
+ Utils.mkdirs(versionDir);
// Create symbolic link
logger.info("Creating symbolic link for '" + getName() + "' using directory "
@@ -133,29 +134,13 @@ public void open(File versionDir) {
}
/**
- * Find the latest version directory. First checks for 'latest' symbolic
- * link, if it does not exist falls back to max version number
+ * Find the max versioned directory
*
* @param storeDir
- * @return the directory with the latest version
+ * @return the directory with the max version
*/
public File findLatestVersion() {
- // Return latest symbolic link if it exists
- File latestVersion = new File(storeDir, "latest");
- if(latestVersion.exists() && Utils.isSymLink(latestVersion)) {
- File canonicalLatestDir = null;
- try {
- canonicalLatestDir = latestVersion.getCanonicalFile();
- } catch(IOException e) {}
-
- // Check if canonical directory exists, if not fall back to manual
- // search
- if(canonicalLatestDir != null) {
- return setMaxVersion(canonicalLatestDir);
- }
- }
-
- File[] versionDirs = storeDir.listFiles();
+ File[] versionDirs = ReadOnlyUtils.versionDirs(storeDir);
// No version directories exist, create new empty folder
if(versionDirs == null || versionDirs.length == 0) {
@@ -163,45 +148,22 @@ public File findLatestVersion() {
return setMaxVersion(version0);
}
- return setMaxVersion(versionDirs);
+ return setMaxVersion(ReadOnlyUtils.findKthVersionedDir(versionDirs, versionDirs.length));
}
- public int getMaxVersion() {
- return maxVersion;
+ public long getMaxVersionId() {
+ return maxVersionId;
}
public String getStoreDirPath() {
return storeDir.getAbsolutePath();
}
- private boolean checkVersionDirName(File dir) {
- if(dir.isDirectory() && dir.getName().contains("version-")
- && !dir.getName().endsWith(".bak")) {
- return true;
- } else {
- return false;
- }
- }
-
- private File setMaxVersion(File[] versionDirs) {
- int max = 0;
- for(File versionDir: versionDirs) {
- if(checkVersionDirName(versionDir)) {
- int version = Integer.parseInt(versionDir.getName().replace("version-", ""));
- if(version > max) {
- max = version;
- }
- }
- }
- maxVersion = max;
- return new File(storeDir, "version-" + maxVersion);
- }
-
private File setMaxVersion(File versionDir) {
- if(checkVersionDirName(versionDir)) {
- maxVersion = Integer.parseInt(versionDir.getName().replace("version-", ""));
+ if(ReadOnlyUtils.checkVersionDirName(versionDir)) {
+ maxVersionId = ReadOnlyUtils.getVersionId(versionDir);
}
- return new File(storeDir, "version-" + maxVersion);
+ return new File(storeDir, "version-" + maxVersionId);
}
/**
@@ -238,11 +200,30 @@ public void swapFiles(String newStoreDirectory) {
throw new VoldemortException("File " + newDataDir.getAbsolutePath()
+ " does not exist.");
- if(!(newDataDir.getParentFile().compareTo(storeDir.getAbsoluteFile()) == 0 && checkVersionDirName(newDataDir)))
+ if(!(newDataDir.getParentFile().compareTo(storeDir.getAbsoluteFile()) == 0 && ReadOnlyUtils.checkVersionDirName(newDataDir)))
throw new VoldemortException("Invalid version folder name '"
+ newDataDir
+ "'. Either parent directory is incorrect or format(version-n) is incorrect");
+ // check if we're greater than latest since we want last write to win
+ File latestVersion = new File(storeDir, "latest");
+ if(latestVersion.exists() && Utils.isSymLink(latestVersion)) {
+ File canonicalLatestVersion = null;
+ try {
+ canonicalLatestVersion = latestVersion.getCanonicalFile();
+ } catch(IOException e) {}
+
+ // Check if canonical directory exists, if not fall back to manual
+ // search
+ if(canonicalLatestVersion != null
+ && ReadOnlyUtils.checkVersionDirName(canonicalLatestVersion)) {
+ long latestVersionId = ReadOnlyUtils.getVersionId(canonicalLatestVersion);
+ long newVersionId = ReadOnlyUtils.getVersionId(newDataDir);
+ if(latestVersionId >= newVersionId)
+ return;
+ }
+ }
+
logger.info("Acquiring write lock on '" + getName() + "':");
fileModificationLock.writeLock().lock();
boolean success = false;
@@ -251,7 +232,7 @@ public void swapFiles(String newStoreDirectory) {
logger.info("Opening primary files for store '" + getName() + "' at "
+ newStoreDirectory);
- // open the new store
+ // open the latest store
open(newDataDir);
success = true;
} finally {
@@ -270,8 +251,10 @@ public void swapFiles(String newStoreDirectory) {
}
// okay we have released the lock and the store is now open again, it is
// safe to do a potentially slow delete if we have one too many backups
- File extraBackup = new File(storeDir, "version-" + (maxVersion - numBackups - 1));
- if(extraBackup.exists())
+ File[] storeDirList = ReadOnlyUtils.versionDirs(storeDir);
+ File extraBackup = ReadOnlyUtils.findKthVersionedDir(storeDirList, storeDirList.length
+ - numBackups - 1);
+ if(extraBackup != null && extraBackup.exists())
deleteAsync(extraBackup);
}
@@ -302,15 +285,17 @@ public void rollback() {
try {
if(isOpen)
close();
- File backup = new File(storeDir, "version-" + (maxVersion - 1));
- if(!backup.exists())
- throw new VoldemortException("Version " + (maxVersion - 1)
- + " does not exists, nothing to roll back to.");
- File primary = new File(storeDir, "version-" + maxVersion);
+ // find previous version
+ File[] storeDirList = ReadOnlyUtils.versionDirs(storeDir);
+ File backup = ReadOnlyUtils.findKthVersionedDir(storeDirList, storeDirList.length - 1);
+ if(backup == null || !backup.exists())
+ throw new VoldemortException("Previous version does not exists, nothing to roll back to.");
+
+ File primary = new File(storeDir, "version-" + maxVersionId);
DateFormat df = new SimpleDateFormat("MM-dd-yyyy");
if(primary.exists())
- Utils.move(primary, new File(storeDir, "version-" + maxVersion + "."
+ Utils.move(primary, new File(storeDir, "version-" + maxVersionId + "."
+ df.format(new Date()) + ".bak"));
open(backup);
} finally {
58 src/java/voldemort/store/readonly/ReadOnlyUtils.java
View
@@ -1,6 +1,9 @@
package voldemort.store.readonly;
+import java.io.File;
+import java.io.FilenameFilter;
import java.nio.ByteBuffer;
+import java.util.Random;
import voldemort.utils.ByteUtils;
@@ -21,4 +24,59 @@ public static int chunk(byte[] key, int numChunks) {
return foundKey;
}
+ public static boolean checkVersionDirName(File dir) {
+ return (dir.isDirectory() && dir.getName().contains("version-") && !dir.getName()
+ .endsWith(".bak"));
+ }
+
+ public static long getVersionId(File versionDir) {
+ return Long.parseLong(versionDir.getName().replace("version-", ""));
+ }
+
+ public static File[] versionDirs(File rootDir) {
+ return rootDir.listFiles(new FilenameFilter() {
+
+ public boolean accept(File dir, String name) {
+ return checkVersionDirName(dir);
+ }
+ });
+ }
+
+ public static File findKthVersionedDir(File[] versionDirs, int kth) {
+ if(kth < versionDirs.length) {
+ return null;
+ }
+ return findKthVersionedDir(versionDirs, 0, versionDirs.length, kth);
+ }
+
+ private static int partition(File[] versionDirs, int left, int right, int pivotIndex) {
+ File pivotValue = versionDirs[pivotIndex];
+
+ swap(versionDirs[pivotIndex],versionDirs[right]);
+ int storeIndex = left
+ for i from left to right-1
+ if list[i] < pivotValue
+ swap list[storeIndex] and list[i]
+ storeIndex := storeIndex + 1
+ swap list[right] and list[storeIndex] // Move pivot to its final place
+ return storeIndex
+
+ return 0;
+ }
+
+ private static File findKthVersionedDir(File[] versionDirs, int left, int right, int kth) {
+ Random randomizer = new Random();
+ while(true) {
+ int pivotIndex = randomizer.nextInt(right - left) + left;
+ int pivotNewIndex = partition(versionDirs, left, right, pivotIndex);
+ if(kth == pivotIndex)
+ return versionDirs[kth];
+ else if(kth < pivotNewIndex)
+ right = pivotNewIndex - 1;
+ else {
+ left = pivotNewIndex + 1;
+ kth -= pivotNewIndex;
+ }
+ }
+ }
}
Please sign in to comment.
Something went wrong with that request. Please try again.