Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

First version of better versioned pushes

  • Loading branch information...
commit d4d39955c1e2dd57d732dc06f615ed15e8cf4c00 1 parent 3ba4e35
Roshan Sumbaly rsumbaly authored
1  .classpath
View
@@ -50,5 +50,6 @@
<classpathentry kind="lib" path="lib/avro-modified-jdk5-1.3.0.jar"/>
<classpathentry kind="lib" path="contrib/hadoop/lib/pig-0.7.1-dev-core.jar"/>
<classpathentry kind="lib" path="contrib/krati/lib/krati-0.3.4.jar"/>
+ <classpathentry kind="lib" path="lib/jna.jar"/>
<classpathentry kind="output" path="classes"/>
</classpath>
25 contrib/hadoop-store-builder/src/java/voldemort/store/readonly/fetcher/HdfsFetcher.java
View
@@ -55,8 +55,6 @@
public class HdfsFetcher implements FileFetcher {
private static final Logger logger = Logger.getLogger(HdfsFetcher.class);
- private static final String DEFAULT_TEMP_DIR = new File(System.getProperty("java.io.tmpdir"),
- "hdfs-fetcher").getAbsolutePath();
private static final int REPORTING_INTERVAL_BYTES = 100 * 1024 * 1024;
private static final int DEFAULT_BUFFER_SIZE = 64 * 1024;
@@ -70,45 +68,40 @@
public HdfsFetcher(Props props) {
this(props.containsKey("fetcher.max.bytes.per.sec") ? props.getBytes("fetcher.max.bytes.per.sec")
: null,
- new File(props.getString("hdfs.fetcher.tmp.dir", DEFAULT_TEMP_DIR)),
(int) props.getBytes("hdfs.fetcher.buffer.size", DEFAULT_BUFFER_SIZE));
logger.info("Created hdfs fetcher with temp dir = " + tempDir.getAbsolutePath()
+ " and throttle rate " + maxBytesPerSecond + " and buffer size " + bufferSize);
}
public HdfsFetcher() {
- this((Long) null, null, DEFAULT_BUFFER_SIZE);
+ this((Long) null, DEFAULT_BUFFER_SIZE);
}
- public HdfsFetcher(Long maxBytesPerSecond, File tempDir, int bufferSize) {
- if(tempDir == null)
- this.tempDir = new File(DEFAULT_TEMP_DIR);
- else
- this.tempDir = Utils.notNull(new File(tempDir, "hdfs-fetcher"));
+ public HdfsFetcher(Long maxBytesPerSecond, int bufferSize) {
this.maxBytesPerSecond = maxBytesPerSecond;
if(this.maxBytesPerSecond != null)
this.throttler = new EventThrottler(this.maxBytesPerSecond);
this.bufferSize = bufferSize;
this.status = null;
- Utils.mkdirs(this.tempDir);
}
- public File fetch(String fileUrl, String storeName) throws IOException {
- Path path = new Path(fileUrl);
+ public File fetch(String sourceFileUrl, String destinationFile, String storeName)
+ throws IOException {
+ Path path = new Path(sourceFileUrl);
Configuration config = new Configuration();
config.setInt("io.socket.receive.buffer", bufferSize);
config.set("hadoop.rpc.socket.factory.class.ClientProtocol",
ConfigurableSocketFactory.class.getName());
FileSystem fs = path.getFileSystem(config);
- CopyStats stats = new CopyStats(fileUrl, sizeOfPath(fs, path));
+ CopyStats stats = new CopyStats(sourceFileUrl, sizeOfPath(fs, path));
ObjectName jmxName = JmxUtils.registerMbean("hdfs-copy-" + copyCount.getAndIncrement(),
stats);
try {
File storeDir = new File(this.tempDir, storeName + "_" + System.currentTimeMillis());
Utils.mkdirs(storeDir);
- File destination = new File(storeDir.getAbsoluteFile(), path.getName());
+ File destination = new File(destinationFile);
boolean result = fetch(fs, path, destination, stats);
if(result) {
return destination;
@@ -342,9 +335,9 @@ public static void main(String[] args) throws Exception {
config.setInt("io.socket.receive.buffer", 1 * 1024 * 1024 - 10000);
FileStatus status = p.getFileSystem(config).getFileStatus(p);
long size = status.getLen();
- HdfsFetcher fetcher = new HdfsFetcher(maxBytesPerSec, null, DEFAULT_BUFFER_SIZE);
+ HdfsFetcher fetcher = new HdfsFetcher(maxBytesPerSec, DEFAULT_BUFFER_SIZE);
long start = System.currentTimeMillis();
- File location = fetcher.fetch(url, storeName);
+ File location = fetcher.fetch(url, System.getProperty("java.io.tmpdir"), storeName);
double rate = size * Time.MS_PER_SECOND / (double) (System.currentTimeMillis() - start);
NumberFormat nf = NumberFormat.getInstance();
nf.setMaximumFractionDigits(2);
51 contrib/hadoop-store-builder/test/voldemort/store/readonly/fetcher/HdfsFetcherTest.java
View
@@ -33,72 +33,87 @@
public class HdfsFetcherTest extends TestCase {
public void testFetch() throws Exception {
- File testDirectory = TestUtils.createTempDir();
+ File testSourceDirectory = TestUtils.createTempDir();
+ File testDestinationDirectory = TestUtils.createTempDir();
- File testFile = File.createTempFile("test", ".dat", testDirectory);
+ File testFile = File.createTempFile("test", ".dat", testSourceDirectory);
testFile.createNewFile();
// Test 1: No checksum file - return correctly
// Required for backward compatibility with existing hadoop stores
HdfsFetcher fetcher = new HdfsFetcher();
- File fetchedFile = fetcher.fetch(testDirectory.getAbsolutePath(), "storeName");
+ File fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(),
+ testDestinationDirectory.getAbsolutePath(),
+ "storeName");
assertNotNull(fetchedFile);
// Test 2: Add checksum file with incorrect fileName, should not fail
- File checkSumFile = new File(testDirectory, "blahcheckSum.txt");
+ File checkSumFile = new File(testSourceDirectory, "blahcheckSum.txt");
checkSumFile.createNewFile();
- fetchedFile = fetcher.fetch(testDirectory.getAbsolutePath(), "storeName");
+ fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(),
+ testDestinationDirectory.getAbsolutePath(),
+ "storeName");
assertNotNull(fetchedFile);
checkSumFile.delete();
// Test 3: Add checksum file with correct fileName, but empty = wrong
// md5
- checkSumFile = new File(testDirectory, "adler32checkSum.txt");
+ checkSumFile = new File(testSourceDirectory, "adler32checkSum.txt");
checkSumFile.createNewFile();
- fetchedFile = fetcher.fetch(testDirectory.getAbsolutePath(), "storeName");
+ fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(),
+ testDestinationDirectory.getAbsolutePath(),
+ "storeName");
assertNull(fetchedFile);
// Test 4: Add wrong contents to file i.e. contents of CRC32 instead of
// Adler
- byte[] checkSumBytes = CheckSumTests.calculateCheckSum(testDirectory.listFiles(),
+ byte[] checkSumBytes = CheckSumTests.calculateCheckSum(testSourceDirectory.listFiles(),
CheckSumType.CRC32);
DataOutputStream os = new DataOutputStream(new FileOutputStream(checkSumFile));
os.write(checkSumBytes);
os.close();
- fetchedFile = fetcher.fetch(testDirectory.getAbsolutePath(), "storeName");
+ fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(),
+ testDestinationDirectory.getAbsolutePath(),
+ "storeName");
assertNull(fetchedFile);
checkSumFile.delete();
// Test 5: Add correct checksum contents - MD5
- checkSumFile = new File(testDirectory, "md5checkSum.txt");
- byte[] checkSumBytes2 = CheckSumTests.calculateCheckSum(testDirectory.listFiles(),
+ checkSumFile = new File(testSourceDirectory, "md5checkSum.txt");
+ byte[] checkSumBytes2 = CheckSumTests.calculateCheckSum(testSourceDirectory.listFiles(),
CheckSumType.MD5);
os = new DataOutputStream(new FileOutputStream(checkSumFile));
os.write(checkSumBytes2);
os.close();
- fetchedFile = fetcher.fetch(testDirectory.getAbsolutePath(), "storeName");
+ fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(),
+ testDestinationDirectory.getAbsolutePath(),
+ "storeName");
assertNotNull(fetchedFile);
checkSumFile.delete();
// Test 6: Add correct checksum contents - ADLER32
- checkSumFile = new File(testDirectory, "adler32checkSum.txt");
- byte[] checkSumBytes3 = CheckSumTests.calculateCheckSum(testDirectory.listFiles(),
+ checkSumFile = new File(testSourceDirectory, "adler32checkSum.txt");
+ byte[] checkSumBytes3 = CheckSumTests.calculateCheckSum(testSourceDirectory.listFiles(),
CheckSumType.ADLER32);
os = new DataOutputStream(new FileOutputStream(checkSumFile));
os.write(checkSumBytes3);
os.close();
- fetchedFile = fetcher.fetch(testDirectory.getAbsolutePath(), "storeName");
+ fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(),
+ testDestinationDirectory.getAbsolutePath(),
+ "storeName");
assertNotNull(fetchedFile);
checkSumFile.delete();
// Test 7: Add correct checksum contents - CRC32
- checkSumFile = new File(testDirectory, "crc32checkSum.txt");
- byte[] checkSumBytes4 = CheckSumTests.calculateCheckSum(testDirectory.listFiles(),
+ checkSumFile = new File(testSourceDirectory, "crc32checkSum.txt");
+ byte[] checkSumBytes4 = CheckSumTests.calculateCheckSum(testSourceDirectory.listFiles(),
CheckSumType.CRC32);
os = new DataOutputStream(new FileOutputStream(checkSumFile));
os.write(checkSumBytes4);
os.close();
- fetchedFile = fetcher.fetch(testDirectory.getAbsolutePath(), "storeName");
+ fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(),
+ testDestinationDirectory.getAbsolutePath(),
+ "storeName");
assertNotNull(fetchedFile);
checkSumFile.delete();
6 src/java/voldemort/server/http/gui/ReadOnlyStoreManagementServlet.java
View
@@ -172,6 +172,7 @@ private void doFetch(HttpServletRequest req, HttpServletResponse resp) throws IO
ServletException {
String fetchUrl = getRequired(req, "dir");
String storeName = getOptional(req, "store");
+ ReadOnlyStorageEngine store = this.getStore(storeName);
// fetch the files if necessary
File fetchDir = null;
@@ -181,7 +182,10 @@ private void doFetch(HttpServletRequest req, HttpServletResponse resp) throws IO
} else {
logger.info("Executing fetch of " + fetchUrl);
try {
- fetchDir = fileFetcher.fetch(fetchUrl, storeName);
+ fetchDir = fileFetcher.fetch(fetchUrl,
+ store.getStoreDirPath() + File.separator + "version-"
+ + (store.getMaxVersion() + 1),
+ storeName);
} catch(Exception e) {
throw new ServletException("Exception in Fetcher = " + e.getMessage());
}
16 src/java/voldemort/server/protocol/admin/AdminServiceRequestHandler.java
View
@@ -329,10 +329,9 @@ public StreamRequestHandler handleUpdatePartitionEntries(VAdminProto.UpdateParti
VAdminProto.SwapStoreResponse.Builder response = VAdminProto.SwapStoreResponse.newBuilder();
try {
- ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) storeRepository.getStorageEngine(storeName);
- if(store == null)
- throw new VoldemortException("'" + storeName
- + "' is not a registered read-only store.");
+ ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository,
+ storeName);
+
if(!Utils.isReadableDir(dir))
throw new VoldemortException("Store directory '" + dir
+ "' is not a readable directory.");
@@ -367,6 +366,9 @@ public void markComplete() {
@Override
public void operate() {
+ ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStorageEngine(storeRepository,
+ storeName);
+
File fetchDir = null;
if(fileFetcher == null) {
@@ -377,7 +379,11 @@ public void operate() {
updateStatus("Executing fetch of " + fetchUrl);
try {
fileFetcher.setAsyncOperationStatus(status);
- fetchDir = fileFetcher.fetch(fetchUrl, storeName);
+ fetchDir = fileFetcher.fetch(fetchUrl,
+ store.getStoreDirPath() + File.separator
+ + "version-"
+ + (store.getMaxVersion() + 1),
+ storeName);
updateStatus("Completed fetch of " + fetchUrl);
} catch(Exception e) {
throw new VoldemortException("Exception in Fetcher = " + e.getMessage());
3  src/java/voldemort/store/readonly/FileFetcher.java
View
@@ -4,6 +4,7 @@
import java.io.IOException;
import voldemort.server.protocol.admin.AsyncOperationStatus;
+
/**
* An interface to fetch data for readonly store. The fetch could be via rsync
* or hdfs. If the store is already on the local filesystem then no fetcher is
@@ -16,7 +17,7 @@
*/
public interface FileFetcher {
- public File fetch(String fileUrl, String storeName) throws IOException;
+ public File fetch(String source, String dest, String storeName) throws IOException;
public void setAsyncOperationStatus(AsyncOperationStatus status);
}
202 src/java/voldemort/store/readonly/ReadOnlyStorageEngine.java
View
@@ -65,6 +65,7 @@
private final String name;
private final int numBackups;
+ private int maxVersion;
private final File storeDir;
private final ReadWriteLock fileModificationLock;
private final SearchStrategy searchStrategy;
@@ -75,16 +76,9 @@
* Create an instance of the store
*
* @param name The name of the store
- * @param storageDir The directory in which the .data and .index files
- * reside
+ * @param searchStrategy The algorithm to use for searching for keys
+ * @param storeDir The directory in which the .data and .index files reside
* @param numBackups The number of backups of these files to retain
- * @param numFileHandles The number of file descriptors to keep pooled for
- * each file
- * @param bufferWaitTimeoutMs The maximum time to wait to acquire a file
- * handle
- * @param maxCacheSizeBytes The maximum size of the cache, in bytes. The
- * actual size of the cache will be the largest power of two lower
- * than this number
*/
public ReadOnlyStorageEngine(String name,
SearchStrategy searchStrategy,
@@ -95,19 +89,22 @@ public ReadOnlyStorageEngine(String name,
this.name = Utils.notNull(name);
this.searchStrategy = searchStrategy;
this.fileSet = null;
+ this.maxVersion = 0;
/*
* A lock that blocks reads during swap(), open(), and close()
* operations
*/
this.fileModificationLock = new ReentrantReadWriteLock();
this.isOpen = false;
- open();
+ open(null);
}
/**
- * Open the store
+ * Open the store with the version directory specified
+ *
+ * @param versionDir Version Directory to use
*/
- public void open() {
+ public void open(File versionDir) {
/* acquire modification lock */
fileModificationLock.writeLock().lock();
try {
@@ -115,9 +112,20 @@ public void open() {
if(isOpen)
throw new IllegalStateException("Attempt to open already open store.");
- File version0 = new File(storeDir, "version-0");
- version0.mkdirs();
- this.fileSet = new ChunkedFileSet(version0);
+ // Find latest directory
+ if(versionDir == null) {
+ versionDir = findLatestVersion();
+ } else {
+ setMaxVersion(versionDir);
+ }
+ versionDir.mkdirs();
+
+ // Create symbolic link
+ logger.info("Creating symbolic link for '" + getName() + "' using directory "
+ + versionDir.getAbsolutePath());
+ Utils.symlink(versionDir.getAbsolutePath(), storeDir.getAbsolutePath() + File.separator
+ + "latest");
+ this.fileSet = new ChunkedFileSet(versionDir);
isOpen = true;
} finally {
fileModificationLock.writeLock().unlock();
@@ -125,6 +133,69 @@ public void open() {
}
/**
+ * Find the latest version directory. First checks for 'latest' symbolic
+ * link, if it does not exist falls back to max version number
+ *
+ * @param storeDir
+ * @return the directory with the latest version
+ */
+ public File findLatestVersion() {
+ // Return latest symbolic link if it exists
+ File latestVersion = new File(storeDir, "latest");
+ if(latestVersion.exists() && Utils.isSymLink(latestVersion)) {
+ File canonicalLatestDir = null;
+ try {
+ canonicalLatestDir = latestVersion.getCanonicalFile();
+ } catch(IOException e) {}
+
+ // Check if canonical directory exists, if not fall back to manual
+ // search
+ if(canonicalLatestDir != null) {
+ return setMaxVersion(canonicalLatestDir);
+ }
+ }
+
+ File[] versionDirs = storeDir.listFiles();
+
+ // No version directories exist, create new empty folder
+ if(versionDirs == null || versionDirs.length == 0) {
+ File version0 = new File(storeDir, "version-0");
+ return setMaxVersion(version0);
+ }
+
+ return setMaxVersion(versionDirs);
+ }
+
+ public int getMaxVersion() {
+ return maxVersion;
+ }
+
+ public String getStoreDirPath() {
+ return storeDir.getAbsolutePath();
+ }
+
+ private File setMaxVersion(File[] versionDirs) {
+ int max = 0;
+ for(File versionDir: versionDirs) {
+ if(versionDir.isDirectory() && versionDir.getName().contains("version-")) {
+ int version = Integer.parseInt(versionDir.getName().replace("version-", ""));
+ if(version > max) {
+ max = version;
+ }
+ }
+ }
+ maxVersion = max;
+ return new File(storeDir, "version-" + maxVersion);
+ }
+
+ private File setMaxVersion(File versionDir) {
+ if(versionDir.isDirectory() && versionDir.getName().contains("version-")) {
+ maxVersion = Integer.parseInt(versionDir.getName().replace("version-", ""));
+ }
+ return new File(storeDir, "version-" + maxVersion);
+ }
+
+ /**
* Close the store.
*/
public void close() throws VoldemortException {
@@ -149,32 +220,29 @@ public void close() throws VoldemortException {
* @param newIndexFile The path to the new index file
* @param newDataFile The path to the new data file
*/
- @JmxOperation(description = "swapFiles(newIndexFile, newDataFile) changes this store "
- + " to use the given index and data file.")
+ @JmxOperation(description = "swapFiles(newStoreDirectory) changes this store "
+ + " to use the new data directory.")
public void swapFiles(String newStoreDirectory) {
- logger.info("Swapping files for store '" + getName() + "' from " + newStoreDirectory);
+ logger.info("Swapping files for store '" + getName() + "' to " + newStoreDirectory);
File newDataDir = new File(newStoreDirectory);
if(!newDataDir.exists())
throw new VoldemortException("File " + newDataDir.getAbsolutePath()
+ " does not exist.");
+ if(!newDataDir.getName().startsWith("version-"))
+ throw new VoldemortException("Invalid version folder name '" + newDataDir.getName()
+ + "'. Should be of the format 'version-n'");
+
logger.info("Acquiring write lock on '" + getName() + "':");
fileModificationLock.writeLock().lock();
boolean success = false;
try {
close();
- logger.info("Renaming data and index files for '" + getName() + "':");
- shiftBackupsRight();
- // copy in new files
- logger.info("Setting primary files for store '" + getName() + "' to "
+ logger.info("Opening primary files for store '" + getName() + "' at "
+ newStoreDirectory);
- File destDir = new File(storeDir, "version-0");
- if(!newDataDir.renameTo(destDir))
- throw new VoldemortException("Renaming " + newDataDir.getAbsolutePath() + " to "
- + destDir.getAbsolutePath() + " failed!");
// open the new store
- open();
+ open(newDataDir);
success = true;
} finally {
try {
@@ -192,13 +260,13 @@ public void swapFiles(String newStoreDirectory) {
}
// okay we have released the lock and the store is now open again, it is
// safe to do a potentially slow delete if we have one too many backups
- File extraBackup = new File(storeDir, "version-" + (numBackups + 1));
+ File extraBackup = new File(storeDir, "version-" + (maxVersion - numBackups - 1));
if(extraBackup.exists())
deleteAsync(extraBackup);
}
/**
- * Delete the given file in a seperate thread
+ * Delete the given file in a separate thread
*
* @param file The file to delete
*/
@@ -219,82 +287,28 @@ public void run() {
@JmxOperation(description = "Rollback to the most recent backup of the current store.")
public void rollback() {
- logger.info("Rolling back store '" + getName() + "' to version 1.");
+ logger.info("Rolling back store '" + getName() + "'");
fileModificationLock.writeLock().lock();
try {
if(isOpen)
close();
- File backup = new File(storeDir, "version-1");
+ File backup = new File(storeDir, "version-" + (maxVersion - 1));
if(!backup.exists())
- throw new VoldemortException("Version 1 does not exists, nothing to roll back to.");
- shiftBackupsLeft();
- open();
+ throw new VoldemortException("Version " + (maxVersion - 1)
+ + " does not exists, nothing to roll back to.");
+
+ File primary = new File(storeDir, "version-" + maxVersion);
+ DateFormat df = new SimpleDateFormat("MM-dd-yyyy");
+ if(primary.exists())
+ Utils.move(primary, new File(storeDir, "version-" + maxVersion + "."
+ + df.format(new Date()) + ".bak"));
+ open(backup);
} finally {
fileModificationLock.writeLock().unlock();
logger.info("Rollback operation completed on '" + getName() + "', releasing lock.");
}
}
- /**
- * Shift all store versions so that 1 becomes 0, 2 becomes 1, etc.
- */
- private void shiftBackupsLeft() {
- if(isOpen)
- throw new VoldemortException("Can't move backup files while store is open.");
-
- // Turn the current data into a .bak so we can take a look at it
- // manually if we want
- File primary = new File(storeDir, "version-0");
- DateFormat df = new SimpleDateFormat("MM-dd-yyyy");
- if(primary.exists())
- Utils.move(primary, new File(storeDir, "version-0." + df.format(new Date()) + ".bak"));
-
- shiftBackupsLeft(0);
- }
-
- private void shiftBackupsLeft(int beginShift) {
- File source = new File(storeDir, "version-" + Integer.toString(beginShift + 1));
- File dest = new File(storeDir, "version-" + Integer.toString(beginShift));
-
- // if the source file doesn't exist there is nothing to shift
- if(!source.exists())
- return;
-
- // rename the file
- source.renameTo(dest);
-
- // now rename any remaining files
- shiftBackupsLeft(beginShift + 1);
- }
-
- /**
- * Shift all store versions so that 0 becomes 1, 1 becomes 2, etc.
- */
- private void shiftBackupsRight() {
- if(isOpen)
- throw new VoldemortException("Can't move backup files while store is open.");
- shiftBackupsRight(0);
- }
-
- private void shiftBackupsRight(int beginShift) {
- if(isOpen)
- throw new VoldemortException("Can't move backup files while store is open.");
-
- File source = new File(storeDir, "version-" + Integer.toString(beginShift));
-
- // if the source file doesn't exist there is nothing to shift
- if(!source.exists())
- return;
-
- // if the dest file exists, it will need to be shifted too
- File dest = new File(storeDir, "version-" + Integer.toString(beginShift + 1));
- if(dest.exists())
- shiftBackupsRight(beginShift + 1);
-
- // okay finally do the rename
- source.renameTo(dest);
- }
-
public ClosableIterator<ByteArray> keys() {
throw new UnsupportedOperationException("Iteration is not supported for "
+ getClass().getName());
53 src/java/voldemort/utils/Utils.java
View
@@ -17,6 +17,7 @@
package voldemort.utils;
import java.io.File;
+import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
@@ -27,6 +28,9 @@
import voldemort.VoldemortException;
+import com.sun.jna.Library;
+import com.sun.jna.Native;
+
/**
* Helper functions FTW!
*
@@ -107,6 +111,32 @@ public static void rm(Iterable<File> files) {
}
/**
+ * Create a symbolic link to an existing file. Also deletes the existing
+ * symbolic link if it exists
+ *
+ * @param filePath Path of the file for whom to create the symbolic link
+ * @param symLinkPath Path of the symbolic link
+ */
+ public static void symlink(String filePath, String symLinkPath) {
+ File file = new File(filePath);
+ File symLink = new File(symLinkPath);
+ symLink.delete();
+
+ if(!file.exists())
+ throw new VoldemortException("File " + filePath + " does not exist");
+
+ Posix posix = (Posix) Native.loadLibrary("c", Posix.class);
+ int returnCode = posix.symlink(filePath, symLinkPath);
+ if(returnCode < 0)
+ throw new VoldemortException("Unable to create symbolic link for " + filePath);
+ }
+
+ public interface Posix extends Library {
+
+ public int symlink(String oldName, String newName);
+ }
+
+ /**
* Move the source file to the dest file name. If there is a file or
* directory at dest it will be overwritten. If the source file does not
* exist or cannot be copied and exception will be thrown exist
@@ -400,8 +430,29 @@ public static URI parseUri(String uri) {
}
@SuppressWarnings("unchecked")
- public static <T1,T2> T1 uncheckedCast(T2 t2) {
+ public static <T1, T2> T1 uncheckedCast(T2 t2) {
return (T1) t2;
}
+ /**
+ * Check if a file is a symbolic link or not
+ *
+ * @param symlinkFile
+ * @return true if File is symlink else false
+ */
+ public static boolean isSymLink(File symlinkFile) {
+ try {
+ File canonicalFile = null;
+ if(symlinkFile.getParent() != null) {
+ File canonicalDir = symlinkFile.getParentFile().getCanonicalFile();
+ canonicalFile = new File(canonicalDir, symlinkFile.getName());
+ } else {
+ canonicalFile = symlinkFile;
+ }
+ return !canonicalFile.getCanonicalFile().equals(canonicalFile.getAbsoluteFile());
+ } catch(IOException e) {
+ return false;
+ }
+ }
+
}
37 test/unit/voldemort/store/readonly/ReadOnlyStorageEngineTest.java
View
@@ -240,12 +240,13 @@ public void testOpenInvalidStoreFails(int indexBytes, int dataBytes, boolean sho
@Test
public void testSwap() throws IOException {
- createStoreFiles(dir, ReadOnlyUtils.INDEX_ENTRY_SIZE * 5, 4 * 5 * 10, 2);
+ File versionDir = new File(dir, "version-0");
+ createStoreFiles(versionDir, ReadOnlyUtils.INDEX_ENTRY_SIZE * 5, 4 * 5 * 10, 2);
ReadOnlyStorageEngine engine = new ReadOnlyStorageEngine("test", strategy, dir, 2);
assertVersionsExist(dir, 0);
// swap to a new version
- File newDir = TestUtils.createTempDir();
+ File newDir = new File(dir, "version-1");
createStoreFiles(newDir, 0, 0, 2);
engine.swapFiles(newDir.getAbsolutePath());
assertVersionsExist(dir, 0, 1);
@@ -255,17 +256,31 @@ public void testSwap() throws IOException {
}
@Test(expected = VoldemortException.class)
- public void testBadSwapThrows() throws IOException {
- createStoreFiles(dir, ReadOnlyUtils.INDEX_ENTRY_SIZE * 5, 4 * 5 * 10, 2);
+ public void testBadSwapNameThrows() throws IOException {
+ File versionDir = new File(dir, "version-0");
+ createStoreFiles(versionDir, ReadOnlyUtils.INDEX_ENTRY_SIZE * 5, 4 * 5 * 10, 2);
ReadOnlyStorageEngine engine = new ReadOnlyStorageEngine("test", strategy, dir, 2);
assertVersionsExist(dir, 0);
- // swap to a new bad version
+ // swap to a directory with bad name
File newDir = TestUtils.createTempDir();
createStoreFiles(newDir, 73, 1024, 2);
engine.swapFiles(newDir.getAbsolutePath());
}
+ @Test(expected = VoldemortException.class)
+ public void testBadSwapDataThrows() throws IOException {
+ File versionDir = new File(dir, "version-0");
+ createStoreFiles(versionDir, ReadOnlyUtils.INDEX_ENTRY_SIZE * 5, 4 * 5 * 10, 2);
+ ReadOnlyStorageEngine engine = new ReadOnlyStorageEngine("test", strategy, dir, 2);
+ assertVersionsExist(dir, 0);
+
+ // swap to a directory with bad name
+ File newDir = new File(dir, "version-1");
+ createStoreFiles(newDir, 73, 1024, 2);
+ engine.swapFiles(newDir.getAbsolutePath());
+ }
+
@Test
public void testTruncate() throws IOException {
createStoreFiles(dir, ReadOnlyUtils.INDEX_ENTRY_SIZE * 5, 4 * 5 * 10, 2);
@@ -276,11 +291,21 @@ public void testTruncate() throws IOException {
assertEquals(dir.exists(), false);
}
- private void assertVersionsExist(File dir, int... versions) {
+ private void assertVersionsExist(File dir, int... versions) throws IOException {
+ int max = 0;
for(int i = 0; i < versions.length; i++) {
File versionDir = new File(dir, "version-" + versions[i]);
+ if(versions[i] > max)
+ max = versions[i];
assertTrue("Could not find " + dir + "/version-" + versions[i], versionDir.exists());
}
+ // check latest symbolic link exists
+ File latest = new File(dir, "latest");
+ assertTrue(latest.exists());
+
+ // ...and points to max
+ assertTrue(latest.getCanonicalPath().contains("version-" + max));
+
// now check that the next higher version does not exist
File versionDir = new File(dir, "version-" + versions.length);
assertFalse("Found version directory that should not exist.", versionDir.exists());
44 test/unit/voldemort/utils/UtilsTest.java
View
@@ -17,6 +17,7 @@
package voldemort.utils;
import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -42,8 +43,6 @@ public void testMkDir() {
+ System.currentTimeMillis());
// Working case
- tempDir = new File(System.getProperty("java.io.tmpdir"), "temp"
- + System.currentTimeMillis());
Utils.mkdirs(tempDir);
assertTrue(tempDir.exists());
@@ -55,4 +54,45 @@ public void testMkDir() {
} catch(VoldemortException e) {}
}
+
+ public void testSymlink() throws IOException {
+ String tempParentDir = System.getProperty("java.io.tmpdir");
+ File tempDir = new File(tempParentDir, "temp" + (System.currentTimeMillis() + 1));
+ File tempSymLink = new File(tempParentDir, "link" + (System.currentTimeMillis() + 2));
+ File tempDir2 = new File(tempParentDir, "temp" + (System.currentTimeMillis() + 3));
+
+ // Test against non-existing directory
+ assertTrue(!tempDir.exists());
+ try {
+ Utils.symlink(tempDir.getAbsolutePath(), tempSymLink.getAbsolutePath());
+ fail("Symlink should have thrown an exception since directory did not exist");
+ } catch(VoldemortException e) {}
+
+ // Normal test
+ Utils.mkdirs(tempDir);
+ try {
+ Utils.symlink(tempDir.getAbsolutePath(), tempSymLink.getAbsolutePath());
+ } catch(VoldemortException e) {
+ fail("Test against non-existing symlink");
+ }
+
+ assertTrue(!Utils.isSymLink(tempDir));
+ assertTrue(Utils.isSymLink(tempSymLink));
+
+ // Test if existing sym-link can switch to new directory
+ Utils.mkdirs(tempDir2);
+ try {
+ Utils.symlink(tempDir2.getAbsolutePath(), tempSymLink.getAbsolutePath());
+ } catch(VoldemortException e) {
+ e.printStackTrace();
+ fail("Test against already existing symlink ");
+ }
+ assertTrue(Utils.isSymLink(tempSymLink));
+ // Check if it was not deleted with sym-link
+ assertTrue(tempDir.exists());
+
+ File dumbFile = new File(tempDir2, "dumbFile");
+ dumbFile.createNewFile();
+ assertEquals(1, tempSymLink.list().length);
+ }
}
Please sign in to comment.
Something went wrong with that request. Please try again.