diff --git a/contrib/hadoop-store-builder/lib/hadoop-core-1.0.2-p1.jar b/contrib/hadoop-store-builder/lib/hadoop-core-1.0.4-p2-rc2.jar similarity index 82% rename from contrib/hadoop-store-builder/lib/hadoop-core-1.0.2-p1.jar rename to contrib/hadoop-store-builder/lib/hadoop-core-1.0.4-p2-rc2.jar index 861c03ca0f..c7a9027b1a 100644 Binary files a/contrib/hadoop-store-builder/lib/hadoop-core-1.0.2-p1.jar and b/contrib/hadoop-store-builder/lib/hadoop-core-1.0.4-p2-rc2.jar differ diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriter.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriter.java index 4391202d5d..2506cf1c51 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriter.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriter.java @@ -323,12 +323,18 @@ public void close() throws IOException { Path checkSumIndexFile = new Path(nodeDir, fileNamePrefix + ".index.checksum"); Path checkSumValueFile = new Path(nodeDir, fileNamePrefix + ".data.checksum"); + if(outputFs.exists(checkSumIndexFile)) { + outputFs.delete(checkSumIndexFile); + } FSDataOutputStream output = outputFs.create(checkSumIndexFile); outputFs.setPermission(checkSumIndexFile, new FsPermission(HadoopStoreBuilder.HADOOP_FILE_PERMISSION)); output.write(this.checkSumDigestIndex.getCheckSum()); output.close(); + if(outputFs.exists(checkSumValueFile)) { + outputFs.delete(checkSumValueFile); + } output = outputFs.create(checkSumValueFile); outputFs.setPermission(checkSumValueFile, new FsPermission(HadoopStoreBuilder.HADOOP_FILE_PERMISSION)); @@ -346,8 +352,15 @@ public void close() throws IOException { Path valueFile = new Path(nodeDir, fileNamePrefix + ".data"); logger.info("Moving " + this.taskIndexFileName + " to " + indexFile); + if(outputFs.exists(indexFile)) { + outputFs.delete(indexFile); + } outputFs.rename(taskIndexFileName, indexFile); + logger.info("Moving " + this.taskValueFileName + " to " + valueFile); + if(outputFs.exists(valueFile)) { + outputFs.delete(valueFile); + } outputFs.rename(this.taskValueFileName, valueFile); } diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriterPerBucket.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriterPerBucket.java index 69744448f7..ddc50857b2 100644 --- a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriterPerBucket.java +++ b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriterPerBucket.java @@ -301,12 +301,18 @@ public void close() throws IOException { Path checkSumIndexFile = new Path(nodeDir, chunkFileName + ".index.checksum"); Path checkSumValueFile = new Path(nodeDir, chunkFileName + ".data.checksum"); + if(outputFs.exists(checkSumIndexFile)) { + outputFs.delete(checkSumIndexFile); + } FSDataOutputStream output = outputFs.create(checkSumIndexFile); outputFs.setPermission(checkSumIndexFile, new FsPermission(HadoopStoreBuilder.HADOOP_FILE_PERMISSION)); output.write(this.checkSumDigestIndex[chunkId].getCheckSum()); output.close(); + if(outputFs.exists(checkSumValueFile)) { + outputFs.delete(checkSumValueFile); + } output = outputFs.create(checkSumValueFile); outputFs.setPermission(checkSumValueFile, new FsPermission(HadoopStoreBuilder.HADOOP_FILE_PERMISSION)); @@ -324,8 +330,15 @@ public void close() throws IOException { Path valueFile = new Path(nodeDir, chunkFileName + ".data"); logger.info("Moving " + this.taskIndexFileName[chunkId] + " to " + indexFile); + if(outputFs.exists(indexFile)) { + outputFs.delete(indexFile); + } fs.rename(taskIndexFileName[chunkId], indexFile); + logger.info("Moving " + this.taskValueFileName[chunkId] + " to " + valueFile); + if(outputFs.exists(valueFile)) { + outputFs.delete(valueFile); + } fs.rename(this.taskValueFileName[chunkId], valueFile); }