Added the ability to delete old checksum files in the Build and Push …

…reducer. Also updated the hadoop core library to version 1.0.4-p2-rc2 containing the fix to the User ID command usage.
voldemort · Jan 11, 2013 · 5c34750 · 5c34750
1 parent db493a1
commit 5c34750
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 0 deletions.
diff --git a/...tore-builder/lib/hadoop-core-1.0.2-p1.jar → ...-builder/lib/hadoop-core-1.0.4-p2-rc2.jar b/...tore-builder/lib/hadoop-core-1.0.2-p1.jar → ...-builder/lib/hadoop-core-1.0.4-p2-rc2.jar
diff --git a/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriter.java b/contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriter.java
@@ -323,12 +323,18 @@ public void close() throws IOException {
                 Path checkSumIndexFile = new Path(nodeDir, fileNamePrefix + ".index.checksum");
                 Path checkSumValueFile = new Path(nodeDir, fileNamePrefix + ".data.checksum");
 
+                if(outputFs.exists(checkSumIndexFile)) {
+                    outputFs.delete(checkSumIndexFile);
+                }
                 FSDataOutputStream output = outputFs.create(checkSumIndexFile);
                 outputFs.setPermission(checkSumIndexFile,
                                        new FsPermission(HadoopStoreBuilder.HADOOP_FILE_PERMISSION));
                 output.write(this.checkSumDigestIndex.getCheckSum());
                 output.close();
 
+                if(outputFs.exists(checkSumValueFile)) {
+                    outputFs.delete(checkSumValueFile);
+                }
                 output = outputFs.create(checkSumValueFile);
                 outputFs.setPermission(checkSumValueFile,
                                        new FsPermission(HadoopStoreBuilder.HADOOP_FILE_PERMISSION));
@@ -346,8 +352,15 @@ public void close() throws IOException {
         Path valueFile = new Path(nodeDir, fileNamePrefix + ".data");
 
         logger.info("Moving " + this.taskIndexFileName + " to " + indexFile);
+        if(outputFs.exists(indexFile)) {
+            outputFs.delete(indexFile);
+        }
         outputFs.rename(taskIndexFileName, indexFile);
+
         logger.info("Moving " + this.taskValueFileName + " to " + valueFile);
+        if(outputFs.exists(valueFile)) {
+            outputFs.delete(valueFile);
+        }
         outputFs.rename(this.taskValueFileName, valueFile);
     }
 

diff --git a/...doop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriterPerBucket.java b/...doop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriterPerBucket.java
@@ -301,12 +301,18 @@ public void close() throws IOException {
                     Path checkSumIndexFile = new Path(nodeDir, chunkFileName + ".index.checksum");
                     Path checkSumValueFile = new Path(nodeDir, chunkFileName + ".data.checksum");
 
+                    if(outputFs.exists(checkSumIndexFile)) {
+                        outputFs.delete(checkSumIndexFile);
+                    }
                     FSDataOutputStream output = outputFs.create(checkSumIndexFile);
                     outputFs.setPermission(checkSumIndexFile,
                                            new FsPermission(HadoopStoreBuilder.HADOOP_FILE_PERMISSION));
                     output.write(this.checkSumDigestIndex[chunkId].getCheckSum());
                     output.close();
 
+                    if(outputFs.exists(checkSumValueFile)) {
+                        outputFs.delete(checkSumValueFile);
+                    }
                     output = outputFs.create(checkSumValueFile);
                     outputFs.setPermission(checkSumValueFile,
                                            new FsPermission(HadoopStoreBuilder.HADOOP_FILE_PERMISSION));
@@ -324,8 +330,15 @@ public void close() throws IOException {
             Path valueFile = new Path(nodeDir, chunkFileName + ".data");
 
             logger.info("Moving " + this.taskIndexFileName[chunkId] + " to " + indexFile);
+            if(outputFs.exists(indexFile)) {
+                outputFs.delete(indexFile);
+            }
             fs.rename(taskIndexFileName[chunkId], indexFile);
+
             logger.info("Moving " + this.taskValueFileName[chunkId] + " to " + valueFile);
+            if(outputFs.exists(valueFile)) {
+                outputFs.delete(valueFile);
+            }
             fs.rename(this.taskValueFileName[chunkId], valueFile);
 
         }