Skip to content
This repository
Browse code

Doing authentication in a synchronized block for the Hdfs fetcher, se…

…tting correct permission for the hadoop files
  • Loading branch information...
commit e6b24c6ea1114cbbe2512cefe7eb8cbe5d372d8d 1 parent 7693d6b
authored
10  contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriter.java
@@ -26,6 +26,7 @@
26 26
 import org.apache.hadoop.fs.FSDataOutputStream;
27 27
 import org.apache.hadoop.fs.FileSystem;
28 28
 import org.apache.hadoop.fs.Path;
  29
+import org.apache.hadoop.fs.permission.FsPermission;
29 30
 import org.apache.hadoop.io.BytesWritable;
30 31
 import org.apache.hadoop.mapred.FileOutputFormat;
31 32
 import org.apache.hadoop.mapred.JobConf;
@@ -146,7 +147,12 @@ public void conf(JobConf job) {
146 147
                 this.fs = this.taskIndexFileName.getFileSystem(job);
147 148
 
148 149
             this.indexFileStream = fs.create(this.taskIndexFileName);
  150
+            fs.setPermission(this.taskIndexFileName, new FsPermission("755"));
  151
+            logger.info("Setting permission to 755 for " + this.taskIndexFileName);
  152
+
149 153
             this.valueFileStream = fs.create(this.taskValueFileName);
  154
+            fs.setPermission(this.taskValueFileName, new FsPermission("755"));
  155
+            logger.info("Setting permission to 755 for " + this.taskValueFileName);
150 156
 
151 157
             logger.info("Opening " + this.taskIndexFileName + " and " + this.taskValueFileName
152 158
                         + " for writing.");
@@ -304,6 +310,8 @@ public void close() throws IOException {
304 310
         // Create output directory, if it doesn't exist
305 311
         FileSystem outputFs = nodeDir.getFileSystem(this.conf);
306 312
         outputFs.mkdirs(nodeDir);
  313
+        outputFs.setPermission(nodeDir, new FsPermission("755"));
  314
+        logger.info("Setting permission to 755 for " + nodeDir);
307 315
 
308 316
         // Write the checksum and output files
309 317
         if(this.checkSumType != CheckSumType.NONE) {
@@ -313,10 +321,12 @@ public void close() throws IOException {
313 321
                 Path checkSumValueFile = new Path(nodeDir, fileNamePrefix + ".data.checksum");
314 322
 
315 323
                 FSDataOutputStream output = outputFs.create(checkSumIndexFile);
  324
+                outputFs.setPermission(checkSumIndexFile, new FsPermission("755"));
316 325
                 output.write(this.checkSumDigestIndex.getCheckSum());
317 326
                 output.close();
318 327
 
319 328
                 output = outputFs.create(checkSumValueFile);
  329
+                outputFs.setPermission(checkSumValueFile, new FsPermission("755"));
320 330
                 output.write(this.checkSumDigestValue.getCheckSum());
321 331
                 output.close();
322 332
             } else {
10  contrib/hadoop-store-builder/src/java/voldemort/store/readonly/disk/HadoopStoreWriterPerBucket.java
@@ -26,6 +26,7 @@
26 26
 import org.apache.hadoop.fs.FSDataOutputStream;
27 27
 import org.apache.hadoop.fs.FileSystem;
28 28
 import org.apache.hadoop.fs.Path;
  29
+import org.apache.hadoop.fs.permission.FsPermission;
29 30
 import org.apache.hadoop.io.BytesWritable;
30 31
 import org.apache.hadoop.mapred.FileOutputFormat;
31 32
 import org.apache.hadoop.mapred.JobConf;
@@ -117,7 +118,12 @@ public void conf(JobConf job) {
117 118
                     this.fs = this.taskIndexFileName[chunkId].getFileSystem(job);
118 119
 
119 120
                 this.indexFileStream[chunkId] = fs.create(this.taskIndexFileName[chunkId]);
  121
+                fs.setPermission(this.taskIndexFileName[chunkId], new FsPermission("755"));
  122
+                logger.info("Setting permission to 755 for " + this.taskIndexFileName[chunkId]);
  123
+
120 124
                 this.valueFileStream[chunkId] = fs.create(this.taskValueFileName[chunkId]);
  125
+                fs.setPermission(this.taskValueFileName[chunkId], new FsPermission("755"));
  126
+                logger.info("Setting permission to 755 for " + this.taskValueFileName[chunkId]);
121 127
 
122 128
                 logger.info("Opening " + this.taskIndexFileName[chunkId] + " and "
123 129
                             + this.taskValueFileName[chunkId] + " for writing.");
@@ -278,6 +284,8 @@ public void close() throws IOException {
278 284
         // Create output directory, if it doesn't exist
279 285
         FileSystem outputFs = nodeDir.getFileSystem(this.conf);
280 286
         outputFs.mkdirs(nodeDir);
  287
+        outputFs.setPermission(nodeDir, new FsPermission("755"));
  288
+        logger.info("Setting permission to 755 for " + nodeDir);
281 289
 
282 290
         // Write the checksum and output files
283 291
         for(int chunkId = 0; chunkId < getNumChunks(); chunkId++) {
@@ -291,10 +299,12 @@ public void close() throws IOException {
291 299
                     Path checkSumValueFile = new Path(nodeDir, chunkFileName + ".data.checksum");
292 300
 
293 301
                     FSDataOutputStream output = outputFs.create(checkSumIndexFile);
  302
+                    outputFs.setPermission(checkSumIndexFile, new FsPermission("755"));
294 303
                     output.write(this.checkSumDigestIndex[chunkId].getCheckSum());
295 304
                     output.close();
296 305
 
297 306
                     output = outputFs.create(checkSumValueFile);
  307
+                    outputFs.setPermission(checkSumValueFile, new FsPermission("755"));
298 308
                     output.write(this.checkSumDigestValue[chunkId].getCheckSum());
299 309
                     output.close();
300 310
                 } else {
45  contrib/hadoop-store-builder/src/java/voldemort/store/readonly/fetcher/HdfsFetcher.java
@@ -158,26 +158,29 @@ public File fetch(String sourceFileUrl, String destinationFile) throws IOExcepti
158 158
              * 
159 159
              * Otherwise get the default filesystem object.
160 160
              */
161  
-            if(this.keytabLocation.length() > 0) {
162  
-                logger.debug("keytab path = " + keytabLocation + " and proxy user = " + proxyUser);
163  
-                UserGroupInformation.loginUserFromKeytab(proxyUser, keytabLocation);
164  
-                logger.debug("I've logged in and am now Doasing as "
165  
-                             + UserGroupInformation.getCurrentUser().getUserName());
166  
-                try {
167  
-                    fs = UserGroupInformation.getCurrentUser()
168  
-                                             .doAs(new PrivilegedExceptionAction<FileSystem>() {
169  
-
170  
-                                                 public FileSystem run() throws Exception {
171  
-                                                     FileSystem fs = path.getFileSystem(config);
172  
-                                                     return fs;
173  
-                                                 }
174  
-                                             });
175  
-                } catch(InterruptedException e) {
176  
-                    logger.error(e.getMessage());
177  
-                    return null;
  161
+            synchronized(this) {
  162
+                if(this.keytabLocation.length() > 0) {
  163
+                    logger.info("keytab path = " + keytabLocation + " and proxy user = "
  164
+                                + proxyUser);
  165
+                    UserGroupInformation.loginUserFromKeytab(proxyUser, keytabLocation);
  166
+                    logger.info("I've logged in and am now Doasing as "
  167
+                                + UserGroupInformation.getCurrentUser().getUserName());
  168
+                    try {
  169
+                        fs = UserGroupInformation.getCurrentUser()
  170
+                                                 .doAs(new PrivilegedExceptionAction<FileSystem>() {
  171
+
  172
+                                                     public FileSystem run() throws Exception {
  173
+                                                         FileSystem fs = path.getFileSystem(config);
  174
+                                                         return fs;
  175
+                                                     }
  176
+                                                 });
  177
+                    } catch(InterruptedException e) {
  178
+                        logger.error(e.getMessage());
  179
+                        return null;
  180
+                    }
  181
+                } else {
  182
+                    fs = path.getFileSystem(config);
178 183
                 }
179  
-            } else {
180  
-                fs = path.getFileSystem(config);
181 184
             }
182 185
 
183 186
             CopyStats stats = new CopyStats(sourceFileUrl, sizeOfPath(fs, path));
@@ -316,7 +319,7 @@ private void copyFileWithCheckSum(FileSystem fs,
316 319
                                       File dest,
317 320
                                       CopyStats stats,
318 321
                                       CheckSum fileCheckSumGenerator) throws IOException {
319  
-        logger.info("Starting copy of " + source + " to " + dest);
  322
+        logger.debug("Starting copy of " + source + " to " + dest);
320 323
         FSDataInputStream input = null;
321 324
         OutputStream output = null;
322 325
         for(int attempt = 0; attempt < NUM_RETRIES; attempt++) {
@@ -378,7 +381,7 @@ private void copyFileWithCheckSum(FileSystem fs,
378 381
                 }
379 382
 
380 383
             }
381  
-
  384
+            logger.debug("Completed copy of " + source + " to " + dest);
382 385
         }
383 386
     }
384 387
 
8  contrib/hadoop-store-builder/src/java/voldemort/store/readonly/mr/HadoopStoreBuilder.java
@@ -34,6 +34,7 @@
34 34
 import org.apache.hadoop.fs.FileSystem;
35 35
 import org.apache.hadoop.fs.Path;
36 36
 import org.apache.hadoop.fs.PathFilter;
  37
+import org.apache.hadoop.fs.permission.FsPermission;
37 38
 import org.apache.hadoop.io.BytesWritable;
38 39
 import org.apache.hadoop.mapred.Counters;
39 40
 import org.apache.hadoop.mapred.FileInputFormat;
@@ -470,6 +471,8 @@ public void build() {
470 471
                     logger.info("No data generated for node " + node.getId()
471 472
                                 + ". Generating empty folder");
472 473
                     outputFs.mkdirs(nodePath); // Create empty folder
  474
+                    outputFs.setPermission(nodePath, new FsPermission("755"));
  475
+                    logger.info("Setting permission to 755 for " + nodePath);
473 476
                 }
474 477
 
475 478
                 if(checkSumType != CheckSumType.NONE) {
@@ -518,7 +521,10 @@ public boolean accept(Path arg0) {
518 521
                 }
519 522
 
520 523
                 // Write metadata
521  
-                FSDataOutputStream metadataStream = outputFs.create(new Path(nodePath, ".metadata"));
  524
+                Path metadataPath = new Path(nodePath, ".metadata");
  525
+                FSDataOutputStream metadataStream = outputFs.create(metadataPath);
  526
+                outputFs.setPermission(metadataPath, new FsPermission("755"));
  527
+                logger.info("Setting permission to 755 for " + metadataPath);
522 528
                 metadataStream.write(metadata.toJsonString().getBytes());
523 529
                 metadataStream.flush();
524 530
                 metadataStream.close();

0 notes on commit e6b24c6

Please sign in to comment.
Something went wrong with that request. Please try again.