From a0f1b10f5e3f11c3824fa47bfedc3995e45c302d Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Mon, 11 Sep 2017 15:30:35 -0400 Subject: [PATCH 1/3] Refactor the hdfs rest endpoint to support writing with permissions. permissions are passes as three strings ( not required ) that represent symbolic permsions see FsAction. For example "rw-", "rwx", "r-x" --- .../rest/controller/HdfsController.java | 11 ++++-- .../metron/rest/service/HdfsService.java | 2 +- .../rest/service/impl/GrokServiceImpl.java | 2 +- .../rest/service/impl/HdfsServiceImpl.java | 39 ++++++++++++++----- .../impl/HdfsServiceImplExceptionTest.java | 12 +++++- .../service/impl/HdfsServiceImplTest.java | 16 +++++++- 6 files changed, 66 insertions(+), 16 deletions(-) diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java index b8957a9388..80b5942f25 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java @@ -67,9 +67,14 @@ ResponseEntity read(@ApiParam(name = "path", value = "Path to HDFS file" @ApiOperation(value = "Writes contents to an HDFS file. Warning: this will overwrite the contents of a file if it already exists.") @ApiResponse(message = "Contents were written", code = 200) @RequestMapping(method = RequestMethod.POST) - ResponseEntity write(@ApiParam(name="path", value="Path to HDFS file", required=true) @RequestParam String path, - @ApiParam(name="contents", value="File contents", required=true) @RequestBody String contents) throws RestException { - hdfsService.write(new Path(path), contents.getBytes(UTF_8)); + ResponseEntity write( + @ApiParam(name = "path", value = "Path to HDFS file", required = true) @RequestParam String path, + @ApiParam(name = "contents", value = "File contents", required = true) @RequestBody String contents, + @ApiParam(name = "userMode", value = "requested user permissions") @RequestParam(required = false, defaultValue = "") String userMode, + @ApiParam(name = "groupMode", value = "requested group permissions") @RequestParam(required = false, defaultValue = "") String groupMode, + @ApiParam(name = "otherMode", value = "requested other permissions") @RequestParam(required = false, defaultValue = "") String otherMode + ) throws RestException { + hdfsService.write(new Path(path), contents.getBytes(UTF_8), userMode, groupMode, otherMode); return new ResponseEntity<>(HttpStatus.OK); } diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java index 58dbf9bfdd..2748e6657b 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java @@ -26,7 +26,7 @@ public interface HdfsService { String read(Path path) throws RestException; - void write(Path path, byte[] contents) throws RestException; + void write(Path path, byte[] contents,String userMode, String groupMode, String otherMode) throws RestException; List list(Path path) throws RestException; diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java index e185b53bbc..4a3963509d 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java @@ -91,7 +91,7 @@ public Path saveTemporary(String statement, String name) throws RestException { if (statement != null) { Path path = getTemporaryGrokRootPath(); hdfsService.mkdirs(path); - hdfsService.write(new Path(path, name), statement.getBytes(Charset.forName("utf-8"))); + hdfsService.write(new Path(path, name), statement.getBytes(Charset.forName("utf-8")),null,null,null); return path; } else { throw new RestException("A grokStatement must be provided"); diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java index a9ae8ebd3f..d637ecfc53 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java @@ -17,10 +17,13 @@ */ package org.apache.metron.rest.service.impl; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; import org.apache.metron.rest.RestException; import org.apache.metron.rest.service.HdfsService; @@ -68,17 +71,35 @@ public String read(Path path) throws RestException { return new String(byteArrayOutputStream.toByteArray(), UTF_8); } - @Override - public void write(Path path, byte[] contents) throws RestException { - FSDataOutputStream fsDataOutputStream; - try { - fsDataOutputStream = FileSystem.get(configuration).create(path, true); - fsDataOutputStream.write(contents); - fsDataOutputStream.close(); - } catch (IOException e) { - throw new RestException(e); + @Override + public void write(Path path, byte[] contents, String userMode, String groupMode, String otherMode) + throws RestException { + FSDataOutputStream fsDataOutputStream; + try { + FsPermission permission = null; + boolean setPermissions = false; + if(StringUtils.isNotEmpty(userMode) && StringUtils.isNotEmpty(groupMode) && StringUtils.isNotEmpty(otherMode)) { + // invalid actions will return null + FsAction userAction = FsAction.getFsAction(userMode); + FsAction groupAction = FsAction.getFsAction(groupMode); + FsAction otherAction = FsAction.getFsAction(otherMode); + if(userAction == null || groupAction == null || otherAction == null){ + throw new RestException(String.format("Invalid permission set: user[%s] " + + "group[%s] other[%s]", userAction, groupAction, otherAction)); + } + permission = new FsPermission(userAction, groupAction, otherAction); + setPermissions = true; + } + fsDataOutputStream = FileSystem.get(configuration).create(path, true); + fsDataOutputStream.write(contents); + fsDataOutputStream.close(); + if(setPermissions) { + FileSystem.get(configuration).setPermission(path, permission); } + } catch (IOException e) { + throw new RestException(e); } + } @Override public boolean delete(Path path, boolean recursive) throws RestException { diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java index d11f5a4b19..728285981d 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java @@ -85,7 +85,17 @@ public void writeShouldWrapExceptionInRestException() throws Exception { when(FileSystem.get(configuration)).thenReturn(fileSystem); when(fileSystem.create(new Path(testDir), true)).thenThrow(new IOException()); - hdfsService.write(new Path(testDir), "contents".getBytes(UTF_8)); + hdfsService.write(new Path(testDir), "contents".getBytes(UTF_8),null, null,null); + } + + @Test + public void writeShouldThrowIfInvalidPermissions() throws Exception { + exception.expect(RestException.class); + + FileSystem fileSystem = mock(FileSystem.class); + when(FileSystem.get(configuration)).thenReturn(fileSystem); + + hdfsService.write(new Path(testDir,"test"),"oops".getBytes(UTF_8), "foo", "r-x","r--"); } @Test diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java index 005a427f8f..280be63273 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java @@ -19,7 +19,10 @@ import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.permission.FsAction; import org.apache.metron.rest.service.HdfsService; import org.junit.After; import org.junit.Before; @@ -84,11 +87,22 @@ public void readShouldProperlyReadContents() throws Exception { @Test public void writeShouldProperlyWriteContents() throws Exception { String contents = "contents"; - hdfsService.write(new Path(testDir, "writeTest.txt"), contents.getBytes(UTF_8)); + hdfsService.write(new Path(testDir, "writeTest.txt"), contents.getBytes(UTF_8),null,null,null); assertEquals("contents", FileUtils.readFileToString(new File(testDir, "writeTest.txt"))); } + @Test + public void writeShouldProperlyWriteContentsWithPermissions() throws Exception { + String contents = "contents"; + Path thePath = new Path(testDir,"writeTest2.txt"); + hdfsService.write(thePath, contents.getBytes(UTF_8),"rw-","r-x","r--"); + + assertEquals("contents", FileUtils.readFileToString(new File(testDir, "writeTest2.txt"))); + assertEquals(FileSystem.get(configuration).listStatus(thePath)[0].getPermission().toShort(), + new FsPermission(FsAction.READ_WRITE,FsAction.READ_EXECUTE,FsAction.READ).toShort()); + } + @Test public void deleteShouldProperlyDeleteFile() throws Exception { String contents = "contents"; From d2d76aca88a4f34d4ff31fa4446400ecb2be3613 Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Mon, 11 Sep 2017 22:34:13 -0400 Subject: [PATCH 2/3] readme update --- metron-interface/metron-rest/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index ba75ee12ff..eb6cd15570 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -300,6 +300,9 @@ Request and Response objects are JSON formatted. The JSON schemas are available * Input: * path - Path to HDFS file * contents - File contents + * userMode - [optional] symbolic permission string for user portion of the permissions to be set on the file written. For example 'rwx' or read, write, execute. The symbol '-' is used to exclude that permission such as 'rw-' for read, write, no execute + * groupMode - [optional] symbolic permission string for group portion of the permissions to be set on the file written. For example 'rwx' or read, write, execute. The symbol '-' is used to exclude that permission such as 'rw-' for read, write, no execute + * otherMode - [optional] symbolic permission string for other portion of the permissions to be set on the file written. For example 'rwx' or read, write, execute. The symbol '-' is used to exclude that permission such as 'rw-' for read, write, no execute * Returns: * 200 - Contents were written From 87f0a5c08a5e221201fabc40b4872ee32ce08d94 Mon Sep 17 00:00:00 2001 From: Otto Fowler Date: Tue, 12 Sep 2017 10:29:52 -0400 Subject: [PATCH 3/3] update readme per review --- metron-interface/metron-rest/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index eb6cd15570..27e04a32bd 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -296,7 +296,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available * 200 - JSON results ### `POST /api/v1/hdfs` - * Description: Writes contents to an HDFS file. Warning: this will overwrite the contents of a file if it already exists. + * Description: Writes contents to an HDFS file. Warning: this will overwrite the contents of a file if it already exists. Permissions must be set for all three groups if they are to be set. If any are missing, the default permissions will be used, and if any are invalid an exception will be thrown. * Input: * path - Path to HDFS file * contents - File contents