Fix dataset permissions

trustedanalytics · Aug 1, 2016 · ef153cb · ef153cb
1 parent a0e3940
commit ef153cb
Show file tree

Hide file tree

Showing 8 changed files with 60 additions and 28 deletions.
diff --git a/pom.xml b/pom.xml
@@ -2,9 +2,7 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <artifactId>hdfs-broker</artifactId>
-
-    <version>0.10.11</version>
-
+    <version>0.10.12</version>
     <parent>
         <groupId>org.trustedanalytics</groupId>
         <artifactId>spring-project-template</artifactId>
@@ -14,7 +12,7 @@
     <properties>
         <brokerVersion>2.6.0</brokerVersion>
         <brokerTestVersion>2.6.0</brokerTestVersion>
-        <brokerStoreVersion>0.6.2</brokerStoreVersion>
+        <brokerStoreVersion>0.6.3</brokerStoreVersion>
         <start-class>org.trustedanalytics.servicebroker.hdfs.config.Application</start-class>
         <jacoco-measurement-instructions>0.4700</jacoco-measurement-instructions>
         <jacoco-measurement-lines>0.6400</jacoco-measurement-lines>

diff --git a/src/main/java/org/trustedanalytics/servicebroker/hdfs/plans/HdfsPlanCreateUserDirectory.java b/src/main/java/org/trustedanalytics/servicebroker/hdfs/plans/HdfsPlanCreateUserDirectory.java
@@ -63,7 +63,9 @@ public void provision(ServiceInstance serviceInstance, Optional<Map<String, Obje
     String password = RandomStringUtils.randomAlphanumeric(32);
 
     UUID sysUser = groupMappingOperations.createSysUser(orgId, instanceId, password);
-    hdfsOperations.provisionDirectory(instanceId, orgId, sysUser);
+    String path = hdfsOperations.provisionDirectory(instanceId, orgId, sysUser);
+    hdfsOperations.addSystemUsersGroupAcl(path, orgId);
+
     credentialsStore.save(ImmutableMap.of(USER, instanceId.toString(), PASSWORD, password),
         instanceId);
   }

diff --git a/...danalytics/servicebroker/hdfs/plans/provisioning/HdfsDirectoryProvisioningOperations.java b/...danalytics/servicebroker/hdfs/plans/provisioning/HdfsDirectoryProvisioningOperations.java
@@ -15,13 +15,13 @@
  */
 package org.trustedanalytics.servicebroker.hdfs.plans.provisioning;
 
-import java.util.UUID;
-
 import org.cloudfoundry.community.servicebroker.exception.ServiceBrokerException;
 
+import java.util.UUID;
+
 public interface HdfsDirectoryProvisioningOperations {
-  void provisionDirectory(UUID instanceId, UUID orgId) throws ServiceBrokerException;
-  void provisionDirectory(UUID instanceId, UUID orgId, UUID owner) throws ServiceBrokerException;
+  String provisionDirectory(UUID instanceId, UUID orgId) throws ServiceBrokerException;
+  String provisionDirectory(UUID instanceId, UUID orgId, UUID owner) throws ServiceBrokerException;
   void addSystemUsersGroupAcl(String path, UUID orgId) throws ServiceBrokerException;
   void addHiveUserGroupAcl(String path, UUID orgId) throws ServiceBrokerException;
 }
diff --git a/...va/org/trustedanalytics/servicebroker/hdfs/plans/provisioning/HdfsProvisioningClient.java b/...va/org/trustedanalytics/servicebroker/hdfs/plans/provisioning/HdfsProvisioningClient.java
@@ -16,7 +16,9 @@
 package org.trustedanalytics.servicebroker.hdfs.plans.provisioning;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.UUID;
+import java.util.stream.Collectors;
 
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.*;
@@ -47,23 +49,25 @@ public HdfsProvisioningClient(HdfsClient hdfsClient, HdfsClient encryptedHdfsCli
   }
 
   @Override
-  public void provisionDirectory(UUID instanceId, UUID orgId) throws ServiceBrokerException {
+  public String provisionDirectory(UUID instanceId, UUID orgId) throws ServiceBrokerException {
     try {
       String path = HdfsPathTemplateUtils.fill(userspacePathTemplate, instanceId, orgId);
       hdfsClient.createDir(path);
       hdfsClient.setPermission(path, FS_PERMISSION);
-      addHiveUserGroupAcl(path,orgId);
+      addHiveUserGroupAcl(path, orgId);
+      return path;
     } catch (IOException e) {
       throw new ServiceBrokerException("Unable to provision directory for: " + instanceId, e);
     }
   }
 
   @Override
-  public void provisionDirectory(UUID instanceId, UUID orgId, UUID owner) throws ServiceBrokerException {
+  public String provisionDirectory(UUID instanceId, UUID orgId, UUID owner) throws ServiceBrokerException {
     this.provisionDirectory(instanceId, orgId);
     try {
       String path = HdfsPathTemplateUtils.fill(userspacePathTemplate, instanceId, orgId);
       superUserHdfsClient.setOwner(path, owner.toString(), orgId.toString());
+      return path;
     } catch (IOException e) {
       throw new ServiceBrokerException("Unable to provision directory for: " + instanceId, e);
     }
@@ -80,8 +84,8 @@ public void addSystemUsersGroupAcl(String path, UUID orgId) throws ServiceBroker
       AclEntry systemDefaultUserAcl = builder.setScope(AclEntryScope.DEFAULT).build();
       AclEntry systemUserAcl = builder.setScope(AclEntryScope.ACCESS).build();
 
-      superUserHdfsClient.addAclEntry(path, systemUserAcl);
-      superUserHdfsClient.addAclEntry(path, systemDefaultUserAcl);
+      setAclRecursively(path, systemUserAcl);
+      setAclRecursively(path, systemDefaultUserAcl);
     } catch (IOException e) {
       throw new ServiceBrokerException("Unable to add system users groups ACL for path: " + path, e);
     }
@@ -98,8 +102,8 @@ public void addHiveUserGroupAcl(String path, UUID orgId) throws ServiceBrokerExc
       AclEntry hiveDefaultUserAcl = builder.setScope(AclEntryScope.DEFAULT).build();
       AclEntry hiveUserAcl = builder.setScope(AclEntryScope.ACCESS).build();
 
-      superUserHdfsClient.addAclEntry(path, hiveUserAcl);
-      superUserHdfsClient.addAclEntry(path, hiveDefaultUserAcl);
+      setAclRecursively(path, hiveUserAcl);
+      setAclRecursively(path, hiveDefaultUserAcl);
     } catch (IOException e) {
       throw new ServiceBrokerException("Unable to add system users groups ACL for path: " + path, e);
     }
@@ -115,4 +119,13 @@ public void createEncryptedZone(UUID instanceId, UUID orgId) throws ServiceBroke
           "Unable to provision encrypted directory for: " + instanceId, e);
     }
   }
+
+  private void setAclRecursively(String path, AclEntry acl) throws IOException {
+    superUserHdfsClient.addAclEntry(path, acl);
+
+    for(String file:superUserHdfsClient.listFiles(path, true)) {
+      if(superUserHdfsClient.isDirectory(file) || !acl.getScope().equals(AclEntryScope.DEFAULT))
+        superUserHdfsClient.addAclEntry(file , acl);
+    }
+  }
 }
diff --git a/...t/java/org/trustedanalytics/servicebroker/hdfs/plans/HdfsPlanCreateUserDirectoryTest.java b/...t/java/org/trustedanalytics/servicebroker/hdfs/plans/HdfsPlanCreateUserDirectoryTest.java
@@ -88,10 +88,17 @@ public void provision_createNewUserWithDirectory_hdfsDirectoryCreated() throws E
     when(groupMappingOperations.createSysUser(any(UUID.class), any(UUID.class), anyString()))
         .thenReturn(userId);
     planUnderTest.provision(serviceInstance, Optional.of(ImmutableMap.of()));
-    
+
     verify(encryptedHdfsClient).addAclEntry("/org/"+ serviceInstance.getOrganizationGuid()+"/brokers/userspace/"+serviceInstance.getServiceInstanceId(), TestUtil.hiveUserAcl());
     verify(encryptedHdfsClient).addAclEntry("/org/"+ serviceInstance.getOrganizationGuid()+"/brokers/userspace/"+serviceInstance.getServiceInstanceId(), TestUtil.hiveDefaultUserAcl());
 
+    verify(encryptedHdfsClient).addAclEntry("/org/"+ serviceInstance.getOrganizationGuid()+"/brokers/userspace/"+serviceInstance.getServiceInstanceId(),
+        TestUtil.userAcl(orgId.toString() + "_sys"));
+    verify(encryptedHdfsClient).addAclEntry("/org/"+ serviceInstance.getOrganizationGuid()+"/brokers/userspace/"+serviceInstance.getServiceInstanceId(),
+        TestUtil.defaultUserAcl(orgId.toString() + "_sys"));
+
+    verify(encryptedHdfsClient, times(4)).listFiles("/org/"+ serviceInstance.getOrganizationGuid()+"/brokers/userspace/"+serviceInstance.getServiceInstanceId(), true);
+
     verify(groupMappingOperations).createSysUser(eq(orgId), eq(instanceId), anyString());
     verify(hdfsClient).createDir(path);
     verify(hdfsClient).setPermission(path, FS_PERMISSION);

diff --git a/src/test/java/org/trustedanalytics/servicebroker/hdfs/plans/HdfsPlanEncryptedTest.java b/src/test/java/org/trustedanalytics/servicebroker/hdfs/plans/HdfsPlanEncryptedTest.java
@@ -76,6 +76,9 @@ public void provision_templateWithOrgAndInstanceVariables_replaceVariablesWithVa
     verify(encryptedHdfsClient).createKeyAndEncryptedZone(serviceInstance.getServiceInstanceId(),
         new Path(getDirectoryPathToProvision(serviceInstance)));
     verify(hdfsClient).setPermission(getDirectoryPathToProvision(serviceInstance), FS_PERMISSION);
+
+    verify(encryptedHdfsClient, times(2)).listFiles("/org/"+ serviceInstance.getOrganizationGuid()+"/brokers/userspace/"+serviceInstance.getServiceInstanceId(), true);
+
     verifyNoMoreInteractions(hdfsClient, encryptedHdfsClient);
   }
 

diff --git a/src/test/java/org/trustedanalytics/servicebroker/hdfs/plans/HdfsPlanSharedTest.java b/src/test/java/org/trustedanalytics/servicebroker/hdfs/plans/HdfsPlanSharedTest.java
@@ -21,6 +21,7 @@
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Matchers.startsWith;
 import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.trustedanalytics.servicebroker.test.cloudfoundry.CfModelsFactory.getServiceInstance;
@@ -76,6 +77,8 @@ public void provision_templateWithOrgAndInstanceVariables_replaceVariablesWithVa
     verify(encryptedHdfsClient).addAclEntry("/org/"+ serviceInstance.getOrganizationGuid()+"/brokers/userspace/"+serviceInstance.getServiceInstanceId(), TestUtil.hiveUserAcl());
     verify(encryptedHdfsClient).addAclEntry("/org/"+ serviceInstance.getOrganizationGuid()+"/brokers/userspace/"+serviceInstance.getServiceInstanceId(), TestUtil.hiveDefaultUserAcl());
 
+    verify(encryptedHdfsClient, times(2)).listFiles("/org/"+ serviceInstance.getOrganizationGuid()+"/brokers/userspace/"+serviceInstance.getServiceInstanceId(), true);
+
     verifyNoMoreInteractions(hdfsClient, encryptedHdfsClient);
   }
 

diff --git a/src/test/java/org/trustedanalytics/servicebroker/hdfs/util/TestUtil.java b/src/test/java/org/trustedanalytics/servicebroker/hdfs/util/TestUtil.java
@@ -25,19 +25,25 @@ public class TestUtil {
     public static final String HIVE_USER = "hive";
 
     public static AclEntry hiveDefaultUserAcl(){
-        AclEntry.Builder builder = new AclEntry.Builder()
-                .setType(AclEntryType.GROUP)
-                .setPermission(FsAction.ALL)
-                .setName(TestUtil.HIVE_USER);
-
-        return builder.setScope(AclEntryScope.DEFAULT).build();
+        return defaultUserAcl(TestUtil.HIVE_USER);
     }
 
     public static AclEntry hiveUserAcl(){
-        AclEntry.Builder builder = new AclEntry.Builder()
-                .setType(AclEntryType.GROUP)
-                .setPermission(FsAction.ALL)
-                .setName(TestUtil.HIVE_USER);
-        return builder.setScope(AclEntryScope.ACCESS).build();
+        return userAcl(TestUtil.HIVE_USER);
+    }
+
+    public static AclEntry defaultUserAcl(String user){
+        return build(user).setScope(AclEntryScope.DEFAULT).build();
+    }
+
+    public static AclEntry userAcl(String user){
+        return build(user).setScope(AclEntryScope.ACCESS).build();
+    }
+
+    private static AclEntry.Builder build(String user){
+        return new AclEntry.Builder()
+            .setType(AclEntryType.GROUP)
+            .setPermission(FsAction.ALL)
+            .setName(user);
     }
 }