Support Ceph as a notebook storage

apache · Sep 26, 2017 · b1ec41a · b1ec41a
1 parent 84e9bd9
commit b1ec41a
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 9 deletions.
diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template
@@ -138,6 +138,16 @@
 </property>
 -->
 
+<!-- Optional override to control which signature algorithm should be used to sign AWS requests -->
+<!-- Set this property to "S3SignerType" if your AWS S3 compatible APIs support only AWS Signature Version 2 such as Ceph. -->
+<!--
+<property>
+  <name>zeppelin.notebook.s3.signerOverride</name>
+  <value>S3SignerType</value>
+  <description>optional override to control which signature algorithm should be used to sign AWS requests</description>
+</property>
+-->
+
 <!-- If using Azure for storage use the following settings -->
 <!--
 <property>

diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -383,6 +383,10 @@ public boolean isS3ServerSideEncryption() {
     return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_S3_SSE);
   }
 
+  public String getS3SignerOverride() {
+    return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE);
+  }
+
   public String getMongoUri() {
     return getString(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_URI);
   }
@@ -649,6 +653,7 @@ public static enum ConfVars {
     ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID("zeppelin.notebook.s3.kmsKeyID", null),
     ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION("zeppelin.notebook.s3.kmsKeyRegion", null),
     ZEPPELIN_NOTEBOOK_S3_SSE("zeppelin.notebook.s3.sse", false),
+    ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE("zeppelin.notebook.s3.signerOverride", null),
     ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING("zeppelin.notebook.azure.connectionString", null),
     ZEPPELIN_NOTEBOOK_AZURE_SHARE("zeppelin.notebook.azure.share", "zeppelin"),
     ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"),

diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
@@ -42,6 +42,8 @@
 import org.slf4j.LoggerFactory;
 
 import com.amazonaws.AmazonClientException;
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.ClientConfigurationFactory;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
 import com.amazonaws.services.s3.AmazonS3;
@@ -94,33 +96,30 @@ public S3NotebookRepo(ZeppelinConfiguration conf) throws IOException {
 
     // always use the default provider chain
     AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
-    CryptoConfiguration cryptoConf = null;
+    CryptoConfiguration cryptoConf = new CryptoConfiguration();
     String keyRegion = conf.getS3KMSKeyRegion();
 
     if (StringUtils.isNotBlank(keyRegion)) {
-      cryptoConf = new CryptoConfiguration();
       cryptoConf.setAwsKmsRegion(Region.getRegion(Regions.fromName(keyRegion)));
     }
+
+    ClientConfiguration cliConf = createClientConfiguration();
 
     // see if we should be encrypting data in S3
     String kmsKeyID = conf.getS3KMSKeyID();
     if (kmsKeyID != null) {
       // use the AWS KMS to encrypt data
       KMSEncryptionMaterialsProvider emp = new KMSEncryptionMaterialsProvider(kmsKeyID);
-      if (cryptoConf != null) {
-        this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cryptoConf);
-      } else {
-        this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp);
-      }
+      this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cliConf, cryptoConf);
     }
     else if (conf.getS3EncryptionMaterialsProviderClass() != null) {
       // use a custom encryption materials provider class
       EncryptionMaterialsProvider emp = createCustomProvider(conf);
-      this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp);
+      this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cliConf, cryptoConf);
     }
     else {
       // regular S3
-      this.s3client = new AmazonS3Client(credentialsProvider);
+      this.s3client = new AmazonS3Client(credentialsProvider, cliConf);
     }
 
     // set S3 endpoint to use
@@ -154,6 +153,22 @@ private EncryptionMaterialsProvider createCustomProvider(ZeppelinConfiguration c
     return emp;
   }
 
+  /**
+   * Create AWS client configuration and return it.
+   * @return AWS client configuration
+   */
+  private ClientConfiguration createClientConfiguration() {
+    ClientConfigurationFactory configFactory = new ClientConfigurationFactory();
+    ClientConfiguration config = configFactory.getConfig();
+
+    String s3SignerOverride = conf.getS3SignerOverride();
+    if (StringUtils.isNotBlank(s3SignerOverride)) {
+      config.setSignerOverride(s3SignerOverride);
+    }
+
+    return config;
+  }
+
   @Override
   public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
     List<NoteInfo> infos = new LinkedList<>();