From d0a0b03e87961c15164af3a1c97c8c059a2aeda3 Mon Sep 17 00:00:00 2001 From: Corey Huang Date: Thu, 6 Aug 2015 23:46:14 +0000 Subject: [PATCH 1/3] Allow instance profile authentication with S3 --- .../org/apache/zeppelin/conf/Credentials.java | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/Credentials.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/Credentials.java index 87248a6fe23..bb3fca2bd46 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/Credentials.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/Credentials.java @@ -19,7 +19,8 @@ package org.apache.zeppelin.conf; import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.AWSCredentialsProviderChain; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; /** * @@ -27,11 +28,26 @@ * */ public class Credentials { - static String aws_access_key_id = System.getenv("AWS_ACCESS_KEY_ID"); - static String aws_secret_access_key = System.getenv("AWS_SECRET_ACCESS_KEY"); - - private static AWSCredentials credentials = new BasicAWSCredentials(aws_access_key_id, - aws_secret_access_key); + + // Use a credential provider chain so that instance profiles can be utilized + // on an EC2 instance. The order of locations where credentials are searched + // is documented here + // + // http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/ + // auth/DefaultAWSCredentialsProviderChain.html + // + // In summary, the order is: + // + // 1. Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY + // 2. Java System Properties - aws.accessKeyId and aws.secretKey + // 3. Credential profiles file at the default location (~/.aws/credentials) + // shared by all AWS SDKs and the AWS CLI + // 4. Instance profile credentials delivered through the Amazon EC2 metadata service + + private static AWSCredentialsProviderChain credProvider = + new DefaultAWSCredentialsProviderChain(); + + private static AWSCredentials credentials = credProvider.getCredentials(); public AWSCredentials getCredentials() { return credentials; @@ -41,3 +57,4 @@ public static void setCredentials(AWSCredentials credentials) { Credentials.credentials = credentials; } } + From 2fb5de04933bb5effd758c0525fa4dcb8ca031f6 Mon Sep 17 00:00:00 2001 From: Corey Huang Date: Fri, 7 Aug 2015 00:54:44 +0000 Subject: [PATCH 2/3] Fix comment error --- .../src/main/java/org/apache/zeppelin/conf/Credentials.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/Credentials.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/Credentials.java index bb3fca2bd46..24b794281be 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/Credentials.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/Credentials.java @@ -38,7 +38,7 @@ public class Credentials { // // In summary, the order is: // - // 1. Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY + // 1. Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY // 2. Java System Properties - aws.accessKeyId and aws.secretKey // 3. Credential profiles file at the default location (~/.aws/credentials) // shared by all AWS SDKs and the AWS CLI From 237eab2d02b00a0d99ccae73e6af282168393e08 Mon Sep 17 00:00:00 2001 From: Corey Huang Date: Fri, 7 Aug 2015 18:14:04 +0000 Subject: [PATCH 3/3] Use credential provider directly to avoid AWS token expiration with instance profiles --- .../org/apache/zeppelin/conf/Credentials.java | 60 ------------------- .../notebook/repo/S3NotebookRepo.java | 26 ++++++-- 2 files changed, 21 insertions(+), 65 deletions(-) delete mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/Credentials.java diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/Credentials.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/Credentials.java deleted file mode 100644 index 24b794281be..00000000000 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/Credentials.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.apache.zeppelin.conf; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProviderChain; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; - -/** - * - * @author vgmartinez - * - */ -public class Credentials { - - // Use a credential provider chain so that instance profiles can be utilized - // on an EC2 instance. The order of locations where credentials are searched - // is documented here - // - // http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/ - // auth/DefaultAWSCredentialsProviderChain.html - // - // In summary, the order is: - // - // 1. Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - // 2. Java System Properties - aws.accessKeyId and aws.secretKey - // 3. Credential profiles file at the default location (~/.aws/credentials) - // shared by all AWS SDKs and the AWS CLI - // 4. Instance profile credentials delivered through the Amazon EC2 metadata service - - private static AWSCredentialsProviderChain credProvider = - new DefaultAWSCredentialsProviderChain(); - - private static AWSCredentials credentials = credProvider.getCredentials(); - - public AWSCredentials getCredentials() { - return credentials; - } - - public static void setCredentials(AWSCredentials credentials) { - Credentials.credentials = credentials; - } -} - diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java index 0b90262f50c..bb9e5d1571d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java @@ -27,7 +27,6 @@ import java.util.List; import org.apache.commons.io.IOUtils; -import org.apache.zeppelin.conf.Credentials; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.notebook.Note; @@ -39,6 +38,8 @@ import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; +import com.amazonaws.auth.AWSCredentialsProviderChain; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.GetObjectRequest; @@ -55,14 +56,29 @@ * @author vgmartinez * */ -public class S3NotebookRepo implements NotebookRepo{ +public class S3NotebookRepo implements NotebookRepo { Logger logger = LoggerFactory.getLogger(S3NotebookRepo.class); - Credentials aws = new Credentials(); + + // Use a credential provider chain so that instance profiles can be utilized + // on an EC2 instance. The order of locations where credentials are searched + // is documented here + // + // http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/ + // auth/DefaultAWSCredentialsProviderChain.html + // + // In summary, the order is: + // + // 1. Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY + // 2. Java System Properties - aws.accessKeyId and aws.secretKey + // 3. Credential profiles file at the default location (~/.aws/credentials) + // shared by all AWS SDKs and the AWS CLI + // 4. Instance profile credentials delivered through the Amazon EC2 metadata service + private AmazonS3 s3client = new AmazonS3Client(new DefaultAWSCredentialsProviderChain()); + private static String bucketName = ""; - String user = ""; + private String user = ""; - AmazonS3 s3client = new AmazonS3Client(aws.getCredentials()); private ZeppelinConfiguration conf;