feat(s3): Initial aws/s3 support. (#69)

kayenta-aws/kayenta-aws.gradle

@@ -0,0 +1,11 @@
+dependencies {
+  compile project(":kayenta-core")
+
+  // compile spinnaker.dependency('bootWeb')
+  compile "org.springframework.boot:spring-boot-starter-web:$springBootVersion"
+
+  // compile spinnaker.dependency('lombok')
+  compile "org.projectlombok:lombok:1.16.10"
+
+  compile "com.amazonaws:aws-java-sdk-s3:1.11.179"
+}

kayenta-aws/src/main/java/com/netflix/kayenta/aws/config/AwsConfiguration.java

@@ -0,0 +1,133 @@
+/*
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.aws.config;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.regions.Region;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.S3ClientOptions;
+import com.netflix.kayenta.aws.security.AwsCredentials;
+import com.netflix.kayenta.aws.security.AwsNamedAccountCredentials;
+import com.netflix.kayenta.security.AccountCredentials;
+import com.netflix.kayenta.security.AccountCredentialsRepository;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.util.CollectionUtils;
+import org.springframework.util.StringUtils;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Optional;
+
+@Configuration
+@EnableConfigurationProperties
+@ConditionalOnProperty("kayenta.aws.enabled")
+@ComponentScan({"com.netflix.kayenta.aws"})
+@Slf4j
+public class AwsConfiguration {
+
+  @Bean
+  @ConfigurationProperties("kayenta.aws")
+  AwsConfigurationProperties awsConfigurationProperties() {
+    return new AwsConfigurationProperties();
+  }
+
+  @Bean
+  boolean registerAwsCredentials(AwsConfigurationProperties awsConfigurationProperties,
+                                 AWSCredentialsProvider awsCredentialsProvider,
+                                 AccountCredentialsRepository accountCredentialsRepository) throws IOException {
+    for (AwsManagedAccount awsManagedAccount : awsConfigurationProperties.getAccounts()) {
+      String name = awsManagedAccount.getName();
+      List<AccountCredentials.Type> supportedTypes = awsManagedAccount.getSupportedTypes();
+
+      log.info("Registering AWS account {} with supported types {}.", name, supportedTypes);
+
+      ClientConfiguration clientConfiguration = new ClientConfiguration();
+
+      if (awsManagedAccount.getProxyProtocol() != null) {
+        if (awsManagedAccount.getProxyProtocol().equalsIgnoreCase("HTTPS")) {
+          clientConfiguration.setProtocol(Protocol.HTTPS);
+        } else {
+          clientConfiguration.setProtocol(Protocol.HTTP);
+        }
+        Optional.ofNullable(awsManagedAccount.getProxyHost())
+          .ifPresent(clientConfiguration::setProxyHost);
+        Optional.ofNullable(awsManagedAccount.getProxyPort())
+          .map(Integer::parseInt)
+          .ifPresent(clientConfiguration::setProxyPort);
+      }
+
+      AmazonS3Client client = new AmazonS3Client(awsCredentialsProvider, clientConfiguration);
+
+      if (awsManagedAccount.getEndpoint() != null) {
+        client.setEndpoint(awsManagedAccount.getEndpoint());
+        client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
+      } else {
+        Optional.ofNullable(awsManagedAccount.getRegion())
+          .map(Regions::fromName)
+          .map(Region::getRegion)
+          .ifPresent(client::setRegion);
+      }
+
+      try {
+        AwsCredentials awsCredentials = new AwsCredentials();
+        AwsNamedAccountCredentials.AwsNamedAccountCredentialsBuilder awsNamedAccountCredentialsBuilder =
+          AwsNamedAccountCredentials
+            .builder()
+            .name(name)
+            .credentials(awsCredentials);
+
+        if (!CollectionUtils.isEmpty(supportedTypes)) {
+          if (supportedTypes.contains(AccountCredentials.Type.OBJECT_STORE)) {
+            String bucket = awsManagedAccount.getBucket();
+            String rootFolder = awsManagedAccount.getRootFolder();
+
+            if (StringUtils.isEmpty(bucket)) {
+              throw new IllegalArgumentException("AWS/S3 account " + name + " is required to specify a bucket.");
+            }
+
+            if (StringUtils.isEmpty(rootFolder)) {
+              throw new IllegalArgumentException("AWS/S3 account " + name + " is required to specify a rootFolder.");
+            }
+
+            awsNamedAccountCredentialsBuilder.bucket(bucket);
+            awsNamedAccountCredentialsBuilder.region(awsManagedAccount.getRegion());
+            awsNamedAccountCredentialsBuilder.rootFolder(rootFolder);
+            awsNamedAccountCredentialsBuilder.amazonS3(client);
+          }
+
+          awsNamedAccountCredentialsBuilder.supportedTypes(supportedTypes);
+        }
+
+        AwsNamedAccountCredentials awsNamedAccountCredentials = awsNamedAccountCredentialsBuilder.build();
+        accountCredentialsRepository.save(name, awsNamedAccountCredentials);
+      } catch (Throwable t) {
+        log.error("Could not load AWS account " + name + ".", t);
+      }
+    }
+
+    return true;
+  }
+}

kayenta-aws/src/main/java/com/netflix/kayenta/aws/config/AwsConfigurationProperties.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2017 Google, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.aws.config;
+
+import lombok.Getter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class AwsConfigurationProperties {
+
+  @Getter
+  private List<AwsManagedAccount> accounts = new ArrayList<>();
+}

kayenta-aws/src/main/java/com/netflix/kayenta/aws/config/AwsManagedAccount.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2017 Google, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.aws.config;
+
+import com.netflix.kayenta.security.AccountCredentials;
+import lombok.Data;
+
+import javax.validation.constraints.NotNull;
+import java.util.List;
+
+@Data
+public class AwsManagedAccount {
+
+  @NotNull
+  private String name;
+
+  private String bucket;
+  private String region;
+  private String rootFolder;
+  private String endpoint;
+  private String proxyHost;
+  private String proxyPort;
+  private String proxyProtocol;
+
+  private List<AccountCredentials.Type> supportedTypes;
+}

kayenta-aws/src/main/java/com/netflix/kayenta/aws/security/AwsCredentials.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2017 Google, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.aws.security;
+
+import lombok.ToString;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Optional;
+
+@ToString
+@Slf4j
+public class AwsCredentials {
+
+  private static String applicationVersion =
+    Optional.ofNullable(AwsCredentials.class.getPackage().getImplementationVersion()).orElse("Unknown");
+}

kayenta-aws/src/main/java/com/netflix/kayenta/aws/security/AwsNamedAccountCredentials.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2017 Google, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.aws.security;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.netflix.kayenta.security.AccountCredentials;
+import lombok.Builder;
+import lombok.Data;
+import lombok.Singular;
+
+import javax.validation.constraints.NotNull;
+import java.util.List;
+
+@Builder
+@Data
+public class AwsNamedAccountCredentials implements AccountCredentials<AwsCredentials> {
+
+  @NotNull
+  private String name;
+
+  @NotNull
+  @Singular
+  private List<Type> supportedTypes;
+
+  @NotNull
+  private AwsCredentials credentials;
+
+  private String bucket;
+  private String region;
+  private String rootFolder;
+
+  @Override
+  public String getType() {
+    return "aws";
+  }
+
+  @JsonIgnore
+  private AmazonS3 amazonS3;
+}

kayenta-gcs/src/main/java/com/netflix/kayenta/gcs/config/GcsConfiguration.java

@@ -38,7 +38,7 @@ public class GcsConfiguration {
 
   @Bean
   @DependsOn({"registerGoogleCredentials"})
-  public StorageService storageService(AccountCredentialsRepository accountCredentialsRepository) {
+  public GcsStorageService gcsStorageService(AccountCredentialsRepository accountCredentialsRepository) {
     GcsStorageService.GcsStorageServiceBuilder gcsStorageServiceBuilder = GcsStorageService.builder();
 
     accountCredentialsRepository