feat(artifacts) Implement S3 artifacts (#2468)

clouddriver-artifacts/clouddriver-artifacts.gradle

@@ -1,5 +1,8 @@
 dependencies {
   compile project(":clouddriver-core")
+
+  spinnaker.group('amazon')
+
   compile spinnaker.dependency("frigga")
   compile spinnaker.dependency("bootActuator")
   compile spinnaker.dependency("bootWeb")

clouddriver-artifacts/src/main/java/com/netflix/spinnaker/clouddriver/artifacts/ArtifactConfiguration.java

@@ -21,6 +21,7 @@
 import com.netflix.spinnaker.clouddriver.artifacts.gcs.GcsArtifactConfiguration;
 import com.netflix.spinnaker.clouddriver.artifacts.github.GitHubArtifactConfiguration;
 import com.netflix.spinnaker.clouddriver.artifacts.http.HttpArtifactConfiguration;
+import com.netflix.spinnaker.clouddriver.artifacts.s3.S3ArtifactConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
@@ -38,7 +39,8 @@
   EmbeddedArtifactConfiguration.class,
   GcsArtifactConfiguration.class,
   GitHubArtifactConfiguration.class,
-  HttpArtifactConfiguration.class
+  HttpArtifactConfiguration.class,
+  S3ArtifactConfiguration.class
 })
 public class ArtifactConfiguration {
   @Bean

clouddriver-artifacts/src/main/java/com/netflix/spinnaker/clouddriver/artifacts/s3/S3ArtifactAccount.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2018 Datadog, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.artifacts.s3;
+
+import com.netflix.spinnaker.clouddriver.artifacts.config.ArtifactAccount;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+@EqualsAndHashCode(callSuper = true)
+@Data
+public class S3ArtifactAccount extends ArtifactAccount
+{
+  private String name;
+}

clouddriver-artifacts/src/main/java/com/netflix/spinnaker/clouddriver/artifacts/s3/S3ArtifactConfiguration.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2018 Datadog, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.artifacts.s3;
+
+import com.netflix.spinnaker.clouddriver.artifacts.ArtifactCredentialsRepository;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.config.ConfigurableBeanFactory;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Scope;
+import org.springframework.scheduling.annotation.EnableScheduling;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+@Configuration
+@ConditionalOnProperty("artifacts.s3.enabled")
+@EnableScheduling
+@Slf4j
+public class S3ArtifactConfiguration {
+  @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
+  @Bean
+  @ConfigurationProperties("artifacts.s3")
+  S3ArtifactProviderProperties s3ArtifactProviderProperties() { return new S3ArtifactProviderProperties(); }
+
+  @Autowired
+  S3ArtifactProviderProperties s3ArtifactProviderProperties;
+
+  @Autowired
+  ArtifactCredentialsRepository artifactCredentialsRepository;
+
+  @Bean
+  List<? extends S3ArtifactCredentials> s3ArtifactCredentials() {
+    return s3ArtifactProviderProperties.getAccounts()
+      .stream()
+      .map(a -> {
+        try {
+          S3ArtifactCredentials c = new S3ArtifactCredentials(a);
+          artifactCredentialsRepository.save(c);
+          return c;
+        } catch (IllegalArgumentException e) {
+          log.warn("Failure instantiating s3 artifact account {}: ", a, e);
+          return null;
+        }
+      })
+      .filter(Objects::nonNull)
+      .collect(Collectors.toList());
+  }
+}

clouddriver-artifacts/src/main/java/com/netflix/spinnaker/clouddriver/artifacts/s3/S3ArtifactCredentials.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2018 Datadog, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.artifacts.s3;
+
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.S3Object;
+
+import com.netflix.spinnaker.clouddriver.artifacts.config.ArtifactCredentials;
+import com.netflix.spinnaker.kork.artifacts.model.Artifact;
+import groovy.util.logging.Slf4j;
+import lombok.Data;
+
+import java.io.InputStream;
+
+@Slf4j
+@Data
+public class S3ArtifactCredentials implements ArtifactCredentials {
+  private final String name;
+
+  public S3ArtifactCredentials(S3ArtifactAccount account) throws IllegalArgumentException {
+    name = account.getName();
+  }
+
+  @Override
+  public InputStream download(Artifact artifact) throws IllegalArgumentException {
+    String reference = artifact.getReference();
+    if (reference.startsWith("s3://")) {
+      reference = reference.substring("s3://".length());
+    }
+
+    int slash = reference.indexOf("/");
+    if (slash <= 0) {
+      throw new IllegalArgumentException("S3 references must be of the format s3://<bucket>/<file-path>, got: " + artifact);
+    }
+    String bucketName = reference.substring(0, slash);
+    String path = reference.substring(slash + 1);
+    S3Object s3obj = AmazonS3ClientBuilder.defaultClient().getObject(bucketName, path);
+    return s3obj.getObjectContent();
+  }
+
+  @Override
+  public boolean handlesType(String type) {
+    return type.equals("s3/object");
+  }
+}

clouddriver-artifacts/src/main/java/com/netflix/spinnaker/clouddriver/artifacts/s3/S3ArtifactProviderProperties.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2018 Datadog, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.artifacts.s3;
+
+import com.netflix.spinnaker.clouddriver.artifacts.config.ArtifactProvider;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@EqualsAndHashCode(callSuper = true)
+@Data
+public class S3ArtifactProviderProperties extends ArtifactProvider<S3ArtifactAccount> {
+  private boolean enabled;
+  private List<S3ArtifactAccount> accounts = new ArrayList<>();
+}