Add support for the URL-based configbin service (#93)

Implement ConfigBin config storage

kayenta-objectstore-configbin/kayenta-objectstore-configbin.gradle

@@ -0,0 +1,12 @@
+dependencies {
+  compile project(":kayenta-core")
+
+  // compile spinnaker.dependency('bootWeb')
+  compile "org.springframework.boot:spring-boot-starter-web:$springBootVersion"
+
+  // compile spinnaker.dependency("korkSwagger")
+  compile "com.netflix.spinnaker.kork:kork-swagger:$korkVersion"
+
+  // compile spinnaker.dependency('lombok')
+  compile "org.projectlombok:lombok:1.16.10"
+}

kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/config/ConfigBinConfiguration.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.configbin.config;
+
+import com.netflix.kayenta.configbin.security.ConfigBinAccountCredentials;
+import com.netflix.kayenta.configbin.security.ConfigBinNamedAccountCredentials;
+import com.netflix.kayenta.configbin.service.ConfigBinRemoteService;
+import com.netflix.kayenta.configbin.storage.ConfigBinStorageService;
+import com.netflix.kayenta.retrofit.config.RemoteService;
+import com.netflix.kayenta.retrofit.config.RetrofitClientFactory;
+import com.netflix.kayenta.security.AccountCredentials;
+import com.netflix.kayenta.security.AccountCredentialsRepository;
+import com.squareup.okhttp.OkHttpClient;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.util.CollectionUtils;
+
+import java.util.List;
+
+@Configuration
+@EnableConfigurationProperties
+@ConditionalOnProperty("kayenta.configbin.enabled")
+@ComponentScan({"com.netflix.kayenta.configbin"})
+@Slf4j
+public class ConfigBinConfiguration {
+
+  @Bean
+  @ConfigurationProperties("kayenta.configbin")
+  ConfigBinConfigurationProperties configbinConfigurationProperties() {
+    return new ConfigBinConfigurationProperties();
+  }
+
+  @Bean
+  ConfigBinStorageService configBinStorageService(ConfigBinResponseConverter configBinConverter,
+                                                  ConfigBinConfigurationProperties configBinConfigurationProperties,
+                                                  RetrofitClientFactory retrofitClientFactory,
+                                                  OkHttpClient okHttpClient,
+                                                  AccountCredentialsRepository accountCredentialsRepository) {
+    log.debug("Created a ConfigBin StorageService");
+    ConfigBinStorageService.ConfigBinStorageServiceBuilder configbinStorageServiceBuilder = ConfigBinStorageService.builder();
+
+    for (ConfigBinManagedAccount configBinManagedAccount : configBinConfigurationProperties.getAccounts()) {
+      String name = configBinManagedAccount.getName();
+      String ownerApp = configBinManagedAccount.getOwnerApp();
+      String configType = configBinManagedAccount.getConfigType();
+      RemoteService endpoint = configBinManagedAccount.getEndpoint();
+      List<AccountCredentials.Type> supportedTypes = configBinManagedAccount.getSupportedTypes();
+
+      log.info("Registering ConfigBin account {} with supported types {}.", name, supportedTypes);
+
+      ConfigBinAccountCredentials configbinAccountCredentials = ConfigBinAccountCredentials.builder().build();
+      ConfigBinNamedAccountCredentials.ConfigBinNamedAccountCredentialsBuilder configBinNamedAccountCredentialsBuilder =
+        ConfigBinNamedAccountCredentials.builder()
+          .name(name)
+          .ownerApp(ownerApp)
+          .configType(configType)
+          .endpoint(endpoint)
+          .credentials(configbinAccountCredentials);
+
+      if (!CollectionUtils.isEmpty(supportedTypes)) {
+        if (supportedTypes.contains(AccountCredentials.Type.CONFIGURATION_STORE)) {
+          ConfigBinRemoteService configBinRemoteService = retrofitClientFactory.createClient(ConfigBinRemoteService.class,
+                                                                                             configBinConverter,
+                                                                                             configBinManagedAccount.getEndpoint(),
+                                                                                             okHttpClient);
+          configBinNamedAccountCredentialsBuilder.remoteService(configBinRemoteService);
+        }
+        configBinNamedAccountCredentialsBuilder.supportedTypes(supportedTypes);
+      }
+
+      ConfigBinNamedAccountCredentials configbinNamedAccountCredentials = configBinNamedAccountCredentialsBuilder.build();
+      accountCredentialsRepository.save(name, configbinNamedAccountCredentials);
+      configbinStorageServiceBuilder.accountName(name);
+    }
+
+    ConfigBinStorageService configbinStorageService = configbinStorageServiceBuilder.build();
+
+    log.info("Populated ConfigBinStorageService with {} ConfigBin accounts.", configbinStorageService.getAccountNames().size());
+
+    return configbinStorageService;
+  }
+}

kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/config/ConfigBinConfigurationProperties.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.configbin.config;
+
+import com.netflix.kayenta.retrofit.config.RemoteService;
+import lombok.Getter;
+import lombok.Setter;
+
+import javax.validation.constraints.NotNull;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ConfigBinConfigurationProperties {
+
+  @Getter
+  private List<ConfigBinManagedAccount> accounts = new ArrayList<>();
+}

kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/config/ConfigBinManagedAccount.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.configbin.config;
+
+import com.netflix.kayenta.retrofit.config.RemoteService;
+import com.netflix.kayenta.security.AccountCredentials;
+import lombok.Data;
+
+import javax.validation.constraints.NotNull;
+import java.util.List;
+
+@Data
+public class ConfigBinManagedAccount {
+
+  @NotNull
+  private String name;
+
+  @NotNull
+  private RemoteService endpoint;
+
+  @NotNull
+  private String ownerApp;
+
+  @NotNull
+  private String configType;
+
+  private List<AccountCredentials.Type> supportedTypes;
+}

kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/config/ConfigBinResponseConverter.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.configbin.config;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.squareup.okhttp.RequestBody;
+import lombok.extern.slf4j.Slf4j;
+import okio.Buffer;
+import org.springframework.stereotype.Component;
+import retrofit.converter.ConversionException;
+import retrofit.converter.Converter;
+import retrofit.mime.TypedInput;
+import retrofit.mime.TypedOutput;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.lang.reflect.Type;
+import java.nio.charset.StandardCharsets;
+
+import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
+import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
+
+@Component
+@Slf4j
+public class ConfigBinResponseConverter implements Converter {
+
+  private static final ObjectMapper objectMapper = new ObjectMapper()
+    .setSerializationInclusion(NON_NULL)
+    .disable(FAIL_ON_UNKNOWN_PROPERTIES);
+
+  @Override
+  public String fromBody(TypedInput body, Type type) throws ConversionException {
+    try {
+      ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+      int nRead;
+      byte[] data = new byte[1024];
+      InputStream inputStream = body.in();
+      while ((nRead = inputStream.read(data, 0, data.length)) != -1) {
+        buffer.write(data, 0, nRead);
+      }
+      buffer.flush();
+      byte[] byteArray = buffer.toByteArray();
+      return new String(byteArray, StandardCharsets.UTF_8);
+    } catch (IOException e) {
+      log.error("Unable to read response body or convert it to a UTF-8 string", e);
+      return null;
+    }
+  }
+
+  @Override
+  public TypedOutput toBody(Object object) {
+    RequestBody requestBody = (RequestBody)object;
+    return new StringTypedOutput(requestBody);
+  }
+
+  private static class StringTypedOutput implements TypedOutput {
+    private final RequestBody string;
+
+    StringTypedOutput(RequestBody s) { this.string = s; }
+
+    @Override public String fileName() { return null; }
+
+    @Override public String mimeType() { return "application/json; charset=UTF-8"; }
+
+    @Override public long length() {
+      try {
+        return string.contentLength();
+      } catch (IOException e) {
+        log.error("Unable to determine response body length", e);
+        return 0;
+      }
+    }
+
+    @Override public void writeTo(OutputStream out) throws IOException {
+      Buffer buffer = new Buffer();
+      string.writeTo(buffer);
+      buffer.writeTo(out);
+    }
+  }
+}

kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/security/ConfigBinAccountCredentials.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.configbin.security;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Optional;
+
+@Builder
+@Data
+@Slf4j
+public class ConfigBinAccountCredentials {
+  // ConfigBin does not currently require authentication, so we do not need anything here.
+  private static String applicationVersion =
+    Optional.ofNullable(ConfigBinAccountCredentials.class.getPackage().getImplementationVersion()).orElse("Unknown");
+}

kayenta-objectstore-configbin/src/main/java/com/netflix/kayenta/configbin/security/ConfigBinNamedAccountCredentials.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.configbin.security;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.netflix.kayenta.configbin.service.ConfigBinRemoteService;
+import com.netflix.kayenta.retrofit.config.RemoteService;
+import com.netflix.kayenta.security.AccountCredentials;
+import lombok.*;
+
+import javax.validation.constraints.NotNull;
+import java.util.List;
+
+@Builder
+@Data
+public class ConfigBinNamedAccountCredentials implements AccountCredentials<ConfigBinAccountCredentials> {
+
+  @NotNull
+  private String name;
+
+  @NotNull
+  @Singular
+  private List<Type> supportedTypes;
+
+  @NotNull
+  private ConfigBinAccountCredentials credentials;
+
+  @NotNull
+  @Getter
+  @Setter
+  private RemoteService endpoint;
+
+  @NotNull
+  private String ownerApp;
+
+  @NotNull
+  private String configType;
+
+  @Override
+  public String getType() { return "configbin"; }
+
+  @JsonIgnore
+  private ConfigBinRemoteService remoteService;
+}