add datadog integration (#248)

json-formats.md

@@ -168,6 +168,41 @@ Atlas, Stackdriver and Prometheus are used.
   }
 }
 ```
+```JSON
+{
+  "name": "MySampleDatadogCanaryConfig",
+  "description": "Example Kayenta Configuration using Datadog",
+  "configVersion": "1.0",
+  "applications": [
+    "myapp"
+  ],
+  "judge": {
+    "name": "dredd-v1.0",
+    "judgeConfigurations": { }
+  },
+  "metrics": [
+    {
+      "name": "CPU",
+      "query": {
+        "type": "datadog",
+        "metricName": "avg:system.cpu.user"
+      },
+      "groups": ["system"],
+      "analysisConfigurations": { },
+      "scopeName": "default"
+    }
+  ],
+  "classifier": {
+    "groupWeights": {
+      "system": 100.0
+    },
+    "scoreThresholds": {
+      "pass": 95.0,
+      "marginal": 75.0
+    }
+  }
+}
+```
 ## Canary Data Archival Format
 
 This format is used to store the results from a specific canary run.

kayenta-core/src/main/java/com/netflix/kayenta/canary/CanaryMetricSetQueryConfig.java

@@ -20,12 +20,14 @@
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.netflix.kayenta.canary.providers.AtlasCanaryMetricSetQueryConfig;
+import com.netflix.kayenta.canary.providers.DatadogCanaryMetricSetQueryConfig;
 import com.netflix.kayenta.canary.providers.PrometheusCanaryMetricSetQueryConfig;
 import com.netflix.kayenta.canary.providers.StackdriverCanaryMetricSetQueryConfig;
 import lombok.NonNull;
 
 @JsonTypeInfo(use= JsonTypeInfo.Id.NAME, include= JsonTypeInfo.As.PROPERTY, property = "type")
 @JsonSubTypes({@JsonSubTypes.Type(value = AtlasCanaryMetricSetQueryConfig.class, name = "atlas"),
+               @JsonSubTypes.Type(value = DatadogCanaryMetricSetQueryConfig.class, name = "datadog"),
                @JsonSubTypes.Type(value = PrometheusCanaryMetricSetQueryConfig.class, name = "prometheus"),
                @JsonSubTypes.Type(value = StackdriverCanaryMetricSetQueryConfig.class, name = "stackdriver")})
 @JsonInclude(JsonInclude.Include.NON_NULL)

kayenta-core/src/main/java/com/netflix/kayenta/canary/providers/DatadogCanaryMetricSetQueryConfig.java

@@ -0,0 +1,25 @@
+package com.netflix.kayenta.canary.providers;
+
+import com.netflix.kayenta.canary.CanaryMetricSetQueryConfig;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+
+import javax.validation.constraints.NotNull;
+
+@Builder
+@ToString
+@NoArgsConstructor
+@AllArgsConstructor
+public class DatadogCanaryMetricSetQueryConfig implements CanaryMetricSetQueryConfig {
+  @NotNull
+  @Getter
+  private String metricName;
+
+  @Override
+  public String getServiceType() {
+    return "datadog";
+  }
+}

kayenta-datadog/kayenta-datadog.gradle

@@ -0,0 +1,14 @@
+dependencies {
+    compile project(":kayenta-core")
+
+    // compile spinnaker.dependency('bootWeb')
+    compile "org.springframework.boot:spring-boot-starter-web:$springBootVersion"
+
+    // compile spinnaker.dependency("korkSwagger")
+    compile "com.netflix.spinnaker.kork:kork-swagger:$korkVersion"
+
+    // compile spinnaker.dependency('lombok')
+    compile "org.projectlombok:lombok:1.16.18"
+
+    compile "com.netflix.spinnaker.orca:orca-core:$orcaVersion"
+}

kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/canary/DatadogCanaryScope.java

@@ -0,0 +1,14 @@
+package com.netflix.kayenta.datadog.canary;
+
+import com.netflix.kayenta.canary.CanaryScope;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+
+import javax.validation.constraints.NotNull;
+
+@Data
+@EqualsAndHashCode(callSuper = true)
+@ToString(callSuper = true)
+public class DatadogCanaryScope extends CanaryScope {
+}

kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/canary/DatadogCanaryScopeFactory.java

@@ -0,0 +1,18 @@
+package com.netflix.kayenta.datadog.canary;
+
+import com.netflix.kayenta.canary.CanaryScope;
+import com.netflix.kayenta.canary.CanaryScopeFactory;
+import org.springframework.stereotype.Component;
+
+@Component
+public class DatadogCanaryScopeFactory implements CanaryScopeFactory {
+  @Override
+  public boolean handles(String serviceType) {
+    return serviceType.equals("datadog");
+  }
+
+  @Override
+  public CanaryScope buildCanaryScope(CanaryScope scope) {
+    return scope;
+  }
+}

kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/config/DatadogConfiguration.java

@@ -0,0 +1,85 @@
+package com.netflix.kayenta.datadog.config;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.netflix.kayenta.datadog.metrics.DatadogMetricsService;
+import com.netflix.kayenta.datadog.security.DatadogCredentials;
+import com.netflix.kayenta.datadog.security.DatadogNamedAccountCredentials;
+import com.netflix.kayenta.datadog.service.DatadogRemoteService;
+import com.netflix.kayenta.metrics.MetricsService;
+import com.netflix.kayenta.retrofit.config.RetrofitClientFactory;
+import com.netflix.kayenta.security.AccountCredentials;
+import com.netflix.kayenta.security.AccountCredentialsRepository;
+import com.squareup.okhttp.OkHttpClient;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.util.CollectionUtils;
+import retrofit.converter.JacksonConverter;
+
+import java.io.IOException;
+import java.util.List;
+
+@Configuration
+@EnableConfigurationProperties
+@ConditionalOnProperty("kayenta.datadog.enabled")
+@ComponentScan({"com.netflix.kayenta.datadog"})
+@Slf4j
+public class DatadogConfiguration {
+  @Bean
+  @ConfigurationProperties("kayenta.datadog")
+  DatadogConfigurationProperties datadogConfigurationProperties() {
+    return new DatadogConfigurationProperties();
+  }
+
+  @Bean
+  @ConfigurationProperties("kayenta.datadog.testControllerDefaults")
+  DatadogConfigurationTestControllerDefaultProperties datadogConfigurationTestControllerDefaultProperties() {
+    return new DatadogConfigurationTestControllerDefaultProperties();
+  }
+
+  @Bean
+  MetricsService datadogMetricsService(DatadogConfigurationProperties datadogConfigurationProperties, RetrofitClientFactory retrofitClientFactory, ObjectMapper objectMapper, OkHttpClient okHttpClient, AccountCredentialsRepository accountCredentialsRepository) throws IOException {
+    DatadogMetricsService.DatadogMetricsServiceBuilder metricsServiceBuilder = DatadogMetricsService.builder();
+
+    for (DatadogManagedAccount account : datadogConfigurationProperties.getAccounts()) {
+      String name = account.getName();
+      List<AccountCredentials.Type> supportedTypes = account.getSupportedTypes();
+
+      DatadogCredentials credentials = DatadogCredentials
+        .builder()
+        .apiKey(account.getApiKey())
+        .applicationKey(account.getApplicationKey())
+        .build();
+
+      DatadogNamedAccountCredentials.DatadogNamedAccountCredentialsBuilder accountCredentialsBuilder =
+        DatadogNamedAccountCredentials
+          .builder()
+          .name(name)
+          .endpoint(account.getEndpoint())
+          .credentials(credentials);
+
+      if (!CollectionUtils.isEmpty(supportedTypes)) {
+        if (supportedTypes.contains(AccountCredentials.Type.METRICS_STORE)) {
+          accountCredentialsBuilder.datadogRemoteService(retrofitClientFactory.createClient(
+            DatadogRemoteService.class,
+            new JacksonConverter(objectMapper),
+            account.getEndpoint(),
+            okHttpClient
+          ));
+        }
+        accountCredentialsBuilder.supportedTypes(supportedTypes);
+      }
+
+      accountCredentialsRepository.save(name, accountCredentialsBuilder.build());
+      metricsServiceBuilder.accountName(name);
+    }
+
+    log.info("Populated DatadogMetricsService with {} Datadog accounts.", datadogConfigurationProperties.getAccounts().size());
+    return metricsServiceBuilder.build();
+  }
+}
+

kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/config/DatadogConfigurationProperties.java

@@ -0,0 +1,11 @@
+package com.netflix.kayenta.datadog.config;
+
+import lombok.Getter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DatadogConfigurationProperties {
+  @Getter
+  private List<DatadogManagedAccount> accounts = new ArrayList<>();
+}

kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/config/DatadogConfigurationTestControllerDefaultProperties.java

@@ -0,0 +1,22 @@
+package com.netflix.kayenta.datadog.config;
+
+import lombok.Getter;
+import lombok.Setter;
+
+/**
+ * This configuration class allows you to specify default values for the PrometheusFetchController.
+ */
+public class DatadogConfigurationTestControllerDefaultProperties {
+
+    @Getter
+    @Setter
+    private String scope;
+
+    @Getter
+    @Setter
+    private String start;
+
+    @Getter
+    @Setter
+    private String end;
+}

kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/config/DatadogManagedAccount.java

@@ -0,0 +1,24 @@
+package com.netflix.kayenta.datadog.config;
+
+import com.netflix.kayenta.retrofit.config.RemoteService;
+import com.netflix.kayenta.security.AccountCredentials;
+import lombok.Data;
+import lombok.Getter;
+import lombok.Setter;
+
+import javax.validation.constraints.NotNull;
+import java.util.List;
+
+@Data
+public class DatadogManagedAccount {
+  @NotNull
+  private String name;
+  private String apiKey;
+  private String applicationKey;
+
+  @NotNull
+  private RemoteService endpoint;
+
+  private List<AccountCredentials.Type> supportedTypes;
+}
+

kayenta-datadog/src/main/java/com/netflix/kayenta/datadog/controller/DatadogFetchController.java

@@ -0,0 +1,100 @@
+package com.netflix.kayenta.datadog.controller;
+
+import com.netflix.kayenta.canary.CanaryConfig;
+import com.netflix.kayenta.canary.CanaryMetricConfig;
+import com.netflix.kayenta.canary.CanaryScope;
+import com.netflix.kayenta.canary.providers.DatadogCanaryMetricSetQueryConfig;
+import com.netflix.kayenta.datadog.config.DatadogConfigurationTestControllerDefaultProperties;
+import com.netflix.kayenta.metrics.SynchronousQueryProcessor;
+import com.netflix.kayenta.security.AccountCredentials;
+import com.netflix.kayenta.security.AccountCredentialsRepository;
+import com.netflix.kayenta.security.CredentialsHelper;
+import io.swagger.annotations.ApiParam;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.util.StringUtils;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Collections;
+import java.util.Map;
+import static com.netflix.kayenta.canary.util.FetchControllerUtils.determineDefaultProperty;
+
+
+@RestController
+@RequestMapping("/fetch/datadog")
+@Slf4j
+public class DatadogFetchController {
+  private final AccountCredentialsRepository accountCredentialsRepository;
+  private final SynchronousQueryProcessor synchronousQueryProcessor;
+  private final DatadogConfigurationTestControllerDefaultProperties datadogConfigurationTestControllerDefaultProperties;
+
+  @Autowired
+  public DatadogFetchController(AccountCredentialsRepository accountCredentialsRepository,
+                                SynchronousQueryProcessor synchronousQueryProcessor,
+                                DatadogConfigurationTestControllerDefaultProperties datadogConfigurationTestControllerDefaultProperties) {
+    this.accountCredentialsRepository = accountCredentialsRepository;
+    this.synchronousQueryProcessor = synchronousQueryProcessor;
+    this.datadogConfigurationTestControllerDefaultProperties = datadogConfigurationTestControllerDefaultProperties;
+  }
+
+  @RequestMapping(value = "/query", method = RequestMethod.POST)
+  public Map queryMetrics(@RequestParam(required = false) final String metricsAccountName,
+                          @RequestParam(required = false) final String storageAccountName,
+                          @ApiParam(defaultValue = "cpu") @RequestParam String metricSetName,
+                          @ApiParam(defaultValue = "avg:system.cpu.user") @RequestParam String metricName,
+                          @ApiParam(value = "The scope of the Datadog query. e.g. autoscaling_group:myapp-prod-v002")
+                            @RequestParam(required = false) String scope,
+                          @ApiParam(value = "An ISO format timestamp, e.g.: 2018-03-15T01:23:45Z")
+                            @RequestParam String start,
+                          @ApiParam(value = "An ISO format timestamp, e.g.: 2018-03-15T01:23:45Z")
+                            @RequestParam String end,
+                          @ApiParam(defaultValue = "60", value = "seconds") @RequestParam Long step) throws IOException {
+    // Apply defaults.
+    scope = determineDefaultProperty(scope, "scope", datadogConfigurationTestControllerDefaultProperties);
+    start = determineDefaultProperty(start, "start", datadogConfigurationTestControllerDefaultProperties);
+    end = determineDefaultProperty(end, "end", datadogConfigurationTestControllerDefaultProperties);
+
+    if (StringUtils.isEmpty(start)) {
+      throw new IllegalArgumentException("Start time is required.");
+    }
+
+    if (StringUtils.isEmpty(end)) {
+      throw new IllegalArgumentException("End time is required.");
+    }
+
+    String resolvedMetricsAccountName = CredentialsHelper.resolveAccountByNameOrType(metricsAccountName,
+      AccountCredentials.Type.METRICS_STORE,
+      accountCredentialsRepository);
+    String resolvedStorageAccountName = CredentialsHelper.resolveAccountByNameOrType(storageAccountName,
+      AccountCredentials.Type.OBJECT_STORE,
+      accountCredentialsRepository);
+
+    DatadogCanaryMetricSetQueryConfig datadogCanaryMetricSetQueryConfig =
+      DatadogCanaryMetricSetQueryConfig
+        .builder()
+        .metricName(metricName)
+        .build();
+
+    CanaryMetricConfig canaryMetricConfig =
+      CanaryMetricConfig
+        .builder()
+        .name(metricSetName)
+        .query(datadogCanaryMetricSetQueryConfig)
+        .build();
+
+    CanaryScope canaryScope = new CanaryScope(scope, null /* region */, Instant.parse(start), Instant.parse(end), step, Collections.emptyMap());
+
+    String metricSetListId = synchronousQueryProcessor.processQuery(resolvedMetricsAccountName,
+                                                                    resolvedStorageAccountName,
+                                                                    CanaryConfig.builder().metric(canaryMetricConfig).build(),
+                                                                    0,
+                                                                    canaryScope);
+
+    return Collections.singletonMap("metricSetListId", metricSetListId);
+  }
+}