feat(config): Add /metadata/metricsService endpoint. (#220)

spinnaker · Jan 31, 2018 · 5118542 · 5118542
1 parent 89a64e0
commit 5118542
Show file tree

Hide file tree

Showing 7 changed files with 172 additions and 1 deletion.
diff --git a/kayenta-core/src/main/java/com/netflix/kayenta/metrics/MetricsService.java b/kayenta-core/src/main/java/com/netflix/kayenta/metrics/MetricsService.java
@@ -21,7 +21,9 @@
 import com.netflix.kayenta.canary.CanaryScope;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 public interface MetricsService {
   String getType();
@@ -32,4 +34,8 @@ List<MetricSet> queryMetrics(String accountName,
                                CanaryConfig canaryConfig,
                                CanaryMetricConfig canaryMetricConfig,
                                CanaryScope canaryScope) throws IOException;
+
+  default List<Map> getMetadata(String metricsAccountName, String filter) throws IOException {
+    return Collections.emptyList();
+  }
 }
diff --git a/...driver/src/main/java/com/netflix/kayenta/stackdriver/config/StackdriverConfiguration.java b/...driver/src/main/java/com/netflix/kayenta/stackdriver/config/StackdriverConfiguration.java
@@ -23,6 +23,7 @@
 import com.netflix.kayenta.stackdriver.metrics.StackdriverMetricsService;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
@@ -36,6 +37,12 @@
 @Slf4j
 public class StackdriverConfiguration {
 
+  @Bean
+  @ConfigurationProperties("kayenta.stackdriver")
+  StackdriverConfigurationProperties stackdriverConfigurationProperties() {
+    return new StackdriverConfigurationProperties();
+  }
+
   @Bean
   @DependsOn({"registerGoogleCredentials"})
   MetricsService stackdriverMetricsService(AccountCredentialsRepository accountCredentialsRepository) {

diff --git a/.../main/java/com/netflix/kayenta/stackdriver/config/StackdriverConfigurationProperties.java b/.../main/java/com/netflix/kayenta/stackdriver/config/StackdriverConfigurationProperties.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2018 Google, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.stackdriver.config;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.time.Duration;
+
+public class StackdriverConfigurationProperties {
+
+  @Getter
+  @Setter
+  private long metadataCachingIntervalMS = Duration.ofSeconds(60).toMillis();
+}
diff --git a/...iver/src/main/java/com/netflix/kayenta/stackdriver/metrics/StackdriverMetricsService.java b/...iver/src/main/java/com/netflix/kayenta/stackdriver/metrics/StackdriverMetricsService.java
@@ -17,8 +17,10 @@
 package com.netflix.kayenta.stackdriver.metrics;
 
 import com.google.api.services.monitoring.v3.Monitoring;
+import com.google.api.services.monitoring.v3.model.ListMetricDescriptorsResponse;
 import com.google.api.services.monitoring.v3.model.ListTimeSeriesResponse;
 import com.google.api.services.monitoring.v3.model.Metric;
+import com.google.api.services.monitoring.v3.model.MetricDescriptor;
 import com.google.api.services.monitoring.v3.model.MonitoredResource;
 import com.google.api.services.monitoring.v3.model.Point;
 import com.google.api.services.monitoring.v3.model.TimeSeries;
@@ -30,8 +32,11 @@
 import com.netflix.kayenta.google.security.GoogleNamedAccountCredentials;
 import com.netflix.kayenta.metrics.MetricSet;
 import com.netflix.kayenta.metrics.MetricsService;
+import com.netflix.kayenta.security.AccountCredentials;
 import com.netflix.kayenta.security.AccountCredentialsRepository;
+import com.netflix.kayenta.security.CredentialsHelper;
 import com.netflix.kayenta.stackdriver.canary.StackdriverCanaryScope;
+import com.netflix.kayenta.stackdriver.config.StackdriverConfigurationProperties;
 import com.netflix.spectator.api.Id;
 import com.netflix.spectator.api.Registry;
 import freemarker.template.Configuration;
@@ -42,6 +47,7 @@
 import lombok.Singular;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.ui.freemarker.FreeMarkerTemplateUtils;
 import org.springframework.util.CollectionUtils;
 import org.springframework.util.StringUtils;
@@ -56,6 +62,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -74,6 +81,11 @@ public class StackdriverMetricsService implements MetricsService {
   @Autowired
   private final Registry registry;
 
+  @Autowired
+  private final StackdriverConfigurationProperties stackdriverConfigurationProperties;
+
+  private List<MetricDescriptor> metricDescriptorsCache = Collections.emptyList();
+
   @Override
   public String getType() {
     return "stackdriver";
@@ -328,4 +340,48 @@ String expandCustomFilter(CanaryConfig canaryConfig,
 
     return customFilter;
   }
+
+  @Override
+  public List<Map> getMetadata(String metricsAccountName, String filter) throws IOException {
+    if (!StringUtils.isEmpty(filter)) {
+      String lowerCaseFilter = filter.toLowerCase();
+
+      return metricDescriptorsCache
+        .stream()
+        .filter(metricDescriptor -> metricDescriptor.getName().toLowerCase().contains(lowerCaseFilter))
+        .collect(Collectors.toList());
+    } else {
+      return metricDescriptorsCache
+        .stream()
+        .collect(Collectors.toList());
+    }
+  }
+
+  @Scheduled(fixedDelayString = "#{@stackdriverConfigurationProperties.metadataCachingIntervalMS}")
+  public void updateMetricDescriptorsCache() throws IOException {
+    Set<AccountCredentials> accountCredentialsSet =
+      CredentialsHelper.getAllAccountsOfType(AccountCredentials.Type.METRICS_STORE, accountCredentialsRepository);
+
+    for (AccountCredentials credentials : accountCredentialsSet) {
+      if (credentials instanceof GoogleNamedAccountCredentials) {
+        GoogleNamedAccountCredentials stackdriverCredentials = (GoogleNamedAccountCredentials)credentials;
+        ListMetricDescriptorsResponse listMetricDescriptorsResponse =
+          stackdriverCredentials
+            .getMonitoring()
+            .projects()
+            .metricDescriptors()
+            .list("projects/" + stackdriverCredentials.getProject())
+            .execute();
+        List<MetricDescriptor> metricDescriptors = listMetricDescriptorsResponse.getMetricDescriptors();
+
+        if (!CollectionUtils.isEmpty(metricDescriptors)) {
+          metricDescriptorsCache = metricDescriptors;
+
+          log.debug("Updated cache with {} metric descriptors via account {}.", metricDescriptors.size(), stackdriverCredentials.getName());
+        } else {
+          log.debug("While updating cache, found no metric descriptors via account {}.", stackdriverCredentials.getName());
+        }
+      }
+    }
+  }
 }
diff --git a/kayenta-web/config/kayenta.yml b/kayenta-web/config/kayenta.yml
@@ -82,6 +82,7 @@ swagger:
     - /fetch.*
     - /health
     - /judges.*
+    - /metadata.*
     - /metricSetList.*
     - /metricSetPairList.*
     - /pipeline.*
diff --git a/kayenta-web/src/main/java/com/netflix/kayenta/controllers/CanaryConfigController.java b/kayenta-web/src/main/java/com/netflix/kayenta/controllers/CanaryConfigController.java
@@ -24,7 +24,6 @@
 import com.netflix.kayenta.storage.ObjectType;
 import com.netflix.kayenta.storage.StorageService;
 import com.netflix.kayenta.storage.StorageServiceRepository;
-import com.netflix.spectator.api.Registry;
 import com.netflix.spinnaker.kork.web.exceptions.NotFoundException;
 import io.swagger.annotations.ApiOperation;
 import lombok.extern.slf4j.Slf4j;

diff --git a/...a-web/src/main/java/com/netflix/kayenta/controllers/MetricsServiceMetadataController.java b/...a-web/src/main/java/com/netflix/kayenta/controllers/MetricsServiceMetadataController.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2018 Google, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.controllers;
+
+import com.netflix.kayenta.metrics.MetricsService;
+import com.netflix.kayenta.metrics.MetricsServiceRepository;
+import com.netflix.kayenta.security.AccountCredentials;
+import com.netflix.kayenta.security.AccountCredentialsRepository;
+import com.netflix.kayenta.security.CredentialsHelper;
+import io.swagger.annotations.ApiOperation;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+@RestController
+@RequestMapping("/metadata/metricsService")
+@Slf4j
+public class MetricsServiceMetadataController {
+
+  private final AccountCredentialsRepository accountCredentialsRepository;
+  private final MetricsServiceRepository metricsServiceRepository;
+
+  @Autowired
+  public MetricsServiceMetadataController(AccountCredentialsRepository accountCredentialsRepository,
+                                          MetricsServiceRepository metricsServiceRepository) {
+    this.accountCredentialsRepository = accountCredentialsRepository;
+    this.metricsServiceRepository = metricsServiceRepository;
+  }
+
+  @ApiOperation(value = "Retrieve a list of descriptors for use in populating the canary config ui")
+  @RequestMapping(method = RequestMethod.GET)
+  public List<Map> listMetadata(@RequestParam(required = false) final String metricsAccountName,
+                                @RequestParam(required = false) final String filter) throws IOException {
+    String resolvedMetricsAccountName = CredentialsHelper.resolveAccountByNameOrType(metricsAccountName,
+                                                                                     AccountCredentials.Type.METRICS_STORE,
+                                                                                     accountCredentialsRepository);
+    MetricsService metricsService =
+      metricsServiceRepository
+        .getOne(resolvedMetricsAccountName)
+        .orElseThrow(() -> new IllegalArgumentException("No metrics service was configured; unable to read from metrics store."));
+    List<Map> matchingDescriptors = metricsService.getMetadata(resolvedMetricsAccountName, filter);
+
+    if (StringUtils.isEmpty(filter)) {
+      log.debug("Returned all {} descriptors via account {}.", matchingDescriptors.size(), resolvedMetricsAccountName, filter);
+    } else {
+      log.debug("Matched {} descriptors via account {} using filter '{}'.", matchingDescriptors.size(), resolvedMetricsAccountName, filter);
+    }
+
+    return metricsService.getMetadata(resolvedMetricsAccountName, filter);
+  }
+}