Try to work around / diagnose the ongoing timeout issue in atlas (#132)

* set the read timeout on the okhttp client connection for atlas to 90 seconds * add a random uuid to each query to assist in tracking queries and responses, and timeouts
spinnaker · Nov 20, 2017 · 4d3e7fb · 4d3e7fb
1 parent 11bdbfe
commit 4d3e7fb
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 6 deletions.
diff --git a/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/metrics/AtlasMetricsService.java b/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/metrics/AtlasMetricsService.java
@@ -42,10 +42,8 @@
 import java.io.IOException;
 import java.time.Duration;
 import java.time.Instant;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import static java.time.temporal.ChronoUnit.SECONDS;
@@ -84,6 +82,9 @@ public List<MetricSet> queryMetrics(String accountName,
                                       CanaryScope canaryScope) throws IOException {
 
     OkHttpClient okHttpClient = new OkHttpClient();
+    // TODO: (mgraff, duftler) -- we should find out the defaults, and if too small, make this reasonable / configurable
+    okHttpClient.setConnectTimeout(90, TimeUnit.SECONDS);
+    okHttpClient.setReadTimeout(90, TimeUnit.SECONDS);
 
     if (!(canaryScope instanceof AtlasCanaryScope)) {
       throw new IllegalArgumentException("Canary scope not instance of AtlasCanaryScope: " + canaryScope);
@@ -124,7 +125,8 @@ public List<MetricSet> queryMetrics(String accountName,
                                                                    atlasCanaryScope.getStart().toEpochMilli(),
                                                                    atlasCanaryScope.getEnd().toEpochMilli(),
                                                                    isoStep,
-                                                                   credentials.getFetchId());
+                                                                   credentials.getFetchId(),
+                                                                   UUID.randomUUID() + "");
     Map<String, AtlasResults> idToAtlasResultsMap = AtlasResultsHelper.merge(atlasResultsList);
     List<MetricSet> metricSetList = new ArrayList<>();
 

diff --git a/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/service/AtlasRemoteService.java b/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/service/AtlasRemoteService.java
@@ -30,5 +30,6 @@ List<AtlasResults> fetch(@Query("q") String q,
                            @Query("s") Long start,
                            @Query("e") Long end,
                            @Query("step") String step,
-                           @Query("id") String id);
+                           @Query("id") String id,
+                           @Query("kayentaQueryUUID") String kayentaQueryUUID);
 }