Add metrics to track memcached performance

heroic-component/src/main/java/com/spotify/heroic/statistics/HeroicReporter.java

@@ -40,6 +40,8 @@ public interface HeroicReporter {
 
     QueryReporter newQueryReporter();
 
+    MemcachedReporter newMemcachedReporter(final String consumerType);
+
     void registerShards(Set<Map<String, String>> knownShards);
 
     /**

heroic-component/src/main/java/com/spotify/heroic/statistics/MemcachedReporter.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2018 Spotify AB.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.spotify.heroic.statistics;
+
+public interface MemcachedReporter {
+
+  void reportMemcachedHit();
+
+  void reportMemcachedMiss();
+
+  void reportMemcachedTimeout();
+
+  void reportMemcachedError();
+}

heroic-component/src/main/java/com/spotify/heroic/statistics/noop/NoopHeroicReporter.java

@@ -21,6 +21,7 @@
 
 package com.spotify.heroic.statistics.noop;
 
+import com.spotify.heroic.statistics.MemcachedReporter;
 import com.spotify.heroic.statistics.QueryReporter;
 import com.spotify.heroic.statistics.AnalyticsReporter;
 import com.spotify.heroic.statistics.ConsumerReporter;
@@ -70,6 +71,9 @@ public QueryReporter newQueryReporter() {
         return NoopQueryReporter.get();
     }
 
+    @Override
+    public MemcachedReporter newMemcachedReporter(final String consumerType) { return NoopMemcachedReporter.get(); }
+
     @Override
     public void registerShards(Set<Map<String, String>> knownShards) {
     }
@@ -78,6 +82,7 @@ public void registerShards(Set<Map<String, String>> knownShards) {
     public void registerCacheSize(final String id, final Supplier<Long> cacheSize) {
     }
 
+
     private static final NoopHeroicReporter instance = new NoopHeroicReporter();
 
     public static NoopHeroicReporter get() {

heroic-component/src/main/java/com/spotify/heroic/statistics/noop/NoopMemcachedReporter.java

@@ -0,0 +1,36 @@
+package com.spotify.heroic.statistics.noop;
+
+import com.spotify.heroic.statistics.MemcachedReporter;
+
+public class NoopMemcachedReporter implements MemcachedReporter {
+
+  private NoopMemcachedReporter() {
+  }
+
+  @Override
+  public void reportMemcachedHit() {
+
+  }
+
+  @Override
+  public void reportMemcachedMiss() {
+
+  }
+
+  @Override
+  public void reportMemcachedTimeout() {
+
+  }
+
+  @Override
+  public void reportMemcachedError() {
+
+  }
+
+  private static final NoopMemcachedReporter instance = new NoopMemcachedReporter();
+
+  public static NoopMemcachedReporter get() {
+    return instance;
+  }
+
+}

heroic-elasticsearch-utils/src/main/java/com/spotify/heroic/elasticsearch/DistributedRateLimitedCache.java

@@ -26,6 +26,7 @@
 import com.google.common.hash.Hashing;
 import com.google.common.util.concurrent.RateLimiter;
 import com.spotify.folsom.MemcacheClient;
+import com.spotify.heroic.statistics.MemcachedReporter;
 import io.opencensus.common.Scope;
 import io.opencensus.trace.Span;
 import io.opencensus.trace.Status;
@@ -56,10 +57,12 @@ public class DistributedRateLimitedCache<K> implements RateLimitedCache<K> {
   private final RateLimiter rateLimiter;
   private final MemcacheClient memcachedClient;
   private final int memcachedTtlSeconds;
+  private final MemcachedReporter memcachedReporter;
 
   private static final HashFunction HASH_FUNCTION = Hashing.murmur3_128();
   private final Tracer tracer = Tracing.getTracer();
 
+
   /**
    *
    * @param key key to store/lookup in cache.
@@ -80,16 +83,24 @@ public boolean acquire(K key, final Runnable cacheHit) {
       try {
         if (memcachedClient.get(
           cacheKey).toCompletableFuture().get(100, TimeUnit.MILLISECONDS) != null) {
+          memcachedReporter.reportMemcachedHit();
           span.addAnnotation("Found key in memcached");
           cache.putIfAbsent(key, true);
           cacheHit.run();
           return false;
+        } else {
+          memcachedReporter.reportMemcachedMiss();
         }
-      } catch (TimeoutException | InterruptedException | ExecutionException e) {
+      } catch (TimeoutException e) {
+        span.setStatus(Status.INTERNAL.withDescription(e.getMessage()));
+        memcachedReporter.reportMemcachedTimeout();
+        log.error("Failed to get key from memecached");
+      }
+      catch (InterruptedException | ExecutionException e) {
         span.setStatus(Status.INTERNAL.withDescription(e.getMessage()));
+        memcachedReporter.reportMemcachedError();
         log.error("Failed to get key from memecached", e);
       }
-
       span.addAnnotation("Acquiring rate limiter");
       rateLimiter.acquire();
       span.addAnnotation("Acquired rate limiter");

heroic-elasticsearch-utils/src/test/java/com/spotify/heroic/elasticsearch/DistributedRateLimitedCacheTest.java

@@ -10,6 +10,7 @@
 
 import com.google.common.util.concurrent.RateLimiter;
 import com.spotify.folsom.MemcacheClient;
+import com.spotify.heroic.statistics.noop.NoopHeroicReporter;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionStage;
 import java.util.concurrent.ConcurrentHashMap;
@@ -45,7 +46,8 @@ public class DistributedRateLimitedCacheTest {
     public void setUp() {
        cache = new ConcurrentHashMap<>();
        writeCache = new DistributedRateLimitedCache<>(
-         cache, rateLimiter, memcacheClient, MEMCACHED_TTL_SECONDS);
+         cache, rateLimiter, memcacheClient, MEMCACHED_TTL_SECONDS, new NoopHeroicReporter()
+           .newMemcachedReporter("test"));
     }
 
 

metadata/elasticsearch/src/main/java/com/spotify/heroic/metadata/elasticsearch/ElasticsearchMetadataModule.java

@@ -229,7 +229,8 @@ public RateLimitedCache<Pair<String, HashCode>> writeCache(HeroicReporter report
                   cache.asMap(),
                   RateLimiter.create(writesPerSecond, rateLimitSlowStartSeconds, SECONDS),
                   MemcachedConnection.create(distributedCacheSrvRecord),
-                  toIntExact(Duration.of(writeCacheDurationMinutes, MINUTES).convert(SECONDS))
+                  toIntExact(Duration.of(writeCacheDurationMinutes, MINUTES).convert(SECONDS)),
+                  reporter.newMemcachedReporter("metadata")
                 );
             }
 

statistics/semantic/src/main/java/com/spotify/heroic/statistics/semantic/SemanticHeroicReporter.java

@@ -27,6 +27,7 @@
 import com.spotify.heroic.statistics.ConsumerReporter;
 import com.spotify.heroic.statistics.HeroicReporter;
 import com.spotify.heroic.statistics.IngestionManagerReporter;
+import com.spotify.heroic.statistics.MemcachedReporter;
 import com.spotify.heroic.statistics.MetadataBackendReporter;
 import com.spotify.heroic.statistics.MetricBackendReporter;
 import com.spotify.heroic.statistics.QueryReporter;
@@ -86,6 +87,11 @@ public MetricBackendReporter newMetricBackend() {
         return new SemanticMetricBackendReporter(registry);
     }
 
+    @Override
+    public MemcachedReporter newMemcachedReporter(final String consumerType) {
+        return new SemanticMemcachedReporter(registry, consumerType);
+    }
+
     @Override
     public QueryReporter newQueryReporter() {
         return new SemanticQueryReporter(registry);

statistics/semantic/src/main/java/com/spotify/heroic/statistics/semantic/SemanticMemcachedReporter.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2018 Spotify AB.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.spotify.heroic.statistics.semantic;
+
+import com.codahale.metrics.Meter;
+import com.google.common.annotations.VisibleForTesting;
+import com.spotify.heroic.statistics.MemcachedReporter;
+import com.spotify.metrics.core.MetricId;
+import com.spotify.metrics.core.SemanticMetricRegistry;
+import io.opencensus.internal.PublicForTesting;
+import lombok.ToString;
+
+@ToString(of = {})
+@VisibleForTesting
+public class SemanticMemcachedReporter implements MemcachedReporter {
+
+  private static final String COMPONENT = "memcached";
+
+  @VisibleForTesting
+  private final Meter memcachedHit;
+  @PublicForTesting
+  private final Meter memcachedMiss;
+  @PublicForTesting
+  private final Meter memcachedTimeout;
+  @PublicForTesting
+  private final Meter memcachedError;
+
+  public SemanticMemcachedReporter(SemanticMetricRegistry registry, final String consumerType) {
+    final MetricId id = MetricId.build().tagged("component", COMPONENT, "consumer", consumerType);
+
+    memcachedHit = registry.meter(id.tagged("what", "memcached-performance", "result",
+        "hit"));
+    memcachedMiss = registry.meter(id.tagged("what", "memcached-performance", "result",
+        "miss"));
+    memcachedTimeout = registry.meter(id.tagged("what", "memcached-performance", "result",
+        "timeout"));
+
+    memcachedError = registry.meter(id.tagged("what", "memcached-performance", "result",
+        "error"));
+
+
+  }
+
+
+  @Override
+  public void reportMemcachedHit() { memcachedHit.mark(); }
+
+  @Override
+  public void reportMemcachedMiss() { memcachedMiss.mark(); }
+
+  @Override
+  public void reportMemcachedTimeout() { memcachedTimeout.mark(); }
+
+  @Override
+  public void reportMemcachedError() { memcachedError.mark(); }
+}
+

suggest/elasticsearch/src/main/java/com/spotify/heroic/suggest/elasticsearch/ElasticsearchSuggestModule.java

@@ -218,7 +218,8 @@ public RateLimitedCache<Pair<String, HashCode>> writeCache(final HeroicReporter
                   cache.asMap(),
                   RateLimiter.create(writesPerSecond, rateLimitSlowStartSeconds, SECONDS),
                   MemcachedConnection.create(distributedCacheSrvRecord),
-                  toIntExact(Duration.of(writeCacheDurationMinutes, MINUTES).convert(SECONDS))
+                  toIntExact(Duration.of(writeCacheDurationMinutes, MINUTES).convert(SECONDS)),
+                  reporter.newMemcachedReporter("suggest")
                 );
             }