fixes #2196 add metrics-config module

ingress-proxy/src/main/java/com/networknt/proxy/ExternalServiceHandler.java

@@ -281,8 +281,6 @@ private boolean createJavaHttpClient() {
         return true;
     }
 
-
-
     /**
      * Builds the request URL for an HttpRequest.
      *

metrics-config/pom.xml

@@ -0,0 +1,87 @@
+<!--
+  ~ Copyright (c) 2016 Network New Technologies Inc.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~      http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.networknt</groupId>
+        <artifactId>light-4j</artifactId>
+        <version>2.1.34-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>metrics-config</artifactId>
+    <packaging>jar</packaging>
+    <description>A metrics config module to be shared with light-aws-lambda</description>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.networknt</groupId>
+            <artifactId>config</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.networknt</groupId>
+            <artifactId>utility</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.networknt</groupId>
+            <artifactId>http-client</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.hdrhistogram</groupId>
+            <artifactId>HdrHistogram</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.findbugs</groupId>
+            <artifactId>jsr305</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+</project>

metrics-config/src/main/java/com/networknt/metrics/APMAgentReporter.java

@@ -0,0 +1,278 @@
+/*
+ * Copyright 2010-2013 Coda Hale and Yammer, Inc., 2014-2017 Dropwizard Team
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.networknt.metrics;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.dropwizard.metrics.Counter;
+import io.dropwizard.metrics.Counting;
+import io.dropwizard.metrics.Gauge;
+import io.dropwizard.metrics.Histogram;
+import io.dropwizard.metrics.Meter;
+import io.dropwizard.metrics.Metered;
+import io.dropwizard.metrics.MetricFilter;
+import io.dropwizard.metrics.MetricName;
+import io.dropwizard.metrics.MetricRegistry;
+import io.dropwizard.metrics.ScheduledReporter;
+import io.dropwizard.metrics.Snapshot;
+import io.dropwizard.metrics.Timer;
+import io.dropwizard.metrics.influxdb.data.InfluxDbPoint;
+
+public final class APMAgentReporter extends ScheduledReporter {
+    public static final class Builder {
+        private final MetricRegistry registry;
+        private Map<String, String> tags;
+        private TimeUnit rateUnit;
+        private TimeUnit durationUnit;
+        private MetricFilter filter;
+        private boolean skipIdleMetrics;
+
+        private Builder(MetricRegistry registry) {
+            this.registry = registry;
+            this.tags = null;
+            this.rateUnit = TimeUnit.SECONDS;
+            this.durationUnit = TimeUnit.MILLISECONDS;
+            this.filter = MetricFilter.ALL;
+        }
+
+        /**
+         * Add these tags to all metrics.
+         *
+         * @param tags a map containing tags common to all metrics
+         * @return {@code this}
+         */
+        public Builder withTags(Map<String, String> tags) {
+            this.tags = Collections.unmodifiableMap(tags);
+            return this;
+        }
+
+        /**
+         * Convert rates to the given time unit.
+         *
+         * @param rateUnit a unit of time
+         * @return {@code this}
+         */
+        public Builder convertRatesTo(TimeUnit rateUnit) {
+            this.rateUnit = rateUnit;
+            return this;
+        }
+
+        /**
+         * Convert durations to the given time unit.
+         *
+         * @param durationUnit a unit of time
+         * @return {@code this}
+         */
+        public Builder convertDurationsTo(TimeUnit durationUnit) {
+            this.durationUnit = durationUnit;
+            return this;
+        }
+
+        /**
+         * Only report metrics which match the given filter.
+         *
+         * @param filter a {@link MetricFilter}
+         * @return {@code this}
+         */
+        public Builder filter(MetricFilter filter) {
+            this.filter = filter;
+            return this;
+        }
+
+        /**
+         * Only report metrics that have changed.
+         *
+         * @param skipIdleMetrics true/false for skipping metrics not reported
+         * @return {@code this}
+         */
+        public Builder skipIdleMetrics(boolean skipIdleMetrics) {
+            this.skipIdleMetrics = skipIdleMetrics;
+            return this;
+        }
+
+        public APMAgentReporter build(final TimeSeriesDbSender influxDb) {
+            return new APMAgentReporter(registry, influxDb, tags, rateUnit, durationUnit, filter, skipIdleMetrics);
+        }
+    }
+
+    private static final Logger logger = LoggerFactory.getLogger(APMAgentReporter.class);
+    private static final String COUNT = ".count";
+    private final TimeSeriesDbSender influxDb;
+    private final boolean skipIdleMetrics;
+    private final Map<MetricName, Long> previousValues;
+
+    private APMAgentReporter(final MetricRegistry registry, final TimeSeriesDbSender influxDb, final Map<String, String> tags,
+                             final TimeUnit rateUnit, final TimeUnit durationUnit, final MetricFilter filter, final boolean skipIdleMetrics) {
+        super(registry, "apm-reporter", filter, rateUnit, durationUnit);
+        this.influxDb = influxDb;
+        influxDb.setTags(tags);
+        this.skipIdleMetrics = skipIdleMetrics;
+        this.previousValues = new TreeMap<>();
+    }
+
+    public static Builder forRegistry(MetricRegistry registry) {
+        return new Builder(registry);
+    }
+
+    @Override
+    public void report(final SortedMap<MetricName, Gauge> gauges, final SortedMap<MetricName, Counter> counters,
+                       final SortedMap<MetricName, Histogram> histograms, final SortedMap<MetricName, Meter> meters, final SortedMap<MetricName, Timer> timers) {
+        final long now = System.currentTimeMillis();
+        if(logger.isDebugEnabled()) logger.debug("APMAgentReporter report is called with counter size {}", counters.size());
+        try {
+            influxDb.flush();
+
+            for (Map.Entry<MetricName, Gauge> entry : gauges.entrySet()) {
+                reportGauge(entry.getKey(), entry.getValue(), now);
+            }
+
+            for (Map.Entry<MetricName, Counter> entry : counters.entrySet()) {
+                reportCounter(entry.getKey(), entry.getValue(), now);
+            }
+
+            for (Map.Entry<MetricName, Histogram> entry : histograms.entrySet()) {
+                reportHistogram(entry.getKey(), entry.getValue(), now);
+            }
+
+            for (Map.Entry<MetricName, Meter> entry : meters.entrySet()) {
+                reportMeter(entry.getKey(), entry.getValue(), now);
+            }
+
+            for (Map.Entry<MetricName, Timer> entry : timers.entrySet()) {
+                reportTimer(entry.getKey(), entry.getValue(), now);
+            }
+
+            if (influxDb.hasSeriesData()) {
+                influxDb.writeData();
+            }
+            // reset counters
+            for (Map.Entry<MetricName, Counter> entry : counters.entrySet()) {
+                Counter counter = entry.getValue();
+                long count = counter.getCount();
+                counter.dec(count);
+            }
+        } catch (Exception e) {
+            logger.error("Unable to report to APM Agent. Discarding data.", e);
+        }
+    }
+
+    private void reportTimer(MetricName name, Timer timer, long now) {
+        if (canSkipMetric(name, timer)) {
+            return;
+        }
+        final Snapshot snapshot = timer.getSnapshot();
+
+        Map<String, String> apiTags = new HashMap<>(name.getTags());
+        String apiName = apiTags.remove("api");
+
+        influxDb.appendPoints(new InfluxDbPoint(apiName + "." + name.getKey() + ".min", apiTags, now, format(convertDuration(snapshot.getMin()))));
+        influxDb.appendPoints(new InfluxDbPoint(apiName + "." + name.getKey() + ".max", apiTags, now, format(convertDuration(snapshot.getMax()))));
+        influxDb.appendPoints(new InfluxDbPoint(apiName + "." + name.getKey() + ".mean", apiTags, now, format(convertDuration(snapshot.getMean()))));
+
+    }
+
+    private void reportHistogram(MetricName name, Histogram histogram, long now) {
+        if (canSkipMetric(name, histogram)) {
+            return;
+        }
+        final Snapshot snapshot = histogram.getSnapshot();
+        Map<String, String> apiTags = new HashMap<>(name.getTags());
+        String apiName = apiTags.remove("api");
+
+        influxDb.appendPoints(new InfluxDbPoint(apiName + "." + name.getKey() + COUNT, apiTags, now, format(histogram.getCount())));
+        influxDb.appendPoints(new InfluxDbPoint(apiName + "." + name.getKey() + ".min", apiTags, now, format(snapshot.getMin())));
+        influxDb.appendPoints(new InfluxDbPoint(apiName + "." + name.getKey() + ".max", apiTags, now, format(snapshot.getMax())));
+        influxDb.appendPoints(new InfluxDbPoint(apiName + "." + name.getKey() + ".mean", apiTags, now, format(snapshot.getMean())));
+    }
+
+    private void reportCounter(MetricName name, Counter counter, long now) {
+        Map<String, String> apiTags = new HashMap<>(name.getTags());
+        String apiName = apiTags.remove("api");
+
+        influxDb.appendPoints(new InfluxDbPoint(apiName + "." + name.getKey() + COUNT, apiTags, now, format(counter.getCount())));
+    }
+
+    private void reportGauge(MetricName name, Gauge<?> gauge, long now) {
+        final String value = format(gauge.getValue());
+        if(value != null) {
+            Map<String, String> apiTags = new HashMap<>(name.getTags());
+            String apiName = apiTags.remove("api");
+
+            influxDb.appendPoints(new InfluxDbPoint(apiName + "." + name.getKey(), apiTags, now, value));
+        }
+    }
+
+    private void reportMeter(MetricName name, Metered meter, long now) {
+        if (canSkipMetric(name, meter)) {
+            return;
+        }
+        Map<String, String> apiTags = new HashMap<>(name.getTags());
+        String apiName = apiTags.remove("api");
+
+        influxDb.appendPoints(new InfluxDbPoint(apiName + "." + name.getKey() + COUNT, apiTags, now, format(meter.getCount())));
+    }
+
+    private boolean canSkipMetric(MetricName name, Counting counting) {
+        boolean isIdle = (calculateDelta(name, counting.getCount()) == 0);
+        if (skipIdleMetrics && !isIdle) {
+            previousValues.put(name, counting.getCount());
+        }
+        return skipIdleMetrics && isIdle;
+    }
+
+    private long calculateDelta(MetricName name, long count) {
+        Long previous = previousValues.get(name);
+        if (previous == null) {
+            return -1;
+        }
+        if (count < previous) {
+            logger.warn("Saw a non-monotonically increasing value for metric '{}'", name);
+            return 0;
+        }
+        return count - previous;
+    }
+
+    private String format(Object o) {
+        if (o instanceof Float) {
+            return format(((Float) o).doubleValue());
+        } else if (o instanceof Double) {
+            return format(((Double) o).doubleValue());
+        } else if (o instanceof Byte) {
+            return format(((Byte) o).longValue());
+        } else if (o instanceof Short) {
+            return format(((Short) o).longValue());
+        } else if (o instanceof Integer) {
+            return format(((Integer) o).longValue());
+        } else if (o instanceof Long) {
+            return format(((Long) o).longValue());
+        }
+        return null;
+    }
+    private String format(long n) {
+        return Long.toString(n);
+    }
+
+    private String format(double v) {
+        // the Carbon plaintext format is pretty underspecified, but it seems like it just wants
+        // US-formatted digits
+        return String.format(Locale.US, "%2.2f", v);
+    }
+}