Adding CloudWatchEmbeddedMetricsReporter

reporters/src/main/java/com/nextdoor/bender/monitoring/ReporterUtils.java

@@ -0,0 +1,20 @@
+package com.nextdoor.bender.monitoring;
+
+import com.amazonaws.services.cloudwatch.model.Dimension;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * Utils class to capture static methods used across multiple reporters
+ */
+public class ReporterUtils {
+
+    public static List<Dimension> tagsToDimensions(final Set<Tag> tags) {
+        return tags.stream().map(e -> tagToDim(e.getKey(), e.getValue())).collect(Collectors.toList());
+    }
+
+    private static Dimension tagToDim(String name, String value) {
+        return new Dimension().withName(name).withValue(value != null ? value : "None");
+    }
+}

reporters/src/main/java/com/nextdoor/bender/monitoring/cw/CloudwatchReporter.java

@@ -20,7 +20,6 @@
 import java.util.Date;
 import java.util.List;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 import org.apache.commons.collections4.ListUtils;
 import com.amazonaws.services.cloudwatch.AmazonCloudWatch;
@@ -33,6 +32,8 @@
 import com.nextdoor.bender.monitoring.StatFilter;
 import com.nextdoor.bender.monitoring.Tag;
 
+import static com.nextdoor.bender.monitoring.ReporterUtils.tagsToDimensions;
+
 /**
  * Writes metrics to Amazon Cloudwatch.
  */
@@ -90,14 +91,6 @@ public void write(ArrayList<Stat> stats, long invokeTimeMs, Set<Tag> tags) {
     }
   }
 
-  private Collection<Dimension> tagsToDimensions(final Set<Tag> tags) {
-    return tags.stream().map(e -> tagToDim(e.getKey(), e.getValue())).collect(Collectors.toList());
-  }
-
-  private Dimension tagToDim(String name, String value) {
-    return new Dimension().withName(name).withValue(value != null ? value : "None");
-  }
-
   @Override
   public List<StatFilter> getStatFilters() {
     return this.statFilters;

reporters/src/main/java/com/nextdoor/bender/monitoring/embedded/metrics/AWSMetricObject.java

@@ -0,0 +1,35 @@
+package com.nextdoor.bender.monitoring.embedded.metrics;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.util.List;
+
+public class AWSMetricObject {
+    @SerializedName(value = "Timestamp")
+    private long timestamp;
+
+    @SerializedName(value = "CloudWatchMetrics")
+    private List<CloudWatchMetricObject> cloudWatchMetricObjects;
+
+    public AWSMetricObject(long timestamp,
+                           List<CloudWatchMetricObject> cloudWatchMetricObjects) {
+        this.timestamp = timestamp;
+        this.cloudWatchMetricObjects = cloudWatchMetricObjects;
+    }
+
+    public long getTimestamp() {
+        return timestamp;
+    }
+
+    public void setTimestamp(int timestamp) {
+        this.timestamp = timestamp;
+    }
+
+    public List<CloudWatchMetricObject> getCloudWatchMetricObjects() {
+        return cloudWatchMetricObjects;
+    }
+
+    public void setCloudWatchMetricObjects(List<CloudWatchMetricObject> cloudWatchMetricObjects) {
+        this.cloudWatchMetricObjects = cloudWatchMetricObjects;
+    }
+}

reporters/src/main/java/com/nextdoor/bender/monitoring/embedded/metrics/CloudWatchEmbeddedMetricsReporter.java

@@ -0,0 +1,132 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ * Copyright 2017 Nextdoor.com, Inc
+ *
+ */
+
+package com.nextdoor.bender.monitoring.embedded.metrics;
+
+import com.amazonaws.services.cloudwatch.model.Dimension;
+import com.amazonaws.services.cloudwatch.model.StandardUnit;
+import com.google.gson.Gson;
+import com.nextdoor.bender.monitoring.Reporter;
+import com.nextdoor.bender.monitoring.Stat;
+import com.nextdoor.bender.monitoring.StatFilter;
+import com.nextdoor.bender.monitoring.Tag;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static com.nextdoor.bender.monitoring.ReporterUtils.tagsToDimensions;
+
+/**
+ * This reporter writes metrics by outputting to the Lambda stdout in the CW embedded metrics format
+ * Doc link: https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch_Embedded_Metric_Format_Generation.html
+ */
+public class CloudWatchEmbeddedMetricsReporter implements Reporter {
+    private final String namespace;
+    private final List<StatFilter> statFilters;
+    private final Gson gson;
+
+    public CloudWatchEmbeddedMetricsReporter(final String namespace,
+                                             final List<StatFilter> statFilters) {
+        this.namespace = namespace;
+        this.statFilters = statFilters;
+        this.gson = new Gson();
+    }
+
+    @Override
+    public void write(ArrayList<Stat> stats, long invokeTimeMs, Set<Tag> tags) {
+        Map<String, String> dimensions = tagsToDimensions(tags).stream()
+                .collect(Collectors.toMap(Dimension::getName, Dimension::getValue));
+        String metricsJson = getCloudWatchEmbeddedMetricsJson(namespace, invokeTimeMs, dimensions, stats, gson);
+        System.out.println(metricsJson);
+    }
+
+
+    @Override
+    public List<StatFilter> getStatFilters() {
+        return this.statFilters;
+    }
+
+    /**
+     * Example of accepted format:
+     * {
+     *   "_aws": {
+     *     "Timestamp": 1574109732004,
+     *     "CloudWatchMetrics": [
+     *       {
+     *         "Namespace": "lambda-function-metrics",
+     *         "Dimensions": [["functionVersion"]],
+     *         "Metrics": [
+     *           {
+     *             "Name": "time",
+     *             "Unit": "Milliseconds"
+     *           }
+     *         ]
+     *       }
+     *     ]
+     *   },
+     *   "functionVersion": "$LATEST",
+     *   "time": 100,
+     *   "requestId": "989ffbf8-9ace-4817-a57c-e4dd734019ee"
+     * }
+     *
+     * Java code example: https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch_Embedded_Metric_Format_Generation_PutLogEvents.html
+     */
+    public static String getCloudWatchEmbeddedMetricsJson(String namespace,
+                                                          long timestamp,
+                                                          Map<String, String> dimensions,
+                                                          List<Stat> stats,
+                                                          Gson gson) {
+        Map<String, Object> embeddedMetricsObject = new HashMap<>();
+
+        /* first add nodes for each dimension in the embedded object since it'll be shared by all stats/metrics */
+        dimensions.forEach(embeddedMetricsObject::put);
+
+        /* Each stat is essentially a metric so iterate through each stat (metric).
+           We can be confident there are now duplicate stats since handler guarantees that
+         */
+        List<CloudWatchMetricObject> cwMetricObjects = new ArrayList<>();
+        stats.forEach(s -> {
+            /* each CloudWatchMetric object will have the overall dimensions at the operation level */
+            List<String> allDimensions = new ArrayList<>(dimensions.keySet());
+            List<Dimension> statDimensions = tagsToDimensions(s.getTags());
+            allDimensions.addAll(statDimensions.stream()
+                    .map(Dimension::getName)
+                    .collect(Collectors.toList()));
+
+            allDimensions = allDimensions.stream()
+                    .distinct()
+                    .collect(Collectors.toList());
+
+            /* add dimension name/value to main embedded metrics object */
+            statDimensions.forEach(d -> embeddedMetricsObject.put(d.getName(), d.getValue()));
+
+            /* add stat metric info as overall field */
+            embeddedMetricsObject.put(s.getName(), s.getValue());
+
+            /* each stat will have a unique CloudWatchMetric object */
+            cwMetricObjects.add(new CloudWatchMetricObject(namespace,
+                    Collections.singletonList(allDimensions),
+                    Collections.singletonList(new MetricPOJO(s.getName(), StandardUnit.None.toString())))
+            );
+
+        });
+
+        /* finally add overall node with list of CWMetric objects */
+        embeddedMetricsObject.put("_aws", new AWSMetricObject(timestamp, cwMetricObjects));
+
+        return gson.toJson(embeddedMetricsObject);
+    }
+
+}

reporters/src/main/java/com/nextdoor/bender/monitoring/embedded/metrics/CloudWatchEmbeddedMetricsReporterConfig.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ * Copyright 2017 Nextdoor.com, Inc
+ *
+ */
+
+package com.nextdoor.bender.monitoring.embedded.metrics;
+
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaDefault;
+import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaDescription;
+import com.nextdoor.bender.monitoring.RegionalReporterConfig;
+
+@JsonTypeName("CloudWatchEmbeddedMetrics")
+@JsonSchemaDescription("Writes metrics to Cloudwatch by printing to stdout. It is important to consider costs when "
+        + "using this reporter see https://aws.amazon.com/cloudwatch/pricing/.")
+public class CloudWatchEmbeddedMetricsReporterConfig extends RegionalReporterConfig {
+    @JsonSchemaDefault("Nextdoor/bender")
+    @JsonSchemaDescription("Cloudwatch namespace to write metrics under.")
+    private String namespace = "Nextdoor/bender";
+
+    @Override
+    public Class<CloudWatchEmbeddedMetricsReporterFactory> getFactoryClass() {
+        return CloudWatchEmbeddedMetricsReporterFactory.class;
+    }
+
+    public String getNamespace() {
+        return namespace;
+    }
+
+    public void setNamespace(String namespace) {
+        this.namespace = namespace;
+    }
+}

reporters/src/main/java/com/nextdoor/bender/monitoring/embedded/metrics/CloudWatchEmbeddedMetricsReporterFactory.java

@@ -0,0 +1,42 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ * Copyright 2017 Nextdoor.com, Inc
+ *
+ */
+
+package com.nextdoor.bender.monitoring.embedded.metrics;
+
+import com.nextdoor.bender.config.AbstractConfig;
+import com.nextdoor.bender.monitoring.ReporterFactory;
+
+/**
+ * Crate a {@link CloudWatchEmbeddedMetricsReporter}.
+ */
+public class CloudWatchEmbeddedMetricsReporterFactory implements ReporterFactory {
+    private CloudWatchEmbeddedMetricsReporterConfig config;
+
+    @Override
+    public void setConf(AbstractConfig config) {
+        this.config = (CloudWatchEmbeddedMetricsReporterConfig) config;
+    }
+
+    @Override
+    public CloudWatchEmbeddedMetricsReporter newInstance() {
+        return new CloudWatchEmbeddedMetricsReporter(this.config.getNamespace(),
+                this.config.getStatFilters());
+    }
+
+    @Override
+    public Class<CloudWatchEmbeddedMetricsReporter> getChildClass() {
+        return CloudWatchEmbeddedMetricsReporter.class;
+    }
+}

reporters/src/main/java/com/nextdoor/bender/monitoring/embedded/metrics/CloudWatchMetricObject.java

@@ -0,0 +1,49 @@
+package com.nextdoor.bender.monitoring.embedded.metrics;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class CloudWatchMetricObject {
+    @SerializedName(value = "Namespace")
+    private String namespace;
+
+    @SerializedName(value = "Dimensions")
+    private List<List<String>> dimensions = new ArrayList<>();
+
+    @SerializedName(value = "Metrics")
+    private List<MetricPOJO> metrics = new ArrayList<>();
+
+    public CloudWatchMetricObject(String namespace,
+                                  List<List<String>> dimensions,
+                                  List<MetricPOJO> metrics) {
+        this.namespace = namespace;
+        this.dimensions = dimensions;
+        this.metrics = metrics;
+    }
+
+    public void setNamespace(String namespace) {
+        this.namespace = namespace;
+    }
+
+    public String getNamespace() {
+        return namespace;
+    }
+
+    public void setDimensions(List<List<String>> dimensions) {
+        this.dimensions = dimensions;
+    }
+
+    public List<List<String>> getDimensions() {
+        return dimensions;
+    }
+
+    public void setMetrics(List<MetricPOJO> metrics) {
+        this.metrics = metrics;
+    }
+
+    public List<MetricPOJO> getMetrics() {
+        return metrics;
+    }
+}