From 1e7cf1f240049491dfd316a1762e8ec5a5a743f3 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 11 Dec 2015 17:45:34 -0800 Subject: [PATCH 1/2] first pass at fixing mplot enum queries --- .../service/LocatorFetchRunnable.java | 1 + .../BatchedMetricsJSONOutputSerializer.java | 2 ++ .../serializers/FakeMetricDataGenerator.java | 11 +++++++ .../JSONBasicRollupsOutputSerializer.java | 19 +++++++----- ...atchedMetricsJSONOutputSerializerTest.java | 29 +++++++++++++++++++ .../JSONBasicRollupOutputSerializerTest.java | 28 ++++++++++++++++++ 6 files changed, 83 insertions(+), 7 deletions(-) diff --git a/blueflood-core/src/main/java/com/rackspacecloud/blueflood/service/LocatorFetchRunnable.java b/blueflood-core/src/main/java/com/rackspacecloud/blueflood/service/LocatorFetchRunnable.java index d692e952c..7c7d3557f 100644 --- a/blueflood-core/src/main/java/com/rackspacecloud/blueflood/service/LocatorFetchRunnable.java +++ b/blueflood-core/src/main/java/com/rackspacecloud/blueflood/service/LocatorFetchRunnable.java @@ -157,6 +157,7 @@ public void run() { if (executionContext.wasSuccessful()) { this.scheduleCtx.clearFromRunning(parentSlotKey); + log.info("Successful completion of rollups for (gran,slot,shard) {} in {}", new Object[] {parentSlotKey, System.currentTimeMillis() - waitStart}); } else { log.error("Performing BasicRollups for {} failed", parentSlotKey); this.scheduleCtx.pushBackToScheduled(parentSlotKey, false); diff --git a/blueflood-http/src/main/java/com/rackspacecloud/blueflood/outputs/serializers/BatchedMetricsJSONOutputSerializer.java b/blueflood-http/src/main/java/com/rackspacecloud/blueflood/outputs/serializers/BatchedMetricsJSONOutputSerializer.java index c5b98eeab..ba161f615 100644 --- a/blueflood-http/src/main/java/com/rackspacecloud/blueflood/outputs/serializers/BatchedMetricsJSONOutputSerializer.java +++ b/blueflood-http/src/main/java/com/rackspacecloud/blueflood/outputs/serializers/BatchedMetricsJSONOutputSerializer.java @@ -36,11 +36,13 @@ public JSONObject transformRollupData(Map metricData, Set one : metricData.entrySet()) { final JSONObject singleMetricJSON = new JSONObject(); singleMetricJSON.put("metric", one.getKey().getMetricName()); singleMetricJSON.put("unit", one.getValue().getUnit() == null ? Util.UNKNOWN : one.getValue().getUnit()); singleMetricJSON.put("type", one.getValue().getType()); + Set oneFilterStats = fixFilterStats(one.getValue(), filterStats); JSONArray values = transformDataToJSONArray(one.getValue(), filterStats); singleMetricJSON.put("data", values); metricsArray.add(singleMetricJSON); diff --git a/blueflood-http/src/main/java/com/rackspacecloud/blueflood/outputs/serializers/FakeMetricDataGenerator.java b/blueflood-http/src/main/java/com/rackspacecloud/blueflood/outputs/serializers/FakeMetricDataGenerator.java index 096779432..239f483a4 100644 --- a/blueflood-http/src/main/java/com/rackspacecloud/blueflood/outputs/serializers/FakeMetricDataGenerator.java +++ b/blueflood-http/src/main/java/com/rackspacecloud/blueflood/outputs/serializers/FakeMetricDataGenerator.java @@ -96,6 +96,17 @@ public static Points generateFakeCounterRollupPoints() { return points; } + public static Points generateFakeEnumRollupPoints() { + Points points = new Points(); + long startTime = 1234567L; + for (int i = 0; i < 5; i++) { + long timeNow = startTime + i*1000; + Points.Point point = new Points.Point(timeNow, new BluefloodEnumRollup().withEnumValue("enum_value_" + i)); + points.add(point); + } + return points; + } + public static Points generateFakeSetRollupPoints() { Points points = new Points(); long startTime = 1234567L; diff --git a/blueflood-http/src/main/java/com/rackspacecloud/blueflood/outputs/serializers/JSONBasicRollupsOutputSerializer.java b/blueflood-http/src/main/java/com/rackspacecloud/blueflood/outputs/serializers/JSONBasicRollupsOutputSerializer.java index ad310fc7f..ff856c3fb 100644 --- a/blueflood-http/src/main/java/com/rackspacecloud/blueflood/outputs/serializers/JSONBasicRollupsOutputSerializer.java +++ b/blueflood-http/src/main/java/com/rackspacecloud/blueflood/outputs/serializers/JSONBasicRollupsOutputSerializer.java @@ -31,13 +31,8 @@ public class JSONBasicRollupsOutputSerializer implements BasicRollupsOutputSerializer { private static final Logger log = LoggerFactory.getLogger(JSONBasicRollupsOutputSerializer.class); - - @Override - public JSONObject transformRollupData(MetricData metricData, Set filterStats) - throws SerializationException { - final JSONObject globalJSON = new JSONObject(); - final JSONObject metaObject = new JSONObject(); - + + protected Set fixFilterStats(MetricData metricData, Set filterStats) { // if no stats were entered, figure out what type we are dealing with and select out default stats. if (metricData.getData().getPoints().size() > 0 && filterStats == PlotRequestParser.DEFAULT_STATS) { Class dataClass = metricData.getData().getDataClass(); @@ -56,6 +51,16 @@ else if (dataClass.equals(BluefloodEnumRollup.class)) // else, I got nothing. } + return filterStats; + } + + @Override + public JSONObject transformRollupData(MetricData metricData, Set filterStats) + throws SerializationException { + final JSONObject globalJSON = new JSONObject(); + final JSONObject metaObject = new JSONObject(); + filterStats = fixFilterStats(metricData, filterStats); + final JSONArray valuesArray = transformDataToJSONArray(metricData, filterStats); metaObject.put("count", valuesArray.size()); diff --git a/blueflood-http/src/test/java/com/rackspacecloud/blueflood/outputs/serializers/BatchedMetricsJSONOutputSerializerTest.java b/blueflood-http/src/test/java/com/rackspacecloud/blueflood/outputs/serializers/BatchedMetricsJSONOutputSerializerTest.java index 7375ded56..87306972d 100644 --- a/blueflood-http/src/test/java/com/rackspacecloud/blueflood/outputs/serializers/BatchedMetricsJSONOutputSerializerTest.java +++ b/blueflood-http/src/test/java/com/rackspacecloud/blueflood/outputs/serializers/BatchedMetricsJSONOutputSerializerTest.java @@ -17,6 +17,7 @@ package com.rackspacecloud.blueflood.outputs.serializers; import com.rackspacecloud.blueflood.outputs.formats.MetricData; +import com.rackspacecloud.blueflood.outputs.utils.PlotRequestParser; import com.rackspacecloud.blueflood.types.Locator; import org.json.simple.JSONArray; import org.json.simple.JSONObject; @@ -65,4 +66,32 @@ public void testBatchedMetricsSerialization() throws Exception { Assert.assertTrue(data != null); } } + + @Test + public void testBatchedEnumMetricsSerialization() throws Exception { + final BatchedMetricsJSONOutputSerializer serializer = new BatchedMetricsJSONOutputSerializer(); + + final Map metrics = new HashMap(); + for (int i = 0; i < 2; i++) { + final MetricData metricData = new MetricData(FakeMetricDataGenerator.generateFakeEnumRollupPoints(), "unknown", + MetricData.Type.ENUM); + + metrics.put(Locator.createLocatorFromPathComponents(tenantId, String.valueOf(i)), metricData); + } + + JSONObject jsonMetrics = serializer.transformRollupData(metrics, PlotRequestParser.DEFAULT_STATS); + Assert.assertTrue(jsonMetrics.get("metrics") != null); + JSONArray jsonMetricsArray = (JSONArray) jsonMetrics.get("metrics"); + + Iterator metricsObjects = jsonMetricsArray.iterator(); + Assert.assertTrue(metricsObjects.hasNext()); + + while (metricsObjects.hasNext()) { + JSONObject singleMetricObject = metricsObjects.next(); + Assert.assertTrue(singleMetricObject.get("unit").equals("unknown")); + Assert.assertTrue(singleMetricObject.get("type").equals("enum")); + JSONArray data = (JSONArray) singleMetricObject.get("data"); + Assert.assertTrue(data != null); + } + } } diff --git a/blueflood-http/src/test/java/com/rackspacecloud/blueflood/outputs/serializers/JSONBasicRollupOutputSerializerTest.java b/blueflood-http/src/test/java/com/rackspacecloud/blueflood/outputs/serializers/JSONBasicRollupOutputSerializerTest.java index e5f503d9c..cd56b2fd2 100644 --- a/blueflood-http/src/test/java/com/rackspacecloud/blueflood/outputs/serializers/JSONBasicRollupOutputSerializerTest.java +++ b/blueflood-http/src/test/java/com/rackspacecloud/blueflood/outputs/serializers/JSONBasicRollupOutputSerializerTest.java @@ -31,6 +31,7 @@ import java.util.HashSet; import java.util.Set; +import java.util.Map; public class JSONBasicRollupOutputSerializerTest { private final Set filterStats; @@ -161,6 +162,33 @@ public void testCounters() throws Exception { } } + @Test + public void testEnums() throws Exception { + final JSONBasicRollupsOutputSerializer serializer = new JSONBasicRollupsOutputSerializer(); + final MetricData metricData = new MetricData( + FakeMetricDataGenerator.generateFakeEnumRollupPoints(), + "unknown", + MetricData.Type.ENUM); + JSONObject metricDataJSON = serializer.transformRollupData(metricData, PlotRequestParser.DEFAULT_STATS); + final JSONArray data = (JSONArray)metricDataJSON.get("values"); + + Assert.assertEquals(5, data.size()); + for (int i = 0; i < data.size(); i++) { + final JSONObject dataJSON = (JSONObject)data.get(i); + final Map evJSON = (Map)dataJSON.get("enum_values"); + Set keys = evJSON.keySet(); + + Assert.assertEquals(1, keys.size()); + for (String key : keys) { + Assert.assertEquals(key, "enum_value_" + i); + Assert.assertEquals((long)evJSON.get(key), 1); + } + Assert.assertNotNull(dataJSON.get("numPoints")); + Assert.assertEquals(1, dataJSON.get("numPoints")); + Assert.assertEquals(MetricData.Type.ENUM, dataJSON.get("type")); + } + } + @Test public void testGauges() throws Exception { final JSONBasicRollupsOutputSerializer serializer = new JSONBasicRollupsOutputSerializer(); From 21033d29c10a19e13e6cbcb38c385b0152f0f7b7 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Fri, 11 Dec 2015 22:47:20 -0800 Subject: [PATCH 2/2] fixed an integration that was wrong --- .../handlers/HttpMultiRollupsQueryHandlerIntegrationTest.java | 4 ++-- .../serializers/BatchedMetricsJSONOutputSerializer.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/blueflood-http/src/integration-test/java/com/rackspacecloud/blueflood/outputs/handlers/HttpMultiRollupsQueryHandlerIntegrationTest.java b/blueflood-http/src/integration-test/java/com/rackspacecloud/blueflood/outputs/handlers/HttpMultiRollupsQueryHandlerIntegrationTest.java index 03b0d2160..0db86f880 100644 --- a/blueflood-http/src/integration-test/java/com/rackspacecloud/blueflood/outputs/handlers/HttpMultiRollupsQueryHandlerIntegrationTest.java +++ b/blueflood-http/src/integration-test/java/com/rackspacecloud/blueflood/outputs/handlers/HttpMultiRollupsQueryHandlerIntegrationTest.java @@ -51,7 +51,7 @@ public void testMultiplotQuery() throws Exception { " {\n" + " \"numPoints\": 1,\n" + " \"timestamp\": 1382400000,\n" + - " \"average\": 397\n" + + " \"latest\": 397\n" + " }\n" + " ],\n" + " \"type\": \"number\"\n" + @@ -63,7 +63,7 @@ public void testMultiplotQuery() throws Exception { " {\n" + " \"numPoints\": 1,\n" + " \"timestamp\": 1382400000,\n" + - " \"average\": 56\n" + + " \"latest\": 56\n" + " }\n" + " ],\n" + " \"type\": \"number\"\n" + diff --git a/blueflood-http/src/main/java/com/rackspacecloud/blueflood/outputs/serializers/BatchedMetricsJSONOutputSerializer.java b/blueflood-http/src/main/java/com/rackspacecloud/blueflood/outputs/serializers/BatchedMetricsJSONOutputSerializer.java index ba161f615..d30d278d6 100644 --- a/blueflood-http/src/main/java/com/rackspacecloud/blueflood/outputs/serializers/BatchedMetricsJSONOutputSerializer.java +++ b/blueflood-http/src/main/java/com/rackspacecloud/blueflood/outputs/serializers/BatchedMetricsJSONOutputSerializer.java @@ -43,7 +43,7 @@ public JSONObject transformRollupData(Map metricData, Set oneFilterStats = fixFilterStats(one.getValue(), filterStats); - JSONArray values = transformDataToJSONArray(one.getValue(), filterStats); + JSONArray values = transformDataToJSONArray(one.getValue(), oneFilterStats); singleMetricJSON.put("data", values); metricsArray.add(singleMetricJSON); }