Removing query result time meta

src/main/java/com/yahoo/bullet/common/BulletConfig.java

@@ -98,11 +98,11 @@ public class BulletConfig extends Config {
     public static final boolean DEFAULT_RESULT_METADATA_ENABLE = true;
     /** The mapping from Metadata concepts to the keys with which they should logged in the result metadata. */
     public static final List<Map<String, String>> DEFAULT_RESULT_METADATA_METRICS =
-        makeMetadata(ImmutablePair.of(Concept.QUERY_ID, "Query ID"),
-                     ImmutablePair.of(Concept.QUERY_BODY, "Query"),
-                     ImmutablePair.of(Concept.QUERY_RECEIVE_TIME, "Query Receive Time"),
-                     ImmutablePair.of(Concept.RESULT_EMIT_TIME, "Result Emit Time"),
-                     ImmutablePair.of(Concept.QUERY_FINISH_TIME, "Query Finish Time"),
+        makeMetadata(ImmutablePair.of(Concept.QUERY_METADATA, "Query"),
+                     ImmutablePair.of(Concept.QUERY_ID, "ID"),
+                     ImmutablePair.of(Concept.QUERY_BODY, "Body"),
+                     ImmutablePair.of(Concept.QUERY_RECEIVE_TIME, "Receive Time"),
+                     ImmutablePair.of(Concept.QUERY_FINISH_TIME, "Finish Time"),
                      ImmutablePair.of(Concept.SKETCH_METADATA, "Sketch"),
                      ImmutablePair.of(Concept.SKETCH_ESTIMATED_RESULT, "Was Estimated"),
                      ImmutablePair.of(Concept.SKETCH_STANDARD_DEVIATIONS, "Standard Deviations"),

src/main/java/com/yahoo/bullet/querying/Querier.java

@@ -26,10 +26,12 @@
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.function.Consumer;
+
+import static com.yahoo.bullet.result.Meta.addIfNonNull;
 
 /**
  * This manages a {@link RunningQuery} that is currently being executed. It can {@link #consume(BulletRecord)} records for the
@@ -560,9 +562,7 @@ public boolean shouldBuffer() {
      */
     public Clip finish() {
         Clip result = getResult();
-        Meta meta = new Meta();
-        addMetadata(Concept.QUERY_FINISH_TIME, (k) -> meta.add(k, System.currentTimeMillis()));
-        result.add(meta);
+        addFinishTime(result.getMeta());
         return result;
     }
 
@@ -597,19 +597,22 @@ private BulletRecord addAdditionalFields(BulletRecord record) {
     }
 
     private Meta getResultMetadata() {
-        if (metaKeys.isEmpty()) {
+        String metaKey = getMetaKey();
+        if (metaKey == null) {
             return null;
         }
-        Meta meta = new Meta();
-        addMetadata(Concept.QUERY_ID, (k) -> meta.add(k, runningQuery.getId()));
-        addMetadata(Concept.QUERY_BODY, (k) -> meta.add(k, runningQuery.toString()));
-        addMetadata(Concept.QUERY_RECEIVE_TIME, (k) -> meta.add(k, runningQuery.getStartTime()));
-        addMetadata(Concept.RESULT_EMIT_TIME, (k) -> meta.add(k, System.currentTimeMillis()));
-        return meta;
+        Map<String, Object> meta = new HashMap<>();
+        addIfNonNull(meta, metaKeys, Concept.QUERY_ID, runningQuery::getId);
+        addIfNonNull(meta, metaKeys, Concept.QUERY_BODY, runningQuery::toString);
+        addIfNonNull(meta, metaKeys, Concept.QUERY_RECEIVE_TIME, runningQuery::getStartTime);
+        return new Meta().add(metaKey, meta);
     }
 
-    private void addMetadata(Concept concept, Consumer<String> action) {
-        Meta.consumeRegisteredConcept(concept, metaKeys, action);
+    private void addFinishTime(Meta meta) {
+        Map<String, Object> queryMeta = (Map<String, Object>) meta.asMap().get(getMetaKey());
+        if (queryMeta != null) {
+            addIfNonNull(queryMeta, metaKeys, Concept.QUERY_FINISH_TIME, System::currentTimeMillis);
+        }
     }
 
     private boolean isLastWindow() {
@@ -631,4 +634,8 @@ private void incrementRate() {
             rateLimit.increment();
         }
     }
+
+    private String getMetaKey() {
+        return metaKeys.getOrDefault(Meta.Concept.QUERY_METADATA.getName(), null);
+    }
 }

src/main/java/com/yahoo/bullet/result/Meta.java

@@ -24,8 +24,9 @@ public class Meta {
 
     @Getter
     public enum Concept {
+        // Query metadata
+        QUERY_METADATA("Query Metadata"),
         QUERY_RECEIVE_TIME("Query Receive Time"),
-        RESULT_EMIT_TIME("Result Emit Time"),
         QUERY_FINISH_TIME("Query Finish Time"),
         QUERY_ID("Query ID"),
         QUERY_BODY("Query Body"),

src/main/resources/bullet_defaults.yaml

@@ -121,10 +121,10 @@ bullet.query.aggregation.top.k.sketch.error.type: "NFN"
 bullet.result.metadata.enable: true
 
 # Each entry in this list indicates which metadata to collect (the name) and what key to add it as (the key) to the meta
+# Query Metadata adds additional nested metadata about the query if set. These are listed below.
 # Query Identifier adds the ID that was generated for the query.
 # Query Body adds the received query definition. This is useful for diagnosing syntax exceptions when errors are received.
 # Query Receive Time adds the timestamp in milliseconds when the query was received.
-# Result Emit Time adds the timestamp in milliseconds when the result was emitted.
 # Query Finish Time adds the timestamp in milliseconds when the final result was emitted.
 
 # Sketch Metadata adds additional nested metadata about sketches if set. These are listed below.
@@ -143,17 +143,25 @@ bullet.result.metadata.enable: true
 # Maximum Count Error adds the error of the FrequentItems Sketch as a long. This will be the distance between the upper
 #                     bound and lower bound of a count for an item. It will be 0 if the count is exact. (TOP K)
 # Active Items adds the number of items being tracked in the the FrequentItems Sketch at the time of the query completion. (TOP K)
+
+# Window Metadata adds additional nested metadata about the window if set. These are listed below.
+# Name adds the type of the window used for the query.
+# Number adds the number of the window from the start of the query - e.g. 253 means this is the 253rd window.
+# Size adds the number of records in this window. This only applies for record based windows.
+# Emit Time adds the time when the window was emitted. This replaces the old Result Receive Time concept.
+# Expected Emit Time adds the time when the window was supposed to be emitted. This should be <= the Window Emit Time.
+#                    The difference lets you know how long the window was buffered. This only applies for time windows.
 bullet.result.metadata.metrics:
+    - name: "Query Metadata"
+      key: "Query"
     - name: "Query ID"
-      key: "Query ID"
+      key: "ID"
     - name: "Query Body"
-      key: "Query"
+      key: "Body"
     - name: "Query Receive Time"
-      key: "Query Receive Time"
-    - name: "Result Emit Time"
-      key: "Result Emit Time"
+      key: "Receive Time"
     - name: "Query Finish Time"
-      key: "Query Finish Time"
+      key: "Finish Time"
     - name: "Sketch Metadata"
       key: "Sketch"
     - name: "Sketch Estimated Result"

src/test/java/com/yahoo/bullet/parsing/LogicalClauseTest.java

@@ -34,6 +34,8 @@ public void testDefaults() {
     public void testConfigure() {
         LogicalClause logicalClause = new LogicalClause();
         logicalClause.setOperation(AND);
+        logicalClause.configure(new BulletConfig());
+
         FilterClause filterClause =  new FilterClause();
         filterClause.setField("id");
         filterClause.setOperation(REGEX_LIKE);

src/test/java/com/yahoo/bullet/querying/QuerierTest.java

@@ -244,28 +244,50 @@ public void testQueryAsString() {
         Assert.assertEquals(querier.toString(), "ahdf3 : " + query.toString());
     }
 
+    @Test
+    public void testDisabledQueryMeta() {
+        BulletConfig defaults = new BulletConfig();
+        defaults.set(BulletConfig.RESULT_METADATA_METRICS, emptyMap());
+
+        Querier querier = make("{'aggregation' : {}}", defaults);
+        Assert.assertTrue(querier.getMetadata().asMap().isEmpty());
+
+        Clip result = querier.finish();
+        Assert.assertNotNull(result);
+        Assert.assertTrue(result.getRecords().isEmpty());
+        Assert.assertTrue(result.getMeta().asMap().isEmpty());
+    }
+
     @Test
     public void testTimes() {
+        BulletConfig defaults = new BulletConfig();
+        Map<String, String> names = (Map<String, String>) defaults.get(BulletConfig.RESULT_METADATA_METRICS);
+
         long startTime = System.currentTimeMillis();
         Querier querier = make("{'aggregation' : {}}", emptyMap());
         Map<String, Object> meta = querier.getMetadata().asMap();
-        long creationTime = ((Number) meta.get(Meta.Concept.QUERY_RECEIVE_TIME.getName())).longValue();
-        long resultTime = ((Number) meta.get(Meta.Concept.RESULT_EMIT_TIME.getName())).longValue();
+        Assert.assertEquals(meta.size(), 2);
+        Map<String, Object> queryMeta = (Map<String, Object>) meta.get(names.get(Meta.Concept.QUERY_METADATA.getName()));
+        Map<String, Object> windowMeta = (Map<String, Object>) meta.get(names.get(Meta.Concept.WINDOW_METADATA.getName()));
+        long creationTime = ((Number) queryMeta.get(names.get(Meta.Concept.QUERY_RECEIVE_TIME.getName()))).longValue();
+        long emitTime = ((Number) windowMeta.get(names.get(Meta.Concept.WINDOW_EMIT_TIME.getName()))).longValue();
         long endTime = System.currentTimeMillis();
         Assert.assertTrue(creationTime >= startTime && creationTime <= endTime);
-        Assert.assertTrue(resultTime >= creationTime && resultTime <= endTime);
+        Assert.assertTrue(emitTime >= startTime && emitTime <= endTime);
 
         Clip finalResult = querier.finish();
         Assert.assertNotNull(finalResult);
         Assert.assertTrue(finalResult.getRecords().isEmpty());
 
         meta = finalResult.getMeta().asMap();
-        creationTime = ((Number) meta.get(Meta.Concept.QUERY_RECEIVE_TIME.getName())).longValue();
-        resultTime = ((Number) meta.get(Meta.Concept.RESULT_EMIT_TIME.getName())).longValue();
-        long finishTime = ((Number) meta.get(Meta.Concept.QUERY_FINISH_TIME.getName())).longValue();
+        queryMeta = (Map<String, Object>) meta.get(names.get(Meta.Concept.QUERY_METADATA.getName()));
+        windowMeta = (Map<String, Object>) meta.get(names.get(Meta.Concept.WINDOW_METADATA.getName()));
+        creationTime = ((Number) queryMeta.get(names.get(Meta.Concept.QUERY_RECEIVE_TIME.getName()))).longValue();
+        emitTime = ((Number) windowMeta.get(names.get(Meta.Concept.WINDOW_EMIT_TIME.getName()))).longValue();
+        long finishTime = ((Number) queryMeta.get(names.get(Meta.Concept.QUERY_FINISH_TIME.getName()))).longValue();
         endTime = System.currentTimeMillis();
         Assert.assertTrue(creationTime >= startTime && creationTime <= finishTime);
-        Assert.assertTrue(resultTime >= startTime && resultTime <= finishTime);
+        Assert.assertTrue(emitTime >= startTime && emitTime <= finishTime);
         Assert.assertTrue(finishTime <= endTime);
     }