From a0b798f529084ea89ece77be7f08e6d7cfa27d4d Mon Sep 17 00:00:00 2001 From: Andrew Cholewa Date: Thu, 8 Dec 2016 15:55:35 -0600 Subject: [PATCH] Logs a compact description of druid filters. --A map is logged that describes the structure of the filter being sent to druid. For each filter it includes a count of the number of instances of that filter. --- .../druid/model/filter/ComplexFilter.java | 19 ++++++ .../druid/model/filter/NotFilter.java | 3 + .../logging/blocks/DruidFilterInfo.java | 59 +++++++++++++++++++ .../webservice/web/endpoints/DataServlet.java | 2 + .../logging/blocks/DruidFilterInfoSpec.groovy | 36 +++++++++++ 5 files changed, 119 insertions(+) create mode 100644 fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/filter/ComplexFilter.java create mode 100644 fili-core/src/main/java/com/yahoo/bard/webservice/logging/blocks/DruidFilterInfo.java create mode 100644 fili-core/src/test/groovy/com/yahoo/bard/webservice/logging/blocks/DruidFilterInfoSpec.groovy diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/filter/ComplexFilter.java b/fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/filter/ComplexFilter.java new file mode 100644 index 0000000000..bc92601b8a --- /dev/null +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/filter/ComplexFilter.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. +// Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. +package com.yahoo.bard.webservice.druid.model.filter; + +import java.util.List; + +/** + * A Druid filter that is defined by applying an operation to at least one other filter. For example, {@code not} and + * {@code and} filters are complex. A {@code selector} filter is not. + */ +public interface ComplexFilter { + + /** + * Returns the filters that are operated on by this filter. + * + * @return The filters operated on by this filter + */ + List getFields(); +} diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/filter/NotFilter.java b/fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/filter/NotFilter.java index bf41b47c59..4b8afd71cc 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/filter/NotFilter.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/druid/model/filter/NotFilter.java @@ -2,6 +2,8 @@ // Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. package com.yahoo.bard.webservice.druid.model.filter; +import com.fasterxml.jackson.annotation.JsonIgnore; + import java.util.Collections; import java.util.List; import java.util.Objects; @@ -53,6 +55,7 @@ public boolean equals(Object obj) { } @Override + @JsonIgnore public List getFields() { return Collections.singletonList(getField()); } diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/logging/blocks/DruidFilterInfo.java b/fili-core/src/main/java/com/yahoo/bard/webservice/logging/blocks/DruidFilterInfo.java new file mode 100644 index 0000000000..dbc370d214 --- /dev/null +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/logging/blocks/DruidFilterInfo.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. +// Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms. +package com.yahoo.bard.webservice.logging.blocks; + +import com.yahoo.bard.webservice.druid.model.filter.ComplexFilter; +import com.yahoo.bard.webservice.druid.model.filter.Filter; +import com.yahoo.bard.webservice.logging.LogInfo; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Deque; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Logs some structural data about the filter sent to Druid, without actually logging the entire (potentially massive) + * filter. + */ +@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) +public class DruidFilterInfo implements LogInfo { + + protected final Map numEachFilterType; + + /** + * Constructor. + * + * @param filter The filter that needs to be analyzed + */ + public DruidFilterInfo(Filter filter) { + numEachFilterType = buildFilterCount(filter); + } + + /** + * Performs a DFS search of the filter tree, populating the specified map with the number of instances of each + * filter type appearing in the filter. + * + * @param filter The filter that needs to be traversed + * + * @return A map containing a count of each type of filter + */ + private Map buildFilterCount(Filter filter) { + if (filter == null) { + return Collections.emptyMap(); + } + Map filterTypeCounter = new LinkedHashMap<>(); + Deque filterStack = new ArrayDeque<>(); + filterStack.add(filter); + while (!filterStack.isEmpty()) { + Filter currentFilter = filterStack.pop(); + filterTypeCounter.merge(currentFilter.getClass().getSimpleName(), 1L, (old, increment) -> old + increment); + if (currentFilter instanceof ComplexFilter) { + filterStack.addAll(((ComplexFilter) currentFilter).getFields()); + } + } + return filterTypeCounter; + } +} diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/DataServlet.java b/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/DataServlet.java index 4454ad7401..ff71e8891a 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/DataServlet.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/web/endpoints/DataServlet.java @@ -32,6 +32,7 @@ import com.yahoo.bard.webservice.logging.RequestLog; import com.yahoo.bard.webservice.logging.blocks.BardQueryInfo; import com.yahoo.bard.webservice.logging.blocks.DataRequest; +import com.yahoo.bard.webservice.logging.blocks.DruidFilterInfo; import com.yahoo.bard.webservice.table.Table; import com.yahoo.bard.webservice.util.Either; import com.yahoo.bard.webservice.web.DataApiRequest; @@ -407,6 +408,7 @@ public void getData( RequestLog.switchTiming("logRequestMetrics"); logRequestMetrics(apiRequest, readCache, druidQuery); + RequestLog.record(new DruidFilterInfo(apiRequest.getFilter())); RequestLog.switchTiming(REQUEST_WORKFLOW_TIMER); // Process the request diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/logging/blocks/DruidFilterInfoSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/logging/blocks/DruidFilterInfoSpec.groovy new file mode 100644 index 0000000000..1215c6daab --- /dev/null +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/logging/blocks/DruidFilterInfoSpec.groovy @@ -0,0 +1,36 @@ +package com.yahoo.bard.webservice.logging.blocks + +import com.yahoo.bard.webservice.data.dimension.Dimension +import com.yahoo.bard.webservice.druid.model.filter.AndFilter +import com.yahoo.bard.webservice.druid.model.filter.Filter +import com.yahoo.bard.webservice.druid.model.filter.NotFilter +import com.yahoo.bard.webservice.druid.model.filter.OrFilter +import com.yahoo.bard.webservice.druid.model.filter.SelectorFilter + +import spock.lang.Shared +import spock.lang.Specification +import spock.lang.Unroll + +class DruidFilterInfoSpec extends Specification { + + @Shared Filter selector = new SelectorFilter(Stub(Dimension), "value") + + static final String SELECTOR_NAME = SelectorFilter.class.simpleName + static final String AND_NAME = AndFilter.class.simpleName + static final String OR_NAME = OrFilter.class.simpleName + static final String NOT_NAME = NotFilter.class.simpleName + + @Unroll + def "The DruidFilterInfo correctly analyzes #filter"() { + expect: + new DruidFilterInfo(filter).numEachFilterType == expectedMap + + where: + filter | expectedMap + null | [:] + selector | [(SELECTOR_NAME): 1L] + new AndFilter([selector, selector]) | [(AND_NAME): 1L, (SELECTOR_NAME): 2L] + new NotFilter(new AndFilter([selector, selector])) | [(NOT_NAME): 1L, (AND_NAME): 1L, (SELECTOR_NAME): 2L] + new OrFilter([new AndFilter([selector, selector]), new NotFilter(new AndFilter([selector, selector]))]) | [(OR_NAME): 1L, (AND_NAME): 2L, (NOT_NAME): 1L, (SELECTOR_NAME): 4L] + } +}