diff --git a/CHANGELOG.md b/CHANGELOG.md index 008b9cbd25..e48a0efd5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,23 +10,34 @@ Current ### Added: +- [QueryTimeLookup Functionality Testing](https://github.com/yahoo/fili/pull/34) + * Added two tests `LookupDimensionFilteringDataServeletSpec` and `LookupDimensionGroupingDataServletSpec` to test QTL functionality + - [Enrich the ApiJobStore interface] (https://github.com/yahoo/fili/pull/23) * `ApiJobStore` Interface now supports filtering `JobRows` in the store * Added support for filtering JobRows in `HashJobStore` * Added `JobRowFilter` to hold filter information -- [Lookup Dimension Serializer]() +- [Lookup Dimension Serializer](https://github.com/yahoo/fili/pull/31) * Created `LookupDimensionToDimensionSpec` serializer for `LookupDimension` * Created corresponding tests for `LookupDimensionToDimensionSpec` in `LookupDimensionToDimensionSpecSpec` + ### Deprecated: +- [QueryTimeLookup Functionality Testing](https://github.com/yahoo/fili/pull/34) + * Deprecated `KeyValueDimensionLoader`, in favor of `TypeAwareDimensionLoader` + ### Removed: ### Changed: +- [QueryTimeLookup Functionality Testing](https://github.com/yahoo/fili/pull/34) + * `AbstractBinderFactory` now uses `TypeAwareDimensionLoader` instead of `KeyValueStoreDimensionLoader` + * Modified some testing resources (PETS table and corresponding dimensions) to allow better testing on `LookupDimension`s + - [Reorganizes asynchronous package structure](https://github.com/yahoo/fili/pull/19) * The `jobs` package is renamed to `async` and split into the following subpackages: - `broadcastchannels` - Everything dealing with broadcast channels diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/application/AbstractBinderFactory.java b/fili-core/src/main/java/com/yahoo/bard/webservice/application/AbstractBinderFactory.java index abd59397b9..c9d5b9dd0d 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/application/AbstractBinderFactory.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/application/AbstractBinderFactory.java @@ -35,7 +35,7 @@ import com.yahoo.bard.webservice.data.config.ResourceDictionaries; import com.yahoo.bard.webservice.data.config.dimension.DimensionConfig; import com.yahoo.bard.webservice.data.config.dimension.DimensionLoader; -import com.yahoo.bard.webservice.data.config.dimension.KeyValueStoreDimensionLoader; +import com.yahoo.bard.webservice.data.config.dimension.TypeAwareDimensionLoader; import com.yahoo.bard.webservice.data.config.metric.MetricLoader; import com.yahoo.bard.webservice.data.config.table.TableLoader; import com.yahoo.bard.webservice.data.dimension.DimensionDictionary; @@ -837,7 +837,7 @@ protected ConfigurationLoader buildConfigurationLoader( * @return a Dimension Loader instance */ protected DimensionLoader getDimensionLoader() { - return new KeyValueStoreDimensionLoader(getDimensionConfigurations()); + return new TypeAwareDimensionLoader(getDimensionConfigurations()); } /** diff --git a/fili-core/src/main/java/com/yahoo/bard/webservice/data/config/dimension/KeyValueStoreDimensionLoader.java b/fili-core/src/main/java/com/yahoo/bard/webservice/data/config/dimension/KeyValueStoreDimensionLoader.java index 698a21e545..9d9c73c5ff 100644 --- a/fili-core/src/main/java/com/yahoo/bard/webservice/data/config/dimension/KeyValueStoreDimensionLoader.java +++ b/fili-core/src/main/java/com/yahoo/bard/webservice/data/config/dimension/KeyValueStoreDimensionLoader.java @@ -12,7 +12,10 @@ /** * Load the dimensions along with its keyValueStore and searchProvider. + * + * @deprecated in favor of TypeAwareDimensionLoader which loads corresponding dimension based on config type */ +@Deprecated public class KeyValueStoreDimensionLoader implements DimensionLoader { private Iterable configSource; diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/application/AbstractBinderFactorySpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/application/AbstractBinderFactorySpec.groovy index aa65cdc174..96b57e31d4 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/application/AbstractBinderFactorySpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/application/AbstractBinderFactorySpec.groovy @@ -32,7 +32,7 @@ import com.yahoo.bard.webservice.data.cache.StubDataCache import com.yahoo.bard.webservice.data.cache.TupleDataCache import com.yahoo.bard.webservice.data.config.ConfigurationLoader import com.yahoo.bard.webservice.data.config.dimension.DimensionConfig -import com.yahoo.bard.webservice.data.config.dimension.KeyValueStoreDimensionLoader +import com.yahoo.bard.webservice.data.config.dimension.TypeAwareDimensionLoader import com.yahoo.bard.webservice.data.config.dimension.TestDimensions import com.yahoo.bard.webservice.data.config.metric.MetricLoader import com.yahoo.bard.webservice.data.config.table.TableLoader @@ -171,7 +171,7 @@ public class AbstractBinderFactorySpec extends Specification { cl.dictionaries != null cl.tableLoader != null cl.metricLoader != null - cl.dimensionLoader instanceof KeyValueStoreDimensionLoader + cl.dimensionLoader instanceof TypeAwareDimensionLoader } def "test configure bindings"() { @@ -436,7 +436,7 @@ public class AbstractBinderFactorySpec extends Specification { ConfigurationLoader cl = localBinderFactory.buildConfigurationLoader() then: - cl.dimensionLoader instanceof KeyValueStoreDimensionLoader + cl.dimensionLoader instanceof TypeAwareDimensionLoader cl.metricLoader == metricLoader cl.tableLoader == tableLoader } diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/data/config/dimension/TypeAwareDimensionLoaderSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/data/config/dimension/TypeAwareDimensionLoaderSpec.groovy index 9a71f0168d..f12b3ec0f1 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/data/config/dimension/TypeAwareDimensionLoaderSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/data/config/dimension/TypeAwareDimensionLoaderSpec.groovy @@ -12,25 +12,23 @@ import spock.lang.Specification */ class TypeAwareDimensionLoaderSpec extends Specification { - LinkedHashSet lookupDimensionConfigurations LinkedHashSet dimensionConfigurations DimensionDictionary dimensionDictionary def setup() { - lookupDimensionConfigurations = new TestLookupDimensions().getAllDimensionConfigurations() dimensionConfigurations = new TestDimensions().getAllDimensionConfigurations() dimensionDictionary = new DimensionDictionary() } def "Test dimension loader for Lookup dimension"() { given: "A Type Aware Dimension Loader with a list of dimension configurations" - TypeAwareDimensionLoader typeAwareDimensionLoader = new TypeAwareDimensionLoader(lookupDimensionConfigurations) + TypeAwareDimensionLoader typeAwareDimensionLoader = new TypeAwareDimensionLoader(dimensionConfigurations) when: typeAwareDimensionLoader.loadDimensionDictionary(dimensionDictionary) then: - dimensionDictionary.findByApiName("size").getClass() == LookupDimension.class + dimensionDictionary.findByApiName("breed").getClass() == LookupDimension.class } def "Test dimension loader for KeyValueStore dimension"() { @@ -41,7 +39,7 @@ class TypeAwareDimensionLoaderSpec extends Specification { typeAwareDimensionLoader.loadDimensionDictionary(dimensionDictionary) then: - dimensionDictionary.findByApiName("breed").getClass() == KeyValueStoreDimension.class + dimensionDictionary.findByApiName("color").getClass() == KeyValueStoreDimension.class } def "Test dimension loader for a dimension type that is not defined"() { @@ -49,8 +47,8 @@ class TypeAwareDimensionLoaderSpec extends Specification { DimensionConfig dimensionConfiguration = Mock(DimensionConfig) dimensionConfiguration.getApiName() >> "foo" dimensionConfiguration.getType() >> String.class - lookupDimensionConfigurations.add(dimensionConfiguration) - TypeAwareDimensionLoader typeAwareDimensionLoader = new TypeAwareDimensionLoader(lookupDimensionConfigurations) + dimensionConfigurations.add(dimensionConfiguration) + TypeAwareDimensionLoader typeAwareDimensionLoader = new TypeAwareDimensionLoader(dimensionConfigurations) when: typeAwareDimensionLoader.loadDimensionDictionary(dimensionDictionary) diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/TableFullViewProcessorSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/TableFullViewProcessorSpec.groovy index 273f1ad6a6..d8323165f2 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/TableFullViewProcessorSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/TableFullViewProcessorSpec.groovy @@ -68,14 +68,14 @@ class TableFullViewProcessorSpec extends Specification { def "Check table meta data info at grain level"() { setup: - String expectedResponse = "[name:all, description:The pets all grain, metrics:[[category:General, name:rowNum, longName:rowNum, uri:http://localhost:9998/v1/rowNum], [category:General, name:limbs, longName:limbs, uri:http://localhost:9998/v1/limbs], [category:General, name:dayAvgLimbs, longName:dayAvgLimbs, uri:http://localhost:9998/v1/dayAvgLimbs]], retention:P1Y, longName:All, dimensions:[[category:General, name:species, longName:species, uri:http://localhost:9998/v1/species, cardinality:0], [category:General, name:sex, longName:sex, uri:http://localhost:9998/v1/sex, cardinality:0], [category:General, name:breed, longName:breed, uri:http://localhost:9998/v1/breed, cardinality:0]]]" + String expectedResponse = "[description:The pets all grain, dimensions:[[category:General, name:sex, longName:sex, uri:http://localhost:9998/v1/sex, cardinality:0], [category:General, name:species, longName:species, uri:http://localhost:9998/v1/species, cardinality:0], [category:General, name:breed, longName:breed, uri:http://localhost:9998/v1/breed, cardinality:0]], longName:All, metrics:[[category:General, name:rowNum, longName:rowNum, uri:http://localhost:9998/v1/rowNum], [category:General, name:limbs, longName:limbs, uri:http://localhost:9998/v1/limbs], [category:General, name:dayAvgLimbs, longName:dayAvgLimbs, uri:http://localhost:9998/v1/dayAvgLimbs]], name:all, retention:P1Y]" LogicalTable petsTable = petsShapesTables.find {it.getName() == "pets"} when: TableGrainView tableGrainView = fullViewProcessor.formatTableGrain(petsTable, "all", uriInfo) then: - tableGrainView.toString() == expectedResponse + tableGrainView.sort().toString() == expectedResponse } def "check all the tables full view at grain level"() { diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/BaseDataServletComponentSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/BaseDataServletComponentSpec.groovy index 6103c33005..96ae1d6929 100644 --- a/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/BaseDataServletComponentSpec.groovy +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/BaseDataServletComponentSpec.groovy @@ -2,14 +2,13 @@ // Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms. package com.yahoo.bard.webservice.web.endpoints -import static com.yahoo.bard.webservice.util.JsonSortStrategy.SORT_BOTH - import com.yahoo.bard.webservice.application.JerseyTestBinder import com.yahoo.bard.webservice.data.time.GranularityParser import com.yahoo.bard.webservice.data.time.StandardGranularityParser import com.yahoo.bard.webservice.models.druid.client.impl.TestDruidWebService import com.yahoo.bard.webservice.table.availability.AvailabilityTestingUtils import com.yahoo.bard.webservice.util.GroovyTestUtils +import com.yahoo.bard.webservice.util.JsonSortStrategy import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.datatype.jdk8.Jdk8Module @@ -106,7 +105,7 @@ abstract class BaseDataServletComponentSpec extends Specification { } // Most aspects of a Druid query are order-independent, so normalize both lists and maps. - GroovyTestUtils.compareJson(jsonQuery, getExpectedDruidQuery(), SORT_BOTH) + compareResult(jsonQuery, getExpectedDruidQuery(), JsonSortStrategy.SORT_BOTH) } @Timeout(10) @@ -135,8 +134,8 @@ abstract class BaseDataServletComponentSpec extends Specification { validateJson(expectedApiResponse) } - boolean compareResult(String result, String expectedResult) { - GroovyTestUtils.compareJson(result, expectedResult) + boolean compareResult(String result, String expectedResult, JsonSortStrategy sortStrategy = JsonSortStrategy.SORT_MAPS) { + GroovyTestUtils.compareJson(result, expectedResult, sortStrategy) } /** diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/LookupDimensionGroupingDataServletSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/LookupDimensionGroupingDataServletSpec.groovy new file mode 100644 index 0000000000..6a1d4c21eb --- /dev/null +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/LookupDimensionGroupingDataServletSpec.groovy @@ -0,0 +1,179 @@ +package com.yahoo.bard.webservice.web.endpoints + +import com.yahoo.bard.webservice.data.dimension.BardDimensionField +import com.yahoo.bard.webservice.data.dimension.DimensionDictionary +import com.yahoo.bard.webservice.util.GroovyTestUtils +import com.yahoo.bard.webservice.util.JsonSortStrategy + +/** + * Testing Query Time Lookup Functionality with Lookup Dimensions in GroupBy query. + */ +class LookupDimensionGroupingDataServletSpec extends BaseDataServletComponentSpec{ + + @Override + def setup() { + DimensionDictionary dimensionStore = jtb.configurationLoader.dimensionDictionary + dimensionStore.findByApiName("sex").with { + addDimensionRow(BardDimensionField.makeDimensionRow(it, "sex1", "sex1Desc")) + addDimensionRow(BardDimensionField.makeDimensionRow(it, "sex2", "sex2Desc")) + addDimensionRow(BardDimensionField.makeDimensionRow(it, "sex3", "sex3Desc")) + } + dimensionStore.findByApiName("breed").with { + addDimensionRow(BardDimensionField.makeDimensionRow(it, "breed1", "breed1Desc")) + addDimensionRow(BardDimensionField.makeDimensionRow(it, "breed2", "breed2Desc")) + addDimensionRow(BardDimensionField.makeDimensionRow(it, "breed3", "breed3Desc")) + } + dimensionStore.findByApiName("species").with { + addDimensionRow(BardDimensionField.makeDimensionRow(it, "species1", "species1Desc")) + addDimensionRow(BardDimensionField.makeDimensionRow(it, "species2", "species2Desc")) + addDimensionRow(BardDimensionField.makeDimensionRow(it, "species3", "species3Desc")) + } + } + + @Override + Class[] getResourceClasses() { + [DataServlet.class] + } + + @Override + String getTarget() { + return "data/pets/day/sex/breed/species" + } + + @Override + Map> getQueryParams() { + [ + "metrics" : ["limbs"], + "dateTime": ["2014-05-01%2F2014-05-02"] + ] + } + + @Override + boolean compareResult(String result, String expectedResult, JsonSortStrategy sortStrategy) { + // Override "test druid query" feature test's compareResult which calls with SORT_BOTH to be default SORT_MAP + GroovyTestUtils.compareJson(result, expectedResult) + } + + @Override + String getExpectedDruidQuery() { + """{ + "aggregations": [{ "fieldName": "limbs", "name": "limbs", "type": "longSum" }], + "context": {}, + "dataSource": { "name": "all_pets", "type": "table" }, + "dimensions": [ + "sex", + { + "dimension": "breed", + "extractionFn": { + "extractionFns": [ + { + "injective": false, + "lookup": { "namespace": "NAMESPACE1", "type": "namespace" }, + "optimize": true, + "replaceMissingValueWith": "Unknown NAMESPACE1", + "retainMissingValue": false, "type": "lookup" + }, + { + "injective": false, + "lookup": { "namespace": "NAMESPACE2", "type": "namespace" }, + "optimize": true, + "replaceMissingValueWith": "Unknown NAMESPACE2", + "retainMissingValue": false, "type": "lookup" + } + ], + "type": "cascade" + }, + "outputName": "breed", "type": "extraction" + }, + { + "dimension": "class", + "extractionFn": { + "injective": false, + "lookup": { "namespace": "NAMESPACE1", "type": "namespace" }, + "optimize": true, + "replaceMissingValueWith": "Unknown NAMESPACE1", "retainMissingValue": false, "type": "lookup" + }, + "outputName": "species", + "type": "extraction" + } + ], + "granularity": { "period": "P1D", "timeZone": "UTC", "type": "period" }, + "intervals": ["2014-05-01T00:00:00.000Z/2014-05-02T00:00:00.000Z"], + "postAggregations": [], + "queryType": "groupBy" + }""" + } + + @Override + String getFakeDruidResponse() { + """[ + { + "version" : "v1", + "timestamp" : "2014-05-01T00:00:00.000Z", + "event" : { + "sex" : "sex1", + "breed" : "breed1", + "species" : "species1", + "limbs" : 1 + } + }, + { + "version" : "v1", + "timestamp" : "2014-05-01T00:00:00.000Z", + "event" : { + "sex" : "sex2", + "breed" : "breed2", + "species" : "species2", + "limbs" : 2 + } + }, + { + "version" : "v1", + "timestamp" : "2014-05-01T00:00:00.000Z", + "event" : { + "sex" : "sex3", + "breed" : "breed3", + "species" : "species3", + "limbs" : 3 + } + } + ]""" + } + + @Override + String getExpectedApiResponse() { + """{ + "rows": [{ + "breed|desc": "breed1Desc", + "breed|id": "breed1", + "dateTime": "2014-05-01 00:00:00.000", + "limbs": 1, + "sex|desc": "sex1Desc", + "sex|id": "sex1", + "species|desc": "species1Desc", + "species|id": "species1" + }, + { + "breed|desc": "breed2Desc", + "breed|id": "breed2", + "dateTime": "2014-05-01 00:00:00.000", + "limbs": 2, + "sex|desc": "sex2Desc", + "sex|id": "sex2", + "species|desc": "species2Desc", + "species|id": "species2" + }, + { + "breed|desc": "breed3Desc", + "breed|id": "breed3", + "dateTime": "2014-05-01 00:00:00.000", + "limbs": 3, + "sex|desc": "sex3Desc", + "sex|id": "sex3", + "species|desc": "species3Desc", + "species|id": "species3" + }] + }""" + } +} + diff --git a/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/LookupDimensionGroupingFilteringDataServletSpec.groovy b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/LookupDimensionGroupingFilteringDataServletSpec.groovy new file mode 100644 index 0000000000..05aed82ab8 --- /dev/null +++ b/fili-core/src/test/groovy/com/yahoo/bard/webservice/web/endpoints/LookupDimensionGroupingFilteringDataServletSpec.groovy @@ -0,0 +1,289 @@ +package com.yahoo.bard.webservice.web.endpoints + +import com.yahoo.bard.webservice.data.dimension.BardDimensionField +import com.yahoo.bard.webservice.data.dimension.DimensionDictionary +import com.yahoo.bard.webservice.util.GroovyTestUtils +import com.yahoo.bard.webservice.util.JsonSortStrategy + +/** + * Testing Query Time Lookup with extraction filter on Lookup Dimensions. + */ +class LookupDimensionGroupingFilteringDataServletSpec extends BaseDataServletComponentSpec{ + + @Override + def setup() { + DimensionDictionary dimensionStore = jtb.configurationLoader.dimensionDictionary + dimensionStore.findByApiName("sex").with { + addDimensionRow(BardDimensionField.makeDimensionRow(it, "sex1", "sex1Desc")) + addDimensionRow(BardDimensionField.makeDimensionRow(it, "sex2", "sex2Desc")) + addDimensionRow(BardDimensionField.makeDimensionRow(it, "sex3", "sex3Desc")) + } + dimensionStore.findByApiName("breed").with { + addDimensionRow(BardDimensionField.makeDimensionRow(it, "breed1", "breed1Desc")) + addDimensionRow(BardDimensionField.makeDimensionRow(it, "breed2", "breed2Desc")) + addDimensionRow(BardDimensionField.makeDimensionRow(it, "breed3", "breed3Desc")) + } + dimensionStore.findByApiName("species").with { + addDimensionRow(BardDimensionField.makeDimensionRow(it, "species1", "species1Desc")) + addDimensionRow(BardDimensionField.makeDimensionRow(it, "species2", "species2Desc")) + addDimensionRow(BardDimensionField.makeDimensionRow(it, "species3", "species3Desc")) + } + } + + @Override + boolean compareResult(String result, String expectedResult, JsonSortStrategy sortStrategy) { + // Override "test druid query" feature test's compareResult which calls with SORT_BOTH to be default SORT_MAP + GroovyTestUtils.compareJson(result, expectedResult) + } + + @Override + Class[] getResourceClasses() { + [DataServlet.class] + } + + @Override + String getTarget() { + return "data/pets/day/sex/breed/species" + } + + @Override + Map> getQueryParams() { + [ + "metrics" : ["limbs"], + "dateTime": ["2014-05-01%2F2014-05-02"], + "filters" : [ + "sex|id-in[sex1,sex2]", + "breed|desc-in[breed1Desc,breed2Desc]", + "species|desc-notin[species3Desc]" + + ] + ] + } + + @Override + String getExpectedDruidQuery() { + """{ + "aggregations": [ + { "fieldName": "limbs", + "name": "limbs", + "type": "longSum" + } + ], + "context": {}, + "dataSource": { + "name": "all_pets", + "type": "table" + }, + "dimensions": [ + "sex", + { + "dimension": "breed", + "extractionFn": { + "extractionFns": [ + { + "injective": false, + "lookup": { "namespace": "NAMESPACE1", "type": "namespace" }, + "optimize": true, + "replaceMissingValueWith": "Unknown NAMESPACE1", + "retainMissingValue": false, + "type": "lookup" + }, + { + "injective": false, + "lookup": { "namespace": "NAMESPACE2", "type": "namespace" }, + "optimize": true, + "replaceMissingValueWith": "Unknown NAMESPACE2", + "retainMissingValue": false, + "type": "lookup" + } + ], + "type": "cascade" + }, + "outputName": "breed", + "type": "extraction" + }, + { + "dimension": "class", + "extractionFn": { + "injective": false, + "lookup": { "namespace": "NAMESPACE1", "type": "namespace" }, + "optimize": true, + "replaceMissingValueWith": "Unknown NAMESPACE1", + "retainMissingValue": false, + "type": "lookup" + }, + "outputName": "species", + "type": "extraction" + } + ], + "filter": { + "fields": [ + { + "fields": [ + { "dimension": "sex", + "type": "selector", + "value": "sex1" + }, + { + "dimension": "sex", "type": "selector", "value": "sex2" + } + ], + "type": "or" + }, + { + "fields": [ + { + "dimension": "breed", + "extractionFn": { + "extractionFns": [ + { + "injective": false, + "lookup": + { + "namespace": "NAMESPACE1", + "type": "namespace" + }, + "optimize": true, + "replaceMissingValueWith": "Unknown NAMESPACE1", + "retainMissingValue": false, + "type": "lookup" + }, + { + "injective": false, + "lookup": { + "namespace": "NAMESPACE2", + "type": "namespace" + }, + "optimize": true, + "replaceMissingValueWith": "Unknown NAMESPACE2", + "retainMissingValue": false, + "type": "lookup" + } + ], + "type": "cascade" + }, + "type": "extraction", + "value": "breed1" + }, + { + "dimension": "breed", + "extractionFn": { + "extractionFns": [ + { + "injective": false, + "lookup": { + "namespace": "NAMESPACE1", + "type": "namespace" + }, + "optimize": true, + "replaceMissingValueWith": "Unknown NAMESPACE1", + "retainMissingValue": false, + "type": "lookup" + }, + { + "injective": false, + "lookup": { + "namespace": "NAMESPACE2", + "type": "namespace" }, + "optimize": true, + "replaceMissingValueWith": "Unknown NAMESPACE2", + "retainMissingValue": false, + "type": "lookup" + } + ], + "type": "cascade" + }, + "type": "extraction", + "value": "breed2" + } + ], "type": "or" + }, + { + "field": { + "fields": [ + { + "dimension": "class", + "extractionFn": { + "injective": false, + "lookup": { + "namespace": "NAMESPACE1", + "type": "namespace" + }, + "optimize": true, + "replaceMissingValueWith": "Unknown NAMESPACE1", + "retainMissingValue": false, + "type": "lookup" + }, + "type": "extraction", + "value": "species3" + } + ], + "type": "or" + }, + "type": "not" + } + ], + "type": "and" + }, + "granularity": { + "period": "P1D", + "timeZone": "UTC", + "type": "period" }, + "intervals": ["2014-05-01T00:00:00.000Z/2014-05-02T00:00:00.000Z"], + "postAggregations": [], + "queryType": "groupBy" + }""" + } + + @Override + String getFakeDruidResponse() { + """[ + { + "version" : "v1", + "timestamp" : "2014-05-01T00:00:00.000Z", + "event" : { + "sex" : "sex1", + "breed" : "breed1", + "species" : "species1", + "limbs" : 1 + } + }, + { + "version" : "v1", + "timestamp" : "2014-05-01T00:00:00.000Z", + "event" : { + "sex" : "sex2", + "breed" : "breed2", + "species" : "species2", + "limbs" : 2 + } + } + ]""" + } + + @Override + String getExpectedApiResponse() { + """{ + "rows": [{ + "breed|desc": "breed1Desc", + "breed|id": "breed1", + "dateTime": "2014-05-01 00:00:00.000", + "limbs": 1, + "sex|desc": "sex1Desc", + "sex|id": "sex1", + "species|desc": "species1Desc", + "species|id": "species1" + }, + { + "breed|desc": "breed2Desc", + "breed|id": "breed2", + "dateTime": "2014-05-01 00:00:00.000", + "limbs": 2, + "sex|desc": "sex2Desc", + "sex|id": "sex2", + "species|desc": "species2Desc", + "species|id": "species2" + }] + }""" + } +} diff --git a/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestDimensionConfig.java b/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestDimensionConfig.java index 70353c400b..a4850b8df5 100644 --- a/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestDimensionConfig.java +++ b/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestDimensionConfig.java @@ -15,7 +15,7 @@ */ public class TestDimensionConfig implements DimensionConfig { private final TestApiDimensionName apiName; - private final String getPhysicalName; + private final String physicalName; private final String description; private LinkedHashSet fields; @@ -42,7 +42,7 @@ public TestDimensionConfig( LinkedHashSet defaultFields ) { this.apiName = apiName; - this.getPhysicalName = physicalName; + this.physicalName = physicalName; this.description = apiName.asName(); this.keyValueStore = keyValueStore; this.searchProvider = searchProvider; @@ -50,6 +50,36 @@ public TestDimensionConfig( this.defaultFields = defaultFields; } + //CHECKSTYLE:OFF + public TestDimensionConfig withApiName(TestApiDimensionName apiName) { + return new TestDimensionConfig(apiName, physicalName, keyValueStore, searchProvider, fields, defaultFields); + } + + public TestDimensionConfig withPhysicalName(String physicalName) { + return new TestDimensionConfig(apiName, physicalName, keyValueStore, searchProvider, fields, defaultFields); + } + + public TestDimensionConfig withKeyValueStore(KeyValueStore keyValueStore) { + return new TestDimensionConfig(apiName, physicalName, keyValueStore, searchProvider, fields, defaultFields); + } + + public TestDimensionConfig withSearchProvider(SearchProvider searchProvider) { + return new TestDimensionConfig(apiName, physicalName, keyValueStore, searchProvider, fields, defaultFields); + } + + public TestDimensionConfig withFields(LinkedHashSet fields) { + return new TestDimensionConfig(apiName, physicalName, keyValueStore, searchProvider, fields, defaultFields); + } + + public TestDimensionConfig withDefaultFields(LinkedHashSet defaultFields) { + return new TestDimensionConfig(apiName, physicalName, keyValueStore, searchProvider, fields, defaultFields); + } + //CHECKSTYLE:ON + + public TestApiDimensionName getApiNameEnum() { + return apiName; + } + @Override public String getApiName() { return apiName.asName(); @@ -67,7 +97,7 @@ public String getCategory() { @Override public String getPhysicalName() { - return getPhysicalName; + return physicalName; } @Override @@ -97,6 +127,6 @@ public SearchProvider getSearchProvider() { @Override public String toString() { - return "Dimension Config: apiName: " + apiName + " getPhysicalName: " + getPhysicalName + " fields: " + fields; + return "Dimension Config: apiName: " + apiName + " physicalName: " + physicalName + " fields: " + fields; } } diff --git a/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestDimensions.java b/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestDimensions.java index 8a407378df..8f03464f11 100644 --- a/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestDimensions.java +++ b/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestDimensions.java @@ -32,6 +32,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -73,9 +74,17 @@ public TestDimensions() { dimensionNameConfigs.put(MODEL, TestDimensions.buildStandardLuceneDimensionConfig(MODEL)); - dimensionNameConfigs.put(BREED, TestDimensions.buildStandardDimensionConfig(BREED)); - dimensionNameConfigs.put(SPECIES, TestDimensions.buildStandardDimensionConfig(SPECIES)); - dimensionNameConfigs.put(SEX, TestDimensions.buildStandardDimensionConfig(SEX)); + dimensionNameConfigs.put( + BREED, + TestDimensions.buildStandardLookupDimensionConfig(BREED, Arrays.asList("NAMESPACE1", "NAMESPACE2")) + ); + dimensionNameConfigs.put( + SPECIES, + TestDimensions.buildStandardLookupDimensionConfig(SPECIES, Arrays.asList("NAMESPACE1")) + .withPhysicalName("class") + ); + dimensionNameConfigs.put(SEX, + TestDimensions.buildStandardLookupDimensionConfig(SEX, Collections.emptyList())); tableDimensions = new HashMap<>(); tableDimensions.put(SHAPES, Utils.asLinkedHashSet(SIZE, SHAPE, COLOR, OTHER)); @@ -118,6 +127,21 @@ public static TestDimensionConfig buildStandardLuceneDimensionConfig(TestApiDime ); } + /** + * Builds a lookup dimension config with a namespace. + * + * @param dimensionName Name of the dimension to build. + * @param namespaces namespaces for lookup. + * + * @return the lookup dimension config + */ + public static TestLookupDimensionConfig buildStandardLookupDimensionConfig( + TestApiDimensionName dimensionName, + List namespaces + ) { + return new TestLookupDimensionConfig(buildStandardDimensionConfig(dimensionName), namespaces); + } + /** * Get the default KeyValueStore for the given store name. * diff --git a/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestLookupDimensionConfig.java b/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestLookupDimensionConfig.java index 072827c16c..520c93bde4 100644 --- a/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestLookupDimensionConfig.java +++ b/fili-core/src/test/java/com/yahoo/bard/webservice/data/config/dimension/TestLookupDimensionConfig.java @@ -3,7 +3,11 @@ package com.yahoo.bard.webservice.data.config.dimension; import com.yahoo.bard.webservice.data.config.names.TestApiDimensionName; +import com.yahoo.bard.webservice.data.dimension.DimensionField; +import com.yahoo.bard.webservice.data.dimension.KeyValueStore; +import com.yahoo.bard.webservice.data.dimension.SearchProvider; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; @@ -21,24 +25,39 @@ public class TestLookupDimensionConfig extends TestDimensionConfig implements Lo * @param dimensionConfig Configuration properties for dimensions * @param namespaces List of namespaces used for Lookup */ - public TestLookupDimensionConfig (@NotNull DimensionConfig dimensionConfig, List namespaces) { - super( + public TestLookupDimensionConfig(@NotNull DimensionConfig dimensionConfig, List namespaces) { + this( TestApiDimensionName.valueOf(dimensionConfig.getApiName().toUpperCase(Locale.ENGLISH)), dimensionConfig.getPhysicalName(), dimensionConfig.getKeyValueStore(), dimensionConfig.getSearchProvider(), dimensionConfig.getFields(), - dimensionConfig.getDefaultDimensionFields() + dimensionConfig.getDefaultDimensionFields(), + namespaces ); - this.setNamespaces(namespaces); } /** - * Sets the namespace chain. + * Constructor. * - * @param namespaces The list of namespaces + * @param apiName API Name of the dimension + * @param physicalName Physical name of the dimension + * @param keyValueStore KeyValueStore for the dimension + * @param searchProvider SearchProvider for the dimension + * @param fields Fields of the dimension + * @param defaultFields Default fields of the dimension + * @param namespaces List of namespaces used for Lookup */ - private void setNamespaces(List namespaces) { + public TestLookupDimensionConfig( + TestApiDimensionName apiName, + String physicalName, + KeyValueStore keyValueStore, + SearchProvider searchProvider, + LinkedHashSet fields, + LinkedHashSet defaultFields, + List namespaces + ) { + super(apiName, physicalName, keyValueStore, searchProvider, fields, defaultFields); this.namespaces = namespaces; } @@ -46,4 +65,40 @@ private void setNamespaces(List namespaces) { public List getNamespaces() { return this.namespaces; } + + //CHECKSTYLE:OFF + @Override + public TestLookupDimensionConfig withApiName(TestApiDimensionName apiName) { + return new TestLookupDimensionConfig(apiName, getPhysicalName(), getKeyValueStore(), getSearchProvider(), getFields(), getDefaultDimensionFields(), namespaces); + } + + @Override + public TestLookupDimensionConfig withPhysicalName(String physicalName) { + return new TestLookupDimensionConfig(getApiNameEnum(), physicalName, getKeyValueStore(), getSearchProvider(), getFields(), getDefaultDimensionFields(), namespaces); + } + + @Override + public TestLookupDimensionConfig withKeyValueStore(KeyValueStore keyValueStore) { + return new TestLookupDimensionConfig(getApiNameEnum(), getPhysicalName(), keyValueStore, getSearchProvider(), getFields(), getDefaultDimensionFields(), namespaces); + } + + @Override + public TestLookupDimensionConfig withSearchProvider(SearchProvider searchProvider) { + return new TestLookupDimensionConfig(getApiNameEnum(), getPhysicalName(), getKeyValueStore(), searchProvider, getFields(), getDefaultDimensionFields(), namespaces); + } + + @Override + public TestLookupDimensionConfig withFields(LinkedHashSet fields) { + return new TestLookupDimensionConfig(getApiNameEnum(), getPhysicalName(), getKeyValueStore(), getSearchProvider(), fields, getDefaultDimensionFields(), namespaces); + } + + @Override + public TestLookupDimensionConfig withDefaultFields(LinkedHashSet defaultFields) { + return new TestLookupDimensionConfig(getApiNameEnum(), getPhysicalName(), getKeyValueStore(), getSearchProvider(), getFields(), defaultFields, namespaces); + } + + public TestLookupDimensionConfig withNamespaces(List namespaces) { + return new TestLookupDimensionConfig(getApiNameEnum(), getPhysicalName(), getKeyValueStore(), getSearchProvider(), getFields(), getDefaultDimensionFields(), namespaces); + } + //CHECKSTYLE:ON }