Skip to content

Commit

Permalink
Added QueryTimeLookup end-to-end tests
Browse files Browse the repository at this point in the history
  • Loading branch information
garyluoex authored and cdeszaq committed Sep 12, 2016
1 parent ec2c7d0 commit d69711a
Show file tree
Hide file tree
Showing 12 changed files with 622 additions and 34 deletions.
13 changes: 12 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,23 +10,34 @@ Current

### Added:

- [QueryTimeLookup Functionality Testing](https://github.com/yahoo/fili/pull/34)
* Added two tests `LookupDimensionFilteringDataServeletSpec` and `LookupDimensionGroupingDataServletSpec` to test QTL functionality

- [Enrich the ApiJobStore interface] (https://github.com/yahoo/fili/pull/23)
* `ApiJobStore` Interface now supports filtering `JobRows` in the store
* Added support for filtering JobRows in `HashJobStore`
* Added `JobRowFilter` to hold filter information

- [Lookup Dimension Serializer]()
- [Lookup Dimension Serializer](https://github.com/yahoo/fili/pull/31)
* Created `LookupDimensionToDimensionSpec` serializer for `LookupDimension`
* Created corresponding tests for `LookupDimensionToDimensionSpec` in `LookupDimensionToDimensionSpecSpec`


### Deprecated:

- [QueryTimeLookup Functionality Testing](https://github.com/yahoo/fili/pull/34)
* Deprecated `KeyValueDimensionLoader`, in favor of `TypeAwareDimensionLoader`


### Removed:


### Changed:

- [QueryTimeLookup Functionality Testing](https://github.com/yahoo/fili/pull/34)
* `AbstractBinderFactory` now uses `TypeAwareDimensionLoader` instead of `KeyValueStoreDimensionLoader`
* Modified some testing resources (PETS table and corresponding dimensions) to allow better testing on `LookupDimension`s

- [Reorganizes asynchronous package structure](https://github.com/yahoo/fili/pull/19)
* The `jobs` package is renamed to `async` and split into the following subpackages:
- `broadcastchannels` - Everything dealing with broadcast channels
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
import com.yahoo.bard.webservice.data.config.ResourceDictionaries;
import com.yahoo.bard.webservice.data.config.dimension.DimensionConfig;
import com.yahoo.bard.webservice.data.config.dimension.DimensionLoader;
import com.yahoo.bard.webservice.data.config.dimension.KeyValueStoreDimensionLoader;
import com.yahoo.bard.webservice.data.config.dimension.TypeAwareDimensionLoader;
import com.yahoo.bard.webservice.data.config.metric.MetricLoader;
import com.yahoo.bard.webservice.data.config.table.TableLoader;
import com.yahoo.bard.webservice.data.dimension.DimensionDictionary;
Expand Down Expand Up @@ -837,7 +837,7 @@ protected ConfigurationLoader buildConfigurationLoader(
* @return a Dimension Loader instance
*/
protected DimensionLoader getDimensionLoader() {
return new KeyValueStoreDimensionLoader(getDimensionConfigurations());
return new TypeAwareDimensionLoader(getDimensionConfigurations());
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@

/**
* Load the dimensions along with its keyValueStore and searchProvider.
*
* @deprecated in favor of TypeAwareDimensionLoader which loads corresponding dimension based on config type
*/
@Deprecated
public class KeyValueStoreDimensionLoader implements DimensionLoader {

private Iterable<DimensionConfig> configSource;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import com.yahoo.bard.webservice.data.cache.StubDataCache
import com.yahoo.bard.webservice.data.cache.TupleDataCache
import com.yahoo.bard.webservice.data.config.ConfigurationLoader
import com.yahoo.bard.webservice.data.config.dimension.DimensionConfig
import com.yahoo.bard.webservice.data.config.dimension.KeyValueStoreDimensionLoader
import com.yahoo.bard.webservice.data.config.dimension.TypeAwareDimensionLoader
import com.yahoo.bard.webservice.data.config.dimension.TestDimensions
import com.yahoo.bard.webservice.data.config.metric.MetricLoader
import com.yahoo.bard.webservice.data.config.table.TableLoader
Expand Down Expand Up @@ -171,7 +171,7 @@ public class AbstractBinderFactorySpec extends Specification {
cl.dictionaries != null
cl.tableLoader != null
cl.metricLoader != null
cl.dimensionLoader instanceof KeyValueStoreDimensionLoader
cl.dimensionLoader instanceof TypeAwareDimensionLoader
}

def "test configure bindings"() {
Expand Down Expand Up @@ -436,7 +436,7 @@ public class AbstractBinderFactorySpec extends Specification {
ConfigurationLoader cl = localBinderFactory.buildConfigurationLoader()

then:
cl.dimensionLoader instanceof KeyValueStoreDimensionLoader
cl.dimensionLoader instanceof TypeAwareDimensionLoader
cl.metricLoader == metricLoader
cl.tableLoader == tableLoader
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,25 +12,23 @@ import spock.lang.Specification
*/
class TypeAwareDimensionLoaderSpec extends Specification {

LinkedHashSet<DimensionConfig> lookupDimensionConfigurations
LinkedHashSet<DimensionConfig> dimensionConfigurations
DimensionDictionary dimensionDictionary

def setup() {
lookupDimensionConfigurations = new TestLookupDimensions().getAllDimensionConfigurations()
dimensionConfigurations = new TestDimensions().getAllDimensionConfigurations()
dimensionDictionary = new DimensionDictionary()
}

def "Test dimension loader for Lookup dimension"() {
given: "A Type Aware Dimension Loader with a list of dimension configurations"
TypeAwareDimensionLoader typeAwareDimensionLoader = new TypeAwareDimensionLoader(lookupDimensionConfigurations)
TypeAwareDimensionLoader typeAwareDimensionLoader = new TypeAwareDimensionLoader(dimensionConfigurations)

when:
typeAwareDimensionLoader.loadDimensionDictionary(dimensionDictionary)

then:
dimensionDictionary.findByApiName("size").getClass() == LookupDimension.class
dimensionDictionary.findByApiName("breed").getClass() == LookupDimension.class
}

def "Test dimension loader for KeyValueStore dimension"() {
Expand All @@ -41,16 +39,16 @@ class TypeAwareDimensionLoaderSpec extends Specification {
typeAwareDimensionLoader.loadDimensionDictionary(dimensionDictionary)

then:
dimensionDictionary.findByApiName("breed").getClass() == KeyValueStoreDimension.class
dimensionDictionary.findByApiName("color").getClass() == KeyValueStoreDimension.class
}

def "Test dimension loader for a dimension type that is not defined"() {
given: "A Type Aware Dimension loader with a list of dimension configurations"
DimensionConfig dimensionConfiguration = Mock(DimensionConfig)
dimensionConfiguration.getApiName() >> "foo"
dimensionConfiguration.getType() >> String.class
lookupDimensionConfigurations.add(dimensionConfiguration)
TypeAwareDimensionLoader typeAwareDimensionLoader = new TypeAwareDimensionLoader(lookupDimensionConfigurations)
dimensionConfigurations.add(dimensionConfiguration)
TypeAwareDimensionLoader typeAwareDimensionLoader = new TypeAwareDimensionLoader(dimensionConfigurations)

when:
typeAwareDimensionLoader.loadDimensionDictionary(dimensionDictionary)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,14 +68,14 @@ class TableFullViewProcessorSpec extends Specification {
def "Check table meta data info at grain level"() {

setup:
String expectedResponse = "[name:all, description:The pets all grain, metrics:[[category:General, name:rowNum, longName:rowNum, uri:http://localhost:9998/v1/rowNum], [category:General, name:limbs, longName:limbs, uri:http://localhost:9998/v1/limbs], [category:General, name:dayAvgLimbs, longName:dayAvgLimbs, uri:http://localhost:9998/v1/dayAvgLimbs]], retention:P1Y, longName:All, dimensions:[[category:General, name:species, longName:species, uri:http://localhost:9998/v1/species, cardinality:0], [category:General, name:sex, longName:sex, uri:http://localhost:9998/v1/sex, cardinality:0], [category:General, name:breed, longName:breed, uri:http://localhost:9998/v1/breed, cardinality:0]]]"
String expectedResponse = "[description:The pets all grain, dimensions:[[category:General, name:sex, longName:sex, uri:http://localhost:9998/v1/sex, cardinality:0], [category:General, name:species, longName:species, uri:http://localhost:9998/v1/species, cardinality:0], [category:General, name:breed, longName:breed, uri:http://localhost:9998/v1/breed, cardinality:0]], longName:All, metrics:[[category:General, name:rowNum, longName:rowNum, uri:http://localhost:9998/v1/rowNum], [category:General, name:limbs, longName:limbs, uri:http://localhost:9998/v1/limbs], [category:General, name:dayAvgLimbs, longName:dayAvgLimbs, uri:http://localhost:9998/v1/dayAvgLimbs]], name:all, retention:P1Y]"
LogicalTable petsTable = petsShapesTables.find {it.getName() == "pets"}

when:
TableGrainView tableGrainView = fullViewProcessor.formatTableGrain(petsTable, "all", uriInfo)

then:
tableGrainView.toString() == expectedResponse
tableGrainView.sort().toString() == expectedResponse
}

def "check all the tables full view at grain level"() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,13 @@
// Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
package com.yahoo.bard.webservice.web.endpoints

import static com.yahoo.bard.webservice.util.JsonSortStrategy.SORT_BOTH

import com.yahoo.bard.webservice.application.JerseyTestBinder
import com.yahoo.bard.webservice.data.time.GranularityParser
import com.yahoo.bard.webservice.data.time.StandardGranularityParser
import com.yahoo.bard.webservice.models.druid.client.impl.TestDruidWebService
import com.yahoo.bard.webservice.table.availability.AvailabilityTestingUtils
import com.yahoo.bard.webservice.util.GroovyTestUtils
import com.yahoo.bard.webservice.util.JsonSortStrategy

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module
Expand Down Expand Up @@ -106,7 +105,7 @@ abstract class BaseDataServletComponentSpec extends Specification {
}

// Most aspects of a Druid query are order-independent, so normalize both lists and maps.
GroovyTestUtils.compareJson(jsonQuery, getExpectedDruidQuery(), SORT_BOTH)
compareResult(jsonQuery, getExpectedDruidQuery(), JsonSortStrategy.SORT_BOTH)
}

@Timeout(10)
Expand Down Expand Up @@ -135,8 +134,8 @@ abstract class BaseDataServletComponentSpec extends Specification {
validateJson(expectedApiResponse)
}

boolean compareResult(String result, String expectedResult) {
GroovyTestUtils.compareJson(result, expectedResult)
boolean compareResult(String result, String expectedResult, JsonSortStrategy sortStrategy = JsonSortStrategy.SORT_MAPS) {
GroovyTestUtils.compareJson(result, expectedResult, sortStrategy)
}

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
package com.yahoo.bard.webservice.web.endpoints

import com.yahoo.bard.webservice.data.dimension.BardDimensionField
import com.yahoo.bard.webservice.data.dimension.DimensionDictionary
import com.yahoo.bard.webservice.util.GroovyTestUtils
import com.yahoo.bard.webservice.util.JsonSortStrategy

/**
* Testing Query Time Lookup Functionality with Lookup Dimensions in GroupBy query.
*/
class LookupDimensionGroupingDataServletSpec extends BaseDataServletComponentSpec{

@Override
def setup() {
DimensionDictionary dimensionStore = jtb.configurationLoader.dimensionDictionary
dimensionStore.findByApiName("sex").with {
addDimensionRow(BardDimensionField.makeDimensionRow(it, "sex1", "sex1Desc"))
addDimensionRow(BardDimensionField.makeDimensionRow(it, "sex2", "sex2Desc"))
addDimensionRow(BardDimensionField.makeDimensionRow(it, "sex3", "sex3Desc"))
}
dimensionStore.findByApiName("breed").with {
addDimensionRow(BardDimensionField.makeDimensionRow(it, "breed1", "breed1Desc"))
addDimensionRow(BardDimensionField.makeDimensionRow(it, "breed2", "breed2Desc"))
addDimensionRow(BardDimensionField.makeDimensionRow(it, "breed3", "breed3Desc"))
}
dimensionStore.findByApiName("species").with {
addDimensionRow(BardDimensionField.makeDimensionRow(it, "species1", "species1Desc"))
addDimensionRow(BardDimensionField.makeDimensionRow(it, "species2", "species2Desc"))
addDimensionRow(BardDimensionField.makeDimensionRow(it, "species3", "species3Desc"))
}
}

@Override
Class<?>[] getResourceClasses() {
[DataServlet.class]
}

@Override
String getTarget() {
return "data/pets/day/sex/breed/species"
}

@Override
Map<String, List<String>> getQueryParams() {
[
"metrics" : ["limbs"],
"dateTime": ["2014-05-01%2F2014-05-02"]
]
}

@Override
boolean compareResult(String result, String expectedResult, JsonSortStrategy sortStrategy) {
// Override "test druid query" feature test's compareResult which calls with SORT_BOTH to be default SORT_MAP
GroovyTestUtils.compareJson(result, expectedResult)
}

@Override
String getExpectedDruidQuery() {
"""{
"aggregations": [{ "fieldName": "limbs", "name": "limbs", "type": "longSum" }],
"context": {},
"dataSource": { "name": "all_pets", "type": "table" },
"dimensions": [
"sex",
{
"dimension": "breed",
"extractionFn": {
"extractionFns": [
{
"injective": false,
"lookup": { "namespace": "NAMESPACE1", "type": "namespace" },
"optimize": true,
"replaceMissingValueWith": "Unknown NAMESPACE1",
"retainMissingValue": false, "type": "lookup"
},
{
"injective": false,
"lookup": { "namespace": "NAMESPACE2", "type": "namespace" },
"optimize": true,
"replaceMissingValueWith": "Unknown NAMESPACE2",
"retainMissingValue": false, "type": "lookup"
}
],
"type": "cascade"
},
"outputName": "breed", "type": "extraction"
},
{
"dimension": "class",
"extractionFn": {
"injective": false,
"lookup": { "namespace": "NAMESPACE1", "type": "namespace" },
"optimize": true,
"replaceMissingValueWith": "Unknown NAMESPACE1", "retainMissingValue": false, "type": "lookup"
},
"outputName": "species",
"type": "extraction"
}
],
"granularity": { "period": "P1D", "timeZone": "UTC", "type": "period" },
"intervals": ["2014-05-01T00:00:00.000Z/2014-05-02T00:00:00.000Z"],
"postAggregations": [],
"queryType": "groupBy"
}"""
}

@Override
String getFakeDruidResponse() {
"""[
{
"version" : "v1",
"timestamp" : "2014-05-01T00:00:00.000Z",
"event" : {
"sex" : "sex1",
"breed" : "breed1",
"species" : "species1",
"limbs" : 1
}
},
{
"version" : "v1",
"timestamp" : "2014-05-01T00:00:00.000Z",
"event" : {
"sex" : "sex2",
"breed" : "breed2",
"species" : "species2",
"limbs" : 2
}
},
{
"version" : "v1",
"timestamp" : "2014-05-01T00:00:00.000Z",
"event" : {
"sex" : "sex3",
"breed" : "breed3",
"species" : "species3",
"limbs" : 3
}
}
]"""
}

@Override
String getExpectedApiResponse() {
"""{
"rows": [{
"breed|desc": "breed1Desc",
"breed|id": "breed1",
"dateTime": "2014-05-01 00:00:00.000",
"limbs": 1,
"sex|desc": "sex1Desc",
"sex|id": "sex1",
"species|desc": "species1Desc",
"species|id": "species1"
},
{
"breed|desc": "breed2Desc",
"breed|id": "breed2",
"dateTime": "2014-05-01 00:00:00.000",
"limbs": 2,
"sex|desc": "sex2Desc",
"sex|id": "sex2",
"species|desc": "species2Desc",
"species|id": "species2"
},
{
"breed|desc": "breed3Desc",
"breed|id": "breed3",
"dateTime": "2014-05-01 00:00:00.000",
"limbs": 3,
"sex|desc": "sex3Desc",
"sex|id": "sex3",
"species|desc": "species3Desc",
"species|id": "species3"
}]
}"""
}
}

Loading

0 comments on commit d69711a

Please sign in to comment.