Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

QTL Integration Style Tests #34

Merged
merged 1 commit into from
Sep 12, 2016
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,23 +10,34 @@ Current

### Added:

- [QueryTimeLookup Functionality Testing](https://github.com/yahoo/fili/pull/34)
* Added two tests `LookupDimensionFilteringDataServeletSpec` and `LookupDimensionGroupingDataServletSpec` to test QTL functionality

- [Enrich the ApiJobStore interface] (https://github.com/yahoo/fili/pull/23)
* `ApiJobStore` Interface now supports filtering `JobRows` in the store
* Added support for filtering JobRows in `HashJobStore`
* Added `JobRowFilter` to hold filter information

- [Lookup Dimension Serializer]()
- [Lookup Dimension Serializer](https://github.com/yahoo/fili/pull/31)
* Created `LookupDimensionToDimensionSpec` serializer for `LookupDimension`
* Created corresponding tests for `LookupDimensionToDimensionSpec` in `LookupDimensionToDimensionSpecSpec`


### Deprecated:

- [QueryTimeLookup Functionality Testing](https://github.com/yahoo/fili/pull/34)
* Deprecated `KeyValueDimensionLoader`, in favor of `TypeAwareDimensionLoader`


### Removed:


### Changed:

- [QueryTimeLookup Functionality Testing](https://github.com/yahoo/fili/pull/34)
* `AbstractBinderFactory` now uses `TypeAwareDimensionLoader` instead of `KeyValueStoreDimensionLoader`
* Modified some testing resources (PETS table and corresponding dimensions) to allow better testing on `LookupDimension`s

- [Reorganizes asynchronous package structure](https://github.com/yahoo/fili/pull/19)
* The `jobs` package is renamed to `async` and split into the following subpackages:
- `broadcastchannels` - Everything dealing with broadcast channels
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
import com.yahoo.bard.webservice.data.config.ResourceDictionaries;
import com.yahoo.bard.webservice.data.config.dimension.DimensionConfig;
import com.yahoo.bard.webservice.data.config.dimension.DimensionLoader;
import com.yahoo.bard.webservice.data.config.dimension.KeyValueStoreDimensionLoader;
import com.yahoo.bard.webservice.data.config.dimension.TypeAwareDimensionLoader;
import com.yahoo.bard.webservice.data.config.metric.MetricLoader;
import com.yahoo.bard.webservice.data.config.table.TableLoader;
import com.yahoo.bard.webservice.data.dimension.DimensionDictionary;
Expand Down Expand Up @@ -837,7 +837,7 @@ protected ConfigurationLoader buildConfigurationLoader(
* @return a Dimension Loader instance
*/
protected DimensionLoader getDimensionLoader() {
return new KeyValueStoreDimensionLoader(getDimensionConfigurations());
return new TypeAwareDimensionLoader(getDimensionConfigurations());
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Need to call out this change in Changelog

}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@

/**
* Load the dimensions along with its keyValueStore and searchProvider.
*
* @deprecated in favor of TypeAwareDimensionLoader which loads corresponding dimension based on config type
*/
@Deprecated
public class KeyValueStoreDimensionLoader implements DimensionLoader {

private Iterable<DimensionConfig> configSource;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import com.yahoo.bard.webservice.data.cache.StubDataCache
import com.yahoo.bard.webservice.data.cache.TupleDataCache
import com.yahoo.bard.webservice.data.config.ConfigurationLoader
import com.yahoo.bard.webservice.data.config.dimension.DimensionConfig
import com.yahoo.bard.webservice.data.config.dimension.KeyValueStoreDimensionLoader
import com.yahoo.bard.webservice.data.config.dimension.TypeAwareDimensionLoader
import com.yahoo.bard.webservice.data.config.dimension.TestDimensions
import com.yahoo.bard.webservice.data.config.metric.MetricLoader
import com.yahoo.bard.webservice.data.config.table.TableLoader
Expand Down Expand Up @@ -171,7 +171,7 @@ public class AbstractBinderFactorySpec extends Specification {
cl.dictionaries != null
cl.tableLoader != null
cl.metricLoader != null
cl.dimensionLoader instanceof KeyValueStoreDimensionLoader
cl.dimensionLoader instanceof TypeAwareDimensionLoader
}

def "test configure bindings"() {
Expand Down Expand Up @@ -436,7 +436,7 @@ public class AbstractBinderFactorySpec extends Specification {
ConfigurationLoader cl = localBinderFactory.buildConfigurationLoader()

then:
cl.dimensionLoader instanceof KeyValueStoreDimensionLoader
cl.dimensionLoader instanceof TypeAwareDimensionLoader
cl.metricLoader == metricLoader
cl.tableLoader == tableLoader
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,25 +12,23 @@ import spock.lang.Specification
*/
class TypeAwareDimensionLoaderSpec extends Specification {

LinkedHashSet<DimensionConfig> lookupDimensionConfigurations
LinkedHashSet<DimensionConfig> dimensionConfigurations
DimensionDictionary dimensionDictionary

def setup() {
lookupDimensionConfigurations = new TestLookupDimensions().getAllDimensionConfigurations()
dimensionConfigurations = new TestDimensions().getAllDimensionConfigurations()
dimensionDictionary = new DimensionDictionary()
}

def "Test dimension loader for Lookup dimension"() {
given: "A Type Aware Dimension Loader with a list of dimension configurations"
TypeAwareDimensionLoader typeAwareDimensionLoader = new TypeAwareDimensionLoader(lookupDimensionConfigurations)
TypeAwareDimensionLoader typeAwareDimensionLoader = new TypeAwareDimensionLoader(dimensionConfigurations)

when:
typeAwareDimensionLoader.loadDimensionDictionary(dimensionDictionary)

then:
dimensionDictionary.findByApiName("size").getClass() == LookupDimension.class
dimensionDictionary.findByApiName("breed").getClass() == LookupDimension.class
}

def "Test dimension loader for KeyValueStore dimension"() {
Expand All @@ -41,16 +39,16 @@ class TypeAwareDimensionLoaderSpec extends Specification {
typeAwareDimensionLoader.loadDimensionDictionary(dimensionDictionary)

then:
dimensionDictionary.findByApiName("breed").getClass() == KeyValueStoreDimension.class
dimensionDictionary.findByApiName("color").getClass() == KeyValueStoreDimension.class
}

def "Test dimension loader for a dimension type that is not defined"() {
given: "A Type Aware Dimension loader with a list of dimension configurations"
DimensionConfig dimensionConfiguration = Mock(DimensionConfig)
dimensionConfiguration.getApiName() >> "foo"
dimensionConfiguration.getType() >> String.class
lookupDimensionConfigurations.add(dimensionConfiguration)
TypeAwareDimensionLoader typeAwareDimensionLoader = new TypeAwareDimensionLoader(lookupDimensionConfigurations)
dimensionConfigurations.add(dimensionConfiguration)
TypeAwareDimensionLoader typeAwareDimensionLoader = new TypeAwareDimensionLoader(dimensionConfigurations)

when:
typeAwareDimensionLoader.loadDimensionDictionary(dimensionDictionary)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,14 +68,14 @@ class TableFullViewProcessorSpec extends Specification {
def "Check table meta data info at grain level"() {

setup:
String expectedResponse = "[name:all, description:The pets all grain, metrics:[[category:General, name:rowNum, longName:rowNum, uri:http://localhost:9998/v1/rowNum], [category:General, name:limbs, longName:limbs, uri:http://localhost:9998/v1/limbs], [category:General, name:dayAvgLimbs, longName:dayAvgLimbs, uri:http://localhost:9998/v1/dayAvgLimbs]], retention:P1Y, longName:All, dimensions:[[category:General, name:species, longName:species, uri:http://localhost:9998/v1/species, cardinality:0], [category:General, name:sex, longName:sex, uri:http://localhost:9998/v1/sex, cardinality:0], [category:General, name:breed, longName:breed, uri:http://localhost:9998/v1/breed, cardinality:0]]]"
String expectedResponse = "[description:The pets all grain, dimensions:[[category:General, name:sex, longName:sex, uri:http://localhost:9998/v1/sex, cardinality:0], [category:General, name:species, longName:species, uri:http://localhost:9998/v1/species, cardinality:0], [category:General, name:breed, longName:breed, uri:http://localhost:9998/v1/breed, cardinality:0]], longName:All, metrics:[[category:General, name:rowNum, longName:rowNum, uri:http://localhost:9998/v1/rowNum], [category:General, name:limbs, longName:limbs, uri:http://localhost:9998/v1/limbs], [category:General, name:dayAvgLimbs, longName:dayAvgLimbs, uri:http://localhost:9998/v1/dayAvgLimbs]], name:all, retention:P1Y]"
LogicalTable petsTable = petsShapesTables.find {it.getName() == "pets"}

when:
TableGrainView tableGrainView = fullViewProcessor.formatTableGrain(petsTable, "all", uriInfo)

then:
tableGrainView.toString() == expectedResponse
tableGrainView.sort().toString() == expectedResponse
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are we sorting here? What changed that we need to sort now but didn't before?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of containing KeyValueStoreDimension, the table now contains LookupDimension which changed the order of the dimensions. I can look into it if you think it is necessary and remove the sort?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nah, it's probably fine.

}

def "check all the tables full view at grain level"() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,13 @@
// Licensed under the terms of the Apache license. Please see LICENSE file distributed with this work for terms.
package com.yahoo.bard.webservice.web.endpoints

import static com.yahoo.bard.webservice.util.JsonSortStrategy.SORT_BOTH

import com.yahoo.bard.webservice.application.JerseyTestBinder
import com.yahoo.bard.webservice.data.time.GranularityParser
import com.yahoo.bard.webservice.data.time.StandardGranularityParser
import com.yahoo.bard.webservice.models.druid.client.impl.TestDruidWebService
import com.yahoo.bard.webservice.table.availability.AvailabilityTestingUtils
import com.yahoo.bard.webservice.util.GroovyTestUtils
import com.yahoo.bard.webservice.util.JsonSortStrategy

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module
Expand Down Expand Up @@ -106,7 +105,7 @@ abstract class BaseDataServletComponentSpec extends Specification {
}

// Most aspects of a Druid query are order-independent, so normalize both lists and maps.
GroovyTestUtils.compareJson(jsonQuery, getExpectedDruidQuery(), SORT_BOTH)
compareResult(jsonQuery, getExpectedDruidQuery(), JsonSortStrategy.SORT_BOTH)
}

@Timeout(10)
Expand Down Expand Up @@ -135,8 +134,8 @@ abstract class BaseDataServletComponentSpec extends Specification {
validateJson(expectedApiResponse)
}

boolean compareResult(String result, String expectedResult) {
GroovyTestUtils.compareJson(result, expectedResult)
boolean compareResult(String result, String expectedResult, JsonSortStrategy sortStrategy = JsonSortStrategy.SORT_MAPS) {
GroovyTestUtils.compareJson(result, expectedResult, sortStrategy)
}

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
package com.yahoo.bard.webservice.web.endpoints

import com.yahoo.bard.webservice.data.dimension.BardDimensionField
import com.yahoo.bard.webservice.data.dimension.DimensionDictionary
import com.yahoo.bard.webservice.util.GroovyTestUtils
import com.yahoo.bard.webservice.util.JsonSortStrategy

/**
* Testing Query Time Lookup Functionality with Lookup Dimensions in GroupBy query.
*/
class LookupDimensionGroupingDataServletSpec extends BaseDataServletComponentSpec{

@Override
def setup() {
DimensionDictionary dimensionStore = jtb.configurationLoader.dimensionDictionary
dimensionStore.findByApiName("sex").with {
addDimensionRow(BardDimensionField.makeDimensionRow(it, "sex1", "sex1Desc"))
addDimensionRow(BardDimensionField.makeDimensionRow(it, "sex2", "sex2Desc"))
addDimensionRow(BardDimensionField.makeDimensionRow(it, "sex3", "sex3Desc"))
}
dimensionStore.findByApiName("breed").with {
addDimensionRow(BardDimensionField.makeDimensionRow(it, "breed1", "breed1Desc"))
addDimensionRow(BardDimensionField.makeDimensionRow(it, "breed2", "breed2Desc"))
addDimensionRow(BardDimensionField.makeDimensionRow(it, "breed3", "breed3Desc"))
}
dimensionStore.findByApiName("species").with {
addDimensionRow(BardDimensionField.makeDimensionRow(it, "species1", "species1Desc"))
addDimensionRow(BardDimensionField.makeDimensionRow(it, "species2", "species2Desc"))
addDimensionRow(BardDimensionField.makeDimensionRow(it, "species3", "species3Desc"))
}
}

@Override
Class<?>[] getResourceClasses() {
[DataServlet.class]
}

@Override
String getTarget() {
return "data/pets/day/sex/breed/species"
}

@Override
Map<String, List<String>> getQueryParams() {
[
"metrics" : ["limbs"],
"dateTime": ["2014-05-01%2F2014-05-02"]
]
}

@Override
boolean compareResult(String result, String expectedResult, JsonSortStrategy sortStrategy) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this need the override? It's doing what the default does anyways I think

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as above.

// Override "test druid query" feature test's compareResult which calls with SORT_BOTH to be default SORT_MAP
GroovyTestUtils.compareJson(result, expectedResult)
}

@Override
String getExpectedDruidQuery() {
"""{
"aggregations": [{ "fieldName": "limbs", "name": "limbs", "type": "longSum" }],
"context": {},
"dataSource": { "name": "all_pets", "type": "table" },
"dimensions": [
"sex",
{
"dimension": "breed",
"extractionFn": {
"extractionFns": [
{
"injective": false,
"lookup": { "namespace": "NAMESPACE1", "type": "namespace" },
"optimize": true,
"replaceMissingValueWith": "Unknown NAMESPACE1",
"retainMissingValue": false, "type": "lookup"
},
{
"injective": false,
"lookup": { "namespace": "NAMESPACE2", "type": "namespace" },
"optimize": true,
"replaceMissingValueWith": "Unknown NAMESPACE2",
"retainMissingValue": false, "type": "lookup"
}
],
"type": "cascade"
},
"outputName": "breed", "type": "extraction"
},
{
"dimension": "class",
"extractionFn": {
"injective": false,
"lookup": { "namespace": "NAMESPACE1", "type": "namespace" },
"optimize": true,
"replaceMissingValueWith": "Unknown NAMESPACE1", "retainMissingValue": false, "type": "lookup"
},
"outputName": "species",
"type": "extraction"
}
],
"granularity": { "period": "P1D", "timeZone": "UTC", "type": "period" },
"intervals": ["2014-05-01T00:00:00.000Z/2014-05-02T00:00:00.000Z"],
"postAggregations": [],
"queryType": "groupBy"
}"""
}

@Override
String getFakeDruidResponse() {
"""[
{
"version" : "v1",
"timestamp" : "2014-05-01T00:00:00.000Z",
"event" : {
"sex" : "sex1",
"breed" : "breed1",
"species" : "species1",
"limbs" : 1
}
},
{
"version" : "v1",
"timestamp" : "2014-05-01T00:00:00.000Z",
"event" : {
"sex" : "sex2",
"breed" : "breed2",
"species" : "species2",
"limbs" : 2
}
},
{
"version" : "v1",
"timestamp" : "2014-05-01T00:00:00.000Z",
"event" : {
"sex" : "sex3",
"breed" : "breed3",
"species" : "species3",
"limbs" : 3
}
}
]"""
}

@Override
String getExpectedApiResponse() {
"""{
"rows": [{
"breed|desc": "breed1Desc",
"breed|id": "breed1",
"dateTime": "2014-05-01 00:00:00.000",
"limbs": 1,
"sex|desc": "sex1Desc",
"sex|id": "sex1",
"species|desc": "species1Desc",
"species|id": "species1"
},
{
"breed|desc": "breed2Desc",
"breed|id": "breed2",
"dateTime": "2014-05-01 00:00:00.000",
"limbs": 2,
"sex|desc": "sex2Desc",
"sex|id": "sex2",
"species|desc": "species2Desc",
"species|id": "species2"
},
{
"breed|desc": "breed3Desc",
"breed|id": "breed3",
"dateTime": "2014-05-01 00:00:00.000",
"limbs": 3,
"sex|desc": "sex3Desc",
"sex|id": "sex3",
"species|desc": "species3Desc",
"species|id": "species3"
}]
}"""
}
}

Loading