Skip to content

Commit

Permalink
Modified according to three person's coments
Browse files Browse the repository at this point in the history
  • Loading branch information
@garyluoex
garyluoex committed Apr 13, 2017
1 parent 8b6212e commit 52ece84
Show file tree
Hide file tree
Showing 9 changed files with 81 additions and 121 deletions.
1 change: 1 addition & 0 deletions fili-core/src/main/java/com/yahoo/bard/webservice/table/BasePhysicalTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
Expand Down
11 changes: 0 additions & 11 deletions fili-core/src/main/java/com/yahoo/bard/webservice/table/PermissiveConcretePhysicalTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@
import com.yahoo.bard.webservice.data.time.ZonedTimeGrain;
import com.yahoo.bard.webservice.metadata.DataSourceMetadataService;
import com.yahoo.bard.webservice.table.availability.PermissiveAvailability;
import com.yahoo.bard.webservice.table.resolver.DataSourceConstraint;
import com.yahoo.bard.webservice.table.resolver.PhysicalDataSourceConstraint;
import com.yahoo.bard.webservice.util.SimplifiedIntervalList;

import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -50,12 +47,4 @@ public PermissiveConcretePhysicalTable(
new PermissiveAvailability(name, dataSourceMetadataService)
);
}

@Override
public SimplifiedIntervalList getAvailableIntervals(DataSourceConstraint constraint) {
return getAvailability().getAvailableIntervals(PhysicalDataSourceConstraint.buildWithSchemaUnion(
constraint,
getSchema()
));
}
}
2 changes: 1 addition & 1 deletion fili-core/src/main/java/com/yahoo/bard/webservice/table/PhysicalTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public interface PhysicalTable extends Table {
DateTime getTableAlignment();

/**
* Getter for all the available intervals for the corresponding column.
* Getter for all the available intervals for the corresponding columns configured on the table.
*
* @return map of column to set of available intervals
*/
Expand Down
8 changes: 4 additions & 4 deletions ...re/src/main/java/com/yahoo/bard/webservice/table/availability/PermissiveAvailability.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,18 +45,18 @@ public PermissiveAvailability(
* )};
* Instead of returning the intersection of all available intervals, this method returns the union of them.
*
* @param constraint Data constraint containing columns and api filters. Constrains are ignored, because
* @param ignoredConstraint Data constraint containing columns and api filters. Constrains are ignored, because
* <tt>PermissiveAvailability</tt> returns as much available intervals as possible by, for example, allowing
* missing intervals and returning unions of available intervals
*
* @return the union of all available intervals
*/
@Override
public SimplifiedIntervalList getAvailableIntervals(PhysicalDataSourceConstraint constraint) {
public SimplifiedIntervalList getAvailableIntervals(PhysicalDataSourceConstraint ignoredConstraint) {
Map<String, List<Interval>> allAvailableIntervals = getAllAvailableIntervals();

return constraint.getAllColumnPhysicalNames().stream()
.map(columnName -> allAvailableIntervals.getOrDefault(columnName, Collections.emptyList()))
return getAllAvailableIntervals().keySet().stream()
.map(columnName -> allAvailableIntervals.getOrDefault(columnName, new SimplifiedIntervalList()))
.flatMap(List::stream)
.collect(SimplifiedIntervalList.getCollector());
}
Expand Down
43 changes: 4 additions & 39 deletions .../src/main/java/com/yahoo/bard/webservice/table/resolver/PhysicalDataSourceConstraint.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@
// Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms.
package com.yahoo.bard.webservice.table.resolver;

import com.yahoo.bard.webservice.table.Column;
import com.yahoo.bard.webservice.table.PhysicalTableSchema;

import java.util.Set;
import java.util.stream.Collectors;

import javax.validation.constraints.NotNull;

/**
* Data source constraint containing physical name of the columns.
*/
Expand All @@ -22,8 +23,8 @@ public class PhysicalDataSourceConstraint extends DataSourceConstraint {
* @param physicalTableSchema A map from logical column name to physical column names
*/
public PhysicalDataSourceConstraint(
DataSourceConstraint dataSourceConstraint,
PhysicalTableSchema physicalTableSchema
@NotNull DataSourceConstraint dataSourceConstraint,
@NotNull PhysicalTableSchema physicalTableSchema
) {
super(dataSourceConstraint);

Expand All @@ -35,27 +36,6 @@ public PhysicalDataSourceConstraint(
.collect(Collectors.toSet());
}

/**
* Private Constructor that union columns in constraint and schema, use buildWithSchemaUnion to invoke.
*
* @param physicalTableSchema A map from logical column name to physical column names
* @param dataSourceConstraint Data source constraint containing all the column names as logical names
*/
private PhysicalDataSourceConstraint(
PhysicalTableSchema physicalTableSchema,
DataSourceConstraint dataSourceConstraint
) {
super(dataSourceConstraint);

Set<String> schemaColumnNames = physicalTableSchema.getColumns().stream()
.map(Column::getName)
.collect(Collectors.toSet());

this.allColumnPhysicalNames = schemaColumnNames.stream()
.map(physicalTableSchema::getPhysicalColumnName)
.collect(Collectors.toSet());
}

/**
* Getter for the all column names as physical names.
*
Expand All @@ -64,19 +44,4 @@ private PhysicalDataSourceConstraint(
public Set<String> getAllColumnPhysicalNames() {
return allColumnPhysicalNames;
}

/**
* Builds a physical data source constraint with the union of columns in schema.
*
* @param dataSourceConstraint Data source constraint containing all the column names as logical names
* @param physicalTableSchema A map from logical column name to physical column names
*
* @return a new physical data source constraint with all columns in schema
*/
public static PhysicalDataSourceConstraint buildWithSchemaUnion(
DataSourceConstraint dataSourceConstraint,
PhysicalTableSchema physicalTableSchema
) {
return new PhysicalDataSourceConstraint(physicalTableSchema, dataSourceConstraint);
}
}
70 changes: 35 additions & 35 deletions ...-core/src/test/groovy/com/yahoo/bard/webservice/data/QueryBuildingTestingResources.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,53 +213,53 @@ public class QueryBuildingTestingResources {
TimeGrain utcHour = HOUR.buildZonedTimeGrain(UTC)
TimeGrain utcDay = DAY.buildZonedTimeGrain(UTC)

volatileHourTable = new ConcretePhysicalTable("hour", HOUR.buildZonedTimeGrain(UTC), [d1, m1].collect{toColumn(it)}.toSet(), [:], metadataService)
volatileDayTable = new ConcretePhysicalTable("day", DAY.buildZonedTimeGrain(UTC), [d1, m1].collect{toColumn(it)}.toSet(), [:], metadataService)
volatileHourTable = new ConcretePhysicalTable("hour", HOUR.buildZonedTimeGrain(UTC), [d1, m1].collect{toColumn(it)} as Set, [:], metadataService)
volatileDayTable = new ConcretePhysicalTable("day", DAY.buildZonedTimeGrain(UTC), [d1, m1].collect{toColumn(it)} as Set, [:], metadataService)

t1h = new ConcretePhysicalTable("table1h", utcHour, [d1, d2, d3, m1, m2, m3].collect{toColumn(it)}.toSet(), ["ageBracket":"age_bracket"], metadataService)
t1d = new ConcretePhysicalTable("table1d", utcDay, [d1, d2, d3, m1, m2, m3].collect{toColumn(it)}.toSet(), ["ageBracket":"age_bracket"], metadataService)
t1hShort = new ConcretePhysicalTable("table1Short", utcHour, [d1, d2, m1, m2, m3].collect{toColumn(it)}.toSet(), [:], metadataService)
t1h = new ConcretePhysicalTable("table1h", utcHour, [d1, d2, d3, m1, m2, m3].collect{toColumn(it)} as Set, ["ageBracket":"age_bracket"], metadataService)
t1d = new ConcretePhysicalTable("table1d", utcDay, [d1, d2, d3, m1, m2, m3].collect{toColumn(it)} as Set, ["ageBracket":"age_bracket"], metadataService)
t1hShort = new ConcretePhysicalTable("table1Short", utcHour, [d1, d2, m1, m2, m3].collect{toColumn(it)} as Set, [:], metadataService)

t2h = new ConcretePhysicalTable("table2", utcHour, [d1, d2, d4, m1, m4, m5].collect{toColumn(it)}.toSet(), [:], metadataService)
t3d = new ConcretePhysicalTable("table3", utcDay, [d1, d2, d5, m6].collect{toColumn(it)}.toSet(), [:], metadataService)
t2h = new ConcretePhysicalTable("table2", utcHour, [d1, d2, d4, m1, m4, m5].collect{toColumn(it)} as Set, [:], metadataService)
t3d = new ConcretePhysicalTable("table3", utcDay, [d1, d2, d5, m6].collect{toColumn(it)} as Set, [:], metadataService)

tna1236d = new ConcretePhysicalTable("tableNA1236", utcDay, [d1, d2, d3, d6].collect{toColumn(it)}.toSet(),["ageBracket":"age_bracket"], metadataService)
tna1237d = new ConcretePhysicalTable("tableNA1237", utcDay, [d1, d2, d3, d7].collect{toColumn(it)}.toSet(), ["ageBracket":"age_bracket"], metadataService)
tna167d = new ConcretePhysicalTable("tableNA167", utcDay, [d1, d6, d7].collect{toColumn(it)}.toSet(), ["ageBracket":"age_bracket", "dim7":"dim_7"], metadataService)
tna267d = new ConcretePhysicalTable("tableNA267", utcDay, [d2, d6, d7].collect{toColumn(it)}.toSet(), ["dim7":"dim_7"], metadataService)
tna1236d = new ConcretePhysicalTable("tableNA1236", utcDay, [d1, d2, d3, d6].collect{toColumn(it)} as Set,["ageBracket":"age_bracket"], metadataService)
tna1237d = new ConcretePhysicalTable("tableNA1237", utcDay, [d1, d2, d3, d7].collect{toColumn(it)} as Set, ["ageBracket":"age_bracket"], metadataService)
tna167d = new ConcretePhysicalTable("tableNA167", utcDay, [d1, d6, d7].collect{toColumn(it)} as Set, ["ageBracket":"age_bracket", "dim7":"dim_7"], metadataService)
tna267d = new ConcretePhysicalTable("tableNA267", utcDay, [d2, d6, d7].collect{toColumn(it)} as Set, ["dim7":"dim_7"], metadataService)

t4h1 = new ConcretePhysicalTable("table4h1", utcHour, [d1, d2, m1, m2, m3].collect{toColumn(it)}.toSet(), [:], metadataService)
t4h2 = new ConcretePhysicalTable("table4h2", utcHour, [d1, d2, m1, m2, m3].collect{toColumn(it)}.toSet(), [:], metadataService)
t4d1 = new ConcretePhysicalTable("table4d1", utcDay, [d1, d2, m1, m2, m3].collect{toColumn(it)}.toSet(), [:], metadataService)
t4d2 = new ConcretePhysicalTable("table4d2", utcDay, [d1, d2, m1, m2, m3].collect{toColumn(it)}.toSet(), [:], metadataService)
t4h1 = new ConcretePhysicalTable("table4h1", utcHour, [d1, d2, m1, m2, m3].collect{toColumn(it)} as Set, [:], metadataService)
t4h2 = new ConcretePhysicalTable("table4h2", utcHour, [d1, d2, m1, m2, m3].collect{toColumn(it)} as Set, [:], metadataService)
t4d1 = new ConcretePhysicalTable("table4d1", utcDay, [d1, d2, m1, m2, m3].collect{toColumn(it)} as Set, [:], metadataService)
t4d2 = new ConcretePhysicalTable("table4d2", utcDay, [d1, d2, m1, m2, m3].collect{toColumn(it)} as Set, [:], metadataService)

Map<String, Set<Interval>> availabilityMap1 = [:]
Map<String, Set<Interval>> availabilityMap2 = [:]

[d1, d2, m1, m2, m3].each {
availabilityMap1.put(toColumn(it).getName(), [interval1].toSet())
availabilityMap2.put(toColumn(it).getName(), [interval2].toSet())
availabilityMap1.put(toColumn(it).getName(), [interval1] as Set)
availabilityMap2.put(toColumn(it).getName(), [interval2] as Set)
}

t4h1.setAvailability(new ConcreteAvailability(t4h1.getTableName(), new TestDataSourceMetadataService(availabilityMap1)))
t4d1.setAvailability(new ConcreteAvailability(t4d1.getTableName(), new TestDataSourceMetadataService(availabilityMap1)))

t5h = new ConcretePhysicalTable("table5d", utcHour, [d8, d9, d10, d11, d12, d13, m1].collect{toColumn(it)}.toSet(), [:], metadataService)
t5h = new ConcretePhysicalTable("table5d", utcHour, [d8, d9, d10, d11, d12, d13, m1].collect{toColumn(it)} as Set, [:], metadataService)

t4h2.setAvailability(new ConcreteAvailability(t4h2.getTableName(), new TestDataSourceMetadataService(availabilityMap2)))
t4d2.setAvailability(new ConcreteAvailability(t4d1.getTableName(), new TestDataSourceMetadataService(availabilityMap2)))

setupPartialData()

tg1h = new TableGroup([t1h, t1d, t1hShort] as LinkedHashSet, [m1, m2, m3].collect {buildMockName(it.getName())}.toSet(), [d1].toSet())
tg1d = new TableGroup([t1d] as LinkedHashSet, [m1, m2, m3].collect {buildMockName(it.getName())}.toSet(), [d1].toSet())
tg1Short = new TableGroup([t1hShort] as LinkedHashSet, [m1, m2, m3].collect {buildMockName(it.getName())}.toSet(), [].toSet())
tg2h = new TableGroup([t2h] as LinkedHashSet, [m1, m2, m3].collect {buildMockName(it.getName())}.toSet(), [].toSet())
tg3d = new TableGroup([t3d] as LinkedHashSet, [m1, m2, m3].collect {buildMockName(it.getName())}.toSet(), [].toSet())
tg4h = new TableGroup([t1h, t2h] as LinkedHashSet, [m1, m2, m3].collect {buildMockName(it.getName())}.toSet(), [].toSet())
tg5h = new TableGroup([t2h, t1h] as LinkedHashSet, [m1, m2, m3].collect {buildMockName(it.getName())}.toSet(), [].toSet())
tg6h = new TableGroup([t5h] as LinkedHashSet, [].toSet(), [].toSet())
tgna = new TableGroup([tna1236d, tna1237d, tna167d, tna267d] as LinkedHashSet, [m1, m2, m3].collect {buildMockName(it.getName())}.toSet(), [].toSet())
tg1h = new TableGroup([t1h, t1d, t1hShort] as LinkedHashSet, [m1, m2, m3].collect {buildMockName(it.getName())} as Set, [d1] as Set)
tg1d = new TableGroup([t1d] as LinkedHashSet, [m1, m2, m3].collect {buildMockName(it.getName())} as Set, [d1] as Set)
tg1Short = new TableGroup([t1hShort] as LinkedHashSet, [m1, m2, m3].collect {buildMockName(it.getName())} as Set, [] as Set)
tg2h = new TableGroup([t2h] as LinkedHashSet, [m1, m2, m3].collect {buildMockName(it.getName())} as Set, [] as Set)
tg3d = new TableGroup([t3d] as LinkedHashSet, [m1, m2, m3].collect {buildMockName(it.getName())} as Set, [] as Set)
tg4h = new TableGroup([t1h, t2h] as LinkedHashSet, [m1, m2, m3].collect {buildMockName(it.getName())} as Set, [] as Set)
tg5h = new TableGroup([t2h, t1h] as LinkedHashSet, [m1, m2, m3].collect {buildMockName(it.getName())} as Set, [] as Set)
tg6h = new TableGroup([t5h] as LinkedHashSet, [] as Set, [] as Set)
tgna = new TableGroup([tna1236d, tna1237d, tna167d, tna267d] as LinkedHashSet, [m1, m2, m3].collect {buildMockName(it.getName())} as Set, [] as Set)

lt12 = new LogicalTable("base12", HOUR, tg1h, metricDictionary)
lt13 = new LogicalTable("base13", DAY, tg1d, metricDictionary)
Expand Down Expand Up @@ -314,26 +314,26 @@ public class QueryBuildingTestingResources {
def setupPartialData() {
// In the event of partiality on all data, the coarsest table will be selected and the leftmost of the
// coarsest tables should be selected
emptyFirst = new ConcretePhysicalTable("emptyFirst", MONTH.buildZonedTimeGrain(UTC), [d1, m1, m2, m3].collect{toColumn(it)}.toSet(), [:], metadataService)
emptyLast = new ConcretePhysicalTable("emptyLast", MONTH.buildZonedTimeGrain(UTC), [d1, m1, m2, m3].collect{toColumn(it)}.toSet(), [:], metadataService)
partialSecond = new ConcretePhysicalTable("partialSecond", MONTH.buildZonedTimeGrain(UTC), [d1, m1, m2, m3].collect{toColumn(it)}.toSet(), [:], metadataService)
wholeThird = new ConcretePhysicalTable("wholeThird", MONTH.buildZonedTimeGrain(UTC), [d1, m1, m2, m3].collect{toColumn(it)}.toSet(), [:], metadataService)
emptyFirst = new ConcretePhysicalTable("emptyFirst", MONTH.buildZonedTimeGrain(UTC), [d1, m1, m2, m3].collect{toColumn(it)} as Set, [:], metadataService)
emptyLast = new ConcretePhysicalTable("emptyLast", MONTH.buildZonedTimeGrain(UTC), [d1, m1, m2, m3].collect{toColumn(it)} as Set, [:], metadataService)
partialSecond = new ConcretePhysicalTable("partialSecond", MONTH.buildZonedTimeGrain(UTC), [d1, m1, m2, m3].collect{toColumn(it)} as Set, [:], metadataService)
wholeThird = new ConcretePhysicalTable("wholeThird", MONTH.buildZonedTimeGrain(UTC), [d1, m1, m2, m3].collect{toColumn(it)} as Set, [:], metadataService)

Map<String, Set<Interval>> availabilityMap1 = [:]
Map<String, Set<Interval>> availabilityMap2 = [:]
Map<String, Set<Interval>> availabilityMap3 = [:]

[d1, d2, m1, m2, m3].each {
availabilityMap1.put(toColumn(it).getName(), [new Interval("2015/2015")].toSet())
availabilityMap2.put(toColumn(it).getName(), [new Interval("2015/2016")].toSet())
availabilityMap3.put(toColumn(it).getName(), [new Interval("2011/2016")].toSet())
availabilityMap1.put(toColumn(it).getName(), [new Interval("2015/2015")] as Set)
availabilityMap2.put(toColumn(it).getName(), [new Interval("2015/2016")] as Set)
availabilityMap3.put(toColumn(it).getName(), [new Interval("2011/2016")] as Set)
}
emptyFirst.setAvailability(new ConcreteAvailability(emptyFirst.getTableName(), new TestDataSourceMetadataService(availabilityMap1)))
emptyLast.setAvailability(new ConcreteAvailability(emptyLast.getTableName(), new TestDataSourceMetadataService(availabilityMap1)))
partialSecond.setAvailability(new ConcreteAvailability(partialSecond.getTableName(), new TestDataSourceMetadataService(availabilityMap2)))
wholeThird.setAvailability(new ConcreteAvailability(wholeThird.getTableName(), new TestDataSourceMetadataService(availabilityMap3)))

tg1All = new TableGroup([emptyFirst, partialSecond, wholeThird, emptyLast] as LinkedHashSet, [].toSet(), [].toSet())
tg1All = new TableGroup([emptyFirst, partialSecond, wholeThird, emptyLast] as LinkedHashSet, [] as Set, [] as Set)
ti1All = new TableIdentifier("base1All", AllGranularity.INSTANCE)
}

Expand Down
Loading

0 comments on commit 52ece84

Please sign in to comment.