-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add interval packing for concepts (#130)
Co-authored-by: Torben Meyer <torben.meyer@bakdata.com>
- Loading branch information
Showing
121 changed files
with
1,676 additions
and
1,180 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
62 changes: 24 additions & 38 deletions
62
...main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,61 +1,47 @@ | ||
| package com.bakdata.conquery.sql.conversion.cqelement.concept; | ||
|
|
||
| import java.util.List; | ||
| import java.util.Optional; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; | ||
| import com.bakdata.conquery.sql.conversion.model.ConceptSelects; | ||
| import com.bakdata.conquery.sql.conversion.model.filter.FilterCondition; | ||
| import com.bakdata.conquery.sql.conversion.model.QueryStep; | ||
| import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; | ||
| import com.bakdata.conquery.sql.conversion.model.Selects; | ||
| import com.bakdata.conquery.sql.conversion.model.filter.FilterCondition; | ||
| import com.bakdata.conquery.sql.conversion.model.select.ExistsSqlSelect; | ||
| import com.bakdata.conquery.sql.conversion.model.select.ExtractingSqlSelect; | ||
| import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; | ||
| import org.jooq.Condition; | ||
|
|
||
| class AggregationFilterCte extends ConceptCte { | ||
|
|
||
| @Override | ||
| public QueryStep.QueryStepBuilder convertStep(CteContext cteContext) { | ||
|
|
||
| String aggregationFilterPredecessorCte = cteContext.getConceptTables().getPredecessorTableName(CteStep.AGGREGATION_FILTER); | ||
|
|
||
| final Optional<ColumnDateRange> validityDate; | ||
| if (cteContext.isExcludedFromDateAggregation()) { | ||
| validityDate = Optional.empty(); | ||
| } | ||
| else { | ||
| validityDate = cteContext.getValidityDateRange().map(_validityDate -> _validityDate.qualify(aggregationFilterPredecessorCte)); | ||
| } | ||
|
|
||
| ConceptSelects aggregationFilterSelect = new ConceptSelects( | ||
| cteContext.getPrimaryColumn(), | ||
| validityDate, | ||
| getAggregationFilterSelects(cteContext, aggregationFilterPredecessorCte) | ||
| public QueryStep.QueryStepBuilder convertStep(ConceptCteContext conceptCteContext) { | ||
|
|
||
| Selects aggregationFilterSelects = Selects.qualified( | ||
| conceptCteContext.getConceptTables().getPredecessorTableName(ConceptCteStep.AGGREGATION_FILTER), | ||
| conceptCteContext.getPrimaryColumn(), | ||
| getForAggregationFilterSelects(conceptCteContext) | ||
| ); | ||
| List<Condition> aggregationFilterConditions = cteContext.getFilters().stream() | ||
| .flatMap(conceptFilter -> conceptFilter.getFilters().getGroup().stream()) | ||
| .map(FilterCondition::filterCondition) | ||
| .toList(); | ||
|
|
||
| List<Condition> aggregationFilterConditions = conceptCteContext.getFilters().stream() | ||
| .flatMap(conceptFilter -> conceptFilter.getFilters().getGroup().stream()) | ||
| .map(FilterCondition::filterCondition) | ||
| .toList(); | ||
|
|
||
| return QueryStep.builder() | ||
| .selects(aggregationFilterSelect) | ||
| .selects(aggregationFilterSelects) | ||
| .conditions(aggregationFilterConditions); | ||
| } | ||
|
|
||
| private List<SqlSelect> getAggregationFilterSelects(CteContext cteContext, String aggregationFilterPredecessorCte) { | ||
| return cteContext.getSelects().stream() | ||
| .flatMap(sqlSelects -> sqlSelects.getForFinalStep().stream()) | ||
| // TODO: EXISTS edge case is only in a concepts final select statement and has no predecessor selects | ||
| .filter(conquerySelect -> !(conquerySelect instanceof ExistsSqlSelect)) | ||
| .map(conquerySelect -> ExtractingSqlSelect.fromConquerySelect(conquerySelect, aggregationFilterPredecessorCte)) | ||
| .distinct() | ||
| .collect(Collectors.toList()); | ||
| private List<SqlSelect> getForAggregationFilterSelects(ConceptCteContext conceptCteContext) { | ||
| return conceptCteContext.getSelects().stream() | ||
| .flatMap(sqlSelects -> sqlSelects.getForFinalStep().stream()) | ||
| // TODO: EXISTS edge case is only in a concepts final select statement and has no predecessor selects | ||
| .filter(conquerySelect -> !(conquerySelect instanceof ExistsSqlSelect)) | ||
| .distinct() | ||
| .toList(); | ||
| } | ||
|
|
||
| @Override | ||
| public CteStep cteStep() { | ||
| return CteStep.AGGREGATION_FILTER; | ||
| public ConceptCteStep cteStep() { | ||
| return ConceptCteStep.AGGREGATION_FILTER; | ||
| } | ||
|
|
||
| } |
44 changes: 12 additions & 32 deletions
44
...main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,51 +1,31 @@ | ||
| package com.bakdata.conquery.sql.conversion.cqelement.concept; | ||
|
|
||
| import java.util.List; | ||
| import java.util.Optional; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; | ||
| import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; | ||
| import com.bakdata.conquery.sql.conversion.model.ConceptSelects; | ||
| import com.bakdata.conquery.sql.conversion.model.QueryStep; | ||
| import com.bakdata.conquery.sql.conversion.model.Selects; | ||
| import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; | ||
|
|
||
| class AggregationSelectCte extends ConceptCte { | ||
|
|
||
| @Override | ||
| public QueryStep.QueryStepBuilder convertStep(CteContext cteContext) { | ||
|
|
||
| // all selects that are required in the aggregation filter step | ||
| String previousCteName = cteContext.getPrevious().getCteName(); | ||
| List<SqlSelect> aggregationFilterSelects = cteContext.allConceptSelects() | ||
| .flatMap(sqlSelects -> sqlSelects.getForAggregationSelectStep().stream()) | ||
| .distinct() | ||
| .collect(Collectors.toList()); | ||
|
|
||
| SqlFunctionProvider functionProvider = cteContext.getContext().getSqlDialect().getFunction(); | ||
| Optional<ColumnDateRange> aggregatedValidityDate = cteContext.getValidityDateRange() | ||
| .map(validityDate -> validityDate.qualify(previousCteName)) | ||
| .map(functionProvider::aggregated) | ||
| .map(validityDate -> validityDate.asValidityDateRange(cteContext.getConceptLabel())); | ||
|
|
||
| ConceptSelects aggregationSelectSelects = new ConceptSelects( | ||
| cteContext.getPrimaryColumn(), | ||
| aggregatedValidityDate, | ||
| aggregationFilterSelects | ||
| ); | ||
| public QueryStep.QueryStepBuilder convertStep(ConceptCteContext conceptCteContext) { | ||
|
|
||
| List<SqlSelect> requiredInAggregationFilterStep = conceptCteContext.allConceptSelects() | ||
| .flatMap(sqlSelects -> sqlSelects.getForAggregationSelectStep().stream()) | ||
| .distinct() | ||
| .toList(); | ||
|
|
||
| Selects aggregationSelectSelects = new Selects(conceptCteContext.getPrimaryColumn(), requiredInAggregationFilterStep); | ||
|
|
||
| return QueryStep.builder() | ||
| // pid normally | ||
| // first value for all existing selects | ||
| // date aggregation for date range | ||
| // new select for all aggregation selects and filter | ||
| .selects(aggregationSelectSelects) | ||
| .isGroupBy(true); | ||
| .groupBy(List.of(conceptCteContext.getPrimaryColumn())); | ||
| } | ||
|
|
||
| @Override | ||
| public CteStep cteStep() { | ||
| return CteStep.AGGREGATION_SELECT; | ||
| public ConceptCteStep cteStep() { | ||
| return ConceptCteStep.AGGREGATION_SELECT; | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.