Implement abstraction for date restriction and validity date date ran…

…ges (#66)
ingef · Jun 28, 2023 · e47718e · e47718e
1 parent 331eb3e
commit e47718e
Show file tree

Hide file tree

Showing 41 changed files with 636 additions and 276 deletions.
diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/CQTable.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/CQTable.java
@@ -91,6 +91,7 @@ public Column findValidityDateColumn() {
 		return null;
 	}
 
+	// TODO: replace above usage function with findValidityDate()
 	@CheckForNull
 	public ValidityDate findValidityDate() {
 

diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java
@@ -1,9 +1,8 @@
 package com.bakdata.conquery.sql.conversion.context;
 
-import java.time.LocalDate;
 import java.util.List;
 
-import com.bakdata.conquery.models.common.Range;
+import com.bakdata.conquery.models.common.daterange.CDateRange;
 import com.bakdata.conquery.models.config.SqlConnectorConfig;
 import com.bakdata.conquery.sql.conversion.NodeConverterService;
 import com.bakdata.conquery.sql.conversion.context.step.QueryStep;
@@ -26,11 +25,11 @@ public class ConversionContext {
 	@Singular
 	List<QueryStep> querySteps;
 	Select<Record> finalQuery;
-	Range<LocalDate> dateRestricionRange;
+	CDateRange dateRestrictionRange;
 	int queryStepCounter;
 
 	public boolean dateRestrictionActive() {
-		return this.dateRestricionRange != null;
+		return this.dateRestrictionRange != null;
 	}
 
 	/**

diff --git a/...end/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java b/...end/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java
@@ -1,14 +1,13 @@
 package com.bakdata.conquery.sql.conversion.context.selects;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
-import java.util.function.Function;
 import java.util.stream.Stream;
 
 import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept;
-import lombok.AccessLevel;
+import com.bakdata.conquery.sql.models.ColumnDateRange;
 import lombok.Builder;
-import lombok.Getter;
 import lombok.Value;
 import org.jooq.Field;
 
@@ -21,10 +20,9 @@ public class ConceptSelects implements Selects {
 
 	Field<Object> primaryColumn;
 
-	Optional<Field<Object>> dateRestriction;
+	Optional<ColumnDateRange> dateRestrictionRange;
 
-	@Getter(AccessLevel.PRIVATE)
-	Optional<Field<Object>> validityDate;
+	Optional<ColumnDateRange> validityDate;
 
 	List<Field<Object>> eventSelect;
 
@@ -35,16 +33,18 @@ public class ConceptSelects implements Selects {
 	List<Field<Object>> groupFilter;
 
 	@Override
-	public List<Field<Object>> getValidityDates() {
-		return this.validityDate.stream().toList();
+	public Selects withValidityDate(ColumnDateRange validityDate) {
+		return this.toBuilder()
+				   .validityDate(Optional.of(validityDate))
+				   .build();
 	}
 
 	@Override
 	public ConceptSelects byName(String qualifier) {
 		return builder()
 				.primaryColumn(this.mapFieldToQualifier(qualifier, this.primaryColumn))
-				.dateRestriction(this.mapFieldStreamToQualifier(qualifier, this.dateRestriction.stream()).findFirst())
-				.validityDate(this.mapFieldStreamToQualifier(qualifier, this.validityDate.stream()).findFirst())
+				.dateRestrictionRange(this.dateRestrictionRange.map(dateRestriction -> dateRestriction.qualify(qualifier)))
+				.validityDate(this.validityDate.map(validityDate -> validityDate.qualify(qualifier)))
 				.eventSelect(this.mapFieldStreamToQualifier(qualifier, this.eventSelect.stream()).toList())
 				.eventFilter(this.mapFieldStreamToQualifier(qualifier, this.eventFilter.stream()).toList())
 				.groupSelect(this.mapFieldStreamToQualifier(qualifier, this.groupSelect.stream()).toList())
@@ -63,19 +63,22 @@ public List<Field<Object>> all() {
 	private Stream<Field<Object>> primaryColumnAndValidityDate() {
 		return Stream.concat(
 				Stream.of(this.primaryColumn),
-				this.validityDate.stream()
+				this.validityDate.isPresent() ? this.validityDate.get().toFields().stream() : Stream.of()
 		);
 	}
 
 	@Override
 	public List<Field<Object>> explicitSelects() {
-		return Stream.of(
-				this.dateRestriction.stream(),
-				this.eventSelect.stream(),
-				this.eventFilter.stream(),
-				this.groupSelect.stream(),
-				this.groupFilter.stream()
-		).flatMap(Function.identity()).toList();
+
+		List<Field<Object>> explicitSelects = new ArrayList<>();
+
+		dateRestrictionRange.ifPresent(columnDateRange -> explicitSelects.addAll(columnDateRange.toFields()));
+		explicitSelects.addAll(eventSelect);
+		explicitSelects.addAll(eventFilter);
+		explicitSelects.addAll(groupSelect);
+		explicitSelects.addAll(groupFilter);
+
+		return explicitSelects;
 	}
 
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java
@@ -1,9 +1,11 @@
 package com.bakdata.conquery.sql.conversion.context.selects;
 
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Stream;
 
 import com.bakdata.conquery.sql.conversion.context.step.QueryStep;
+import com.bakdata.conquery.sql.models.ColumnDateRange;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Value;
@@ -22,9 +24,9 @@ public class MergedSelects implements Selects {
 	Field<Object> primaryColumn;
 
 	/**
-	 * A list of the validity dates of each {@link QueryStep} passed to the {@link MergedSelects} constructor.
+	 * An aggregated validity date of all validity dates of each {@link QueryStep} passed to the {@link MergedSelects} constructor.
 	 */
-	List<Field<Object>> validityDates;
+	Optional<ColumnDateRange> validityDate;
 
 	/**
 	 * A merged list of all select fields, except the primary column and validity date,
@@ -35,37 +37,24 @@ public class MergedSelects implements Selects {
 
 	public MergedSelects(List<QueryStep> querySteps) {
 		this.primaryColumn = this.coalescePrimaryColumns(querySteps);
-		this.validityDates = this.extractValidityDates(querySteps);
+		this.validityDate = this.extractValidityDates(querySteps);
 		this.mergedSelects = this.mergeSelects(querySteps);
 	}
 
-	private Field<Object> coalescePrimaryColumns(List<QueryStep> querySteps) {
-		List<Field<Object>> primaryColumns = querySteps.stream()
-													   .map(queryStep -> this.mapFieldToQualifier(queryStep.getCteName(), queryStep.getSelects().getPrimaryColumn()))
-													   .toList();
-		return DSL.coalesce((Object) primaryColumns.get(0), primaryColumns.subList(1, primaryColumns.size()).toArray())
-				  .as("primary_column");
-	}
-
-	private List<Field<Object>> extractValidityDates(List<QueryStep> querySteps) {
-		// TODO: date aggregation...
-		return querySteps.stream()
-						 .flatMap(queryStep -> this.mapFieldStreamToQualifier(queryStep.getCteName(), queryStep.getSelects().getValidityDates().stream()))
-						 .toList();
-	}
-
-	private List<Field<Object>> mergeSelects(List<QueryStep> queriesToJoin) {
-		return queriesToJoin.stream()
-							.flatMap(queryStep -> queryStep.getSelects().explicitSelects().stream()
-														   .map(field -> this.mapFieldToQualifier(queryStep.getCteName(), field)))
-							.toList();
+	@Override
+	public Selects withValidityDate(ColumnDateRange validityDate) {
+		return new MergedSelects(
+				this.primaryColumn,
+				Optional.of(validityDate),
+				this.mergedSelects
+		);
 	}
 
 	@Override
 	public MergedSelects byName(String qualifier) {
 		return new MergedSelects(
 				this.mapFieldToQualifier(qualifier, this.primaryColumn),
-				this.mapFieldStreamToQualifier(qualifier, this.validityDates.stream()).toList(),
+				this.validityDate.map(columnDateRange -> columnDateRange.qualify(qualifier)),
 				this.mapFieldStreamToQualifier(qualifier, this.mergedSelects.stream()).toList()
 		);
 	}
@@ -78,16 +67,43 @@ public List<Field<Object>> all() {
 		).toList();
 	}
 
+	@Override
+	public List<Field<Object>> explicitSelects() {
+		return this.mergedSelects;
+	}
+
+	private Field<Object> coalescePrimaryColumns(List<QueryStep> querySteps) {
+		List<Field<Object>> primaryColumns = querySteps.stream()
+													   .map(queryStep -> this.mapFieldToQualifier(queryStep.getCteName(), queryStep.getSelects()
+																																   .getPrimaryColumn()))
+													   .toList();
+		return DSL.coalesce((Object) primaryColumns.get(0), primaryColumns.subList(1, primaryColumns.size()).toArray())
+				  .as("primary_column");
+	}
+
+	private Optional<ColumnDateRange> extractValidityDates(List<QueryStep> querySteps) {
+        // TODO: date aggregation...
+		return querySteps.stream()
+						 .filter(queryStep -> queryStep.getSelects().getValidityDate().isPresent())
+						 .map(queryStep -> {
+							 ColumnDateRange validityDate = queryStep.getSelects().getValidityDate().get();
+							 return validityDate.qualify(queryStep.getCteName());
+						 })
+						 .findFirst();
+	}
+
+	private List<Field<Object>> mergeSelects(List<QueryStep> queriesToJoin) {
+		return queriesToJoin.stream()
+							.flatMap(queryStep -> queryStep.getSelects().explicitSelects().stream()
+														   .map(field -> this.mapFieldToQualifier(queryStep.getCteName(), field)))
+							.toList();
+	}
+
 	private Stream<Field<Object>> primaryColumnAndValidityDate() {
 		return Stream.concat(
 				Stream.of(this.primaryColumn),
-				this.validityDates.stream()
+				this.validityDate.isPresent() ? this.validityDate.get().toFields().stream() : Stream.empty()
 		);
 	}
 
-	@Override
-	public List<Field<Object>> explicitSelects() {
-		return this.mergedSelects;
-	}
-
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java
@@ -1,20 +1,20 @@
 package com.bakdata.conquery.sql.conversion.context.selects;
 
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Stream;
 
+import com.bakdata.conquery.sql.models.ColumnDateRange;
 import org.jooq.Field;
 import org.jooq.impl.DSL;
 
 public interface Selects {
 
 	Field<Object> getPrimaryColumn();
 
-	/**
-	 * @return An empty list in case there is no validity date. Exactly 1 validity date in case of {@link ConceptSelects}.
-	 * 	 An arbitrary amount of validity dates in case of {@link MergedSelects}.
-	 */
-	List<Field<Object>> getValidityDates();
+	Optional<ColumnDateRange> getValidityDate();
+
+	Selects withValidityDate(ColumnDateRange validityDate);
 
 	/**
 	 * Returns the selected columns as fully qualified reference.

diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java
@@ -73,23 +73,38 @@ private QueryStep buildDateRestrictionQueryStep(
 			String conceptLabel,
 			QueryStep previous
 	) {
-		if (((ConceptSelects) previous.getSelects()).getDateRestriction().isEmpty()) {
+		if (((ConceptSelects) previous.getSelects()).getDateRestrictionRange().isEmpty()) {
 			return previous;
 		}
 
 		ConceptSelects dateRestrictionSelects = this.prepareDateRestrictionSelects(conceptNode, previous);
-		List<Condition> dateRestriction = this.buildDateRestriction(context, previous);
+		Condition dateRestriction = this.buildDateRestriction(context, previous);
 		String dateRestrictionCteName = "concept_%s_date_restriction".formatted(conceptLabel);
 
 		return QueryStep.builder()
 						.cteName(dateRestrictionCteName)
 						.fromTable(QueryStep.toTableLike(previous.getCteName()))
 						.selects(dateRestrictionSelects)
-						.conditions(dateRestriction)
+						.conditions(List.of(dateRestriction))
 						.predecessors(List.of(previous))
 						.build();
 	}
 
+	private ConceptSelects prepareDateRestrictionSelects(CQConcept conceptNode, QueryStep previous) {
+		ConceptSelects.ConceptSelectsBuilder selectsBuilder = ((ConceptSelects) previous.getQualifiedSelects()).toBuilder();
+		selectsBuilder.dateRestrictionRange(Optional.empty());
+		if (conceptNode.isExcludeFromTimeAggregation()) {
+			selectsBuilder.validityDate(Optional.empty());
+		}
+		return selectsBuilder.build();
+	}
+
+	private Condition buildDateRestriction(ConversionContext context, QueryStep previous) {
+		ConceptSelects previousSelects = (ConceptSelects) previous.getSelects();
+		return context.getSqlDialect().getFunction()
+					  .dateRestriction(previousSelects.getDateRestrictionRange().get(), previousSelects.getValidityDate().get());
+	}
+
 	/**
 	 * selects:
 	 * - all of previous steps
@@ -160,27 +175,6 @@ private QueryStep buildFinalQueryStep(String conceptLabel, QueryStep previous) {
 						.build();
 	}
 
-	private ConceptSelects prepareDateRestrictionSelects(CQConcept conceptNode, QueryStep previous) {
-		ConceptSelects.ConceptSelectsBuilder selectsBuilder = ((ConceptSelects) previous.getQualifiedSelects()).toBuilder();
-		selectsBuilder.dateRestriction(Optional.empty());
-		if (conceptNode.isExcludeFromTimeAggregation()) {
-			selectsBuilder.validityDate(Optional.empty());
-		}
-		return selectsBuilder.build();
-	}
-
-	private List<Condition> buildDateRestriction(ConversionContext context, QueryStep previous) {
-		return ((ConceptSelects) previous.getSelects()).getDateRestriction()
-													   .map(dateRestrictionColumn -> getDateRestrictionAsCondition(context, previous, dateRestrictionColumn))
-													   .orElseGet(List::of);
-	}
-
-	private List<Condition> getDateRestrictionAsCondition(ConversionContext context, QueryStep previous, Field<Object> dateRestrictionColumn) {
-		return previous.getSelects().getValidityDates().stream()
-					   .map(validityDateColumn -> context.getSqlDialect().getFunction().dateRestriction(dateRestrictionColumn, validityDateColumn))
-					   .toList();
-	}
-
 	private ConceptSelects prepareEventSelectSelects(
 			ConversionContext context,
 			CQTable table,

diff --git a/...c/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java b/...c/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java
@@ -1,6 +1,7 @@
 package com.bakdata.conquery.sql.conversion.cqelement;
 
 import com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction;
+import com.bakdata.conquery.models.common.daterange.CDateRange;
 import com.bakdata.conquery.sql.conversion.NodeConverter;
 import com.bakdata.conquery.sql.conversion.context.ConversionContext;
 
@@ -13,8 +14,8 @@ public Class<CQDateRestriction> getConversionClass() {
 
 	@Override
 	public ConversionContext convert(CQDateRestriction dateRestrictionNode, ConversionContext context) {
-		ConversionContext childContext = context.withDateRestricionRange(dateRestrictionNode.getDateRange());
-		return context.getNodeConverterService().convert(dateRestrictionNode.getChild(), childContext).withDateRestricionRange(null);
+		ConversionContext childContext = context.withDateRestrictionRange(CDateRange.of(dateRestrictionNode.getDateRange()));
+		return context.getNodeConverterService().convert(dateRestrictionNode.getChild(), childContext).withDateRestrictionRange(null);
 	}
 
 }