Add PREFIX aggregation select and PREFIX_TEXT event filter (#134)

* Add Prefix select implementation * Add Prefix select test * Add PREFIX_TEXT filter * Add test case * Refactor string aggregation SQL * Improve PREFIX test case * Fix field data type * Improve string agg SQL * Fix formatting Co-authored-by: Jonas Arnhold <jonas.arnhold@bakdata.com> --------- Co-authored-by: Jonas Arnhold <jonas.arnhold@bakdata.com>
ingef · Sep 27, 2023 · bacf36d · bacf36d
1 parent 0bda81a
commit bacf36d
Show file tree

Hide file tree

Showing 12 changed files with 318 additions and 2 deletions.
diff --git a/...m/bakdata/conquery/sql/conversion/cqelement/concept/filter/PrefixTextFilterConverter.java b/...m/bakdata/conquery/sql/conversion/cqelement/concept/filter/PrefixTextFilterConverter.java
@@ -0,0 +1,49 @@
+package com.bakdata.conquery.sql.conversion.cqelement.concept.filter;
+
+import java.util.Collections;
+import java.util.Set;
+
+import com.bakdata.conquery.models.datasets.concepts.filters.specific.PrefixTextFilter;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.CteStep;
+import com.bakdata.conquery.sql.conversion.model.filter.ConceptFilter;
+import com.bakdata.conquery.sql.conversion.model.filter.Filters;
+import com.bakdata.conquery.sql.conversion.model.filter.PrefixTextCondition;
+import com.bakdata.conquery.sql.conversion.model.select.ExtractingSqlSelect;
+import com.bakdata.conquery.sql.conversion.model.select.SqlSelects;
+import org.jooq.Field;
+
+public class PrefixTextFilterConverter implements FilterConverter<String, PrefixTextFilter> {
+
+	@Override
+	public ConceptFilter convert(PrefixTextFilter prefixTextFilter, FilterContext<String> context) {
+
+		ExtractingSqlSelect<Object> rootSelect = new ExtractingSqlSelect<>(
+				context.getConceptTables().getPredecessorTableName(CteStep.PREPROCESSING),
+				prefixTextFilter.getColumn().getName(),
+				Object.class
+		);
+
+		Field<Object> qualifiedRootSelect = context.getConceptTables().qualifyOnPredecessorTableName(CteStep.EVENT_FILTER, rootSelect.aliased());
+		PrefixTextCondition prefixTextCondition = new PrefixTextCondition(qualifiedRootSelect, context.getValue());
+
+		return new ConceptFilter(
+				SqlSelects.builder()
+						  .forPreprocessingStep(Collections.singletonList(rootSelect))
+						  .build(),
+				Filters.builder()
+					   .event(Collections.singletonList(prefixTextCondition))
+					   .build()
+		);
+	}
+
+	@Override
+	public Set<CteStep> requiredSteps() {
+		return CteStep.withOptionalSteps(CteStep.EVENT_FILTER);
+	}
+
+	@Override
+	public Class<PrefixTextFilter> getConversionClass() {
+		return PrefixTextFilter.class;
+	}
+
+}
diff --git a/...a/com/bakdata/conquery/sql/conversion/cqelement/concept/select/PrefixSelectConverter.java b/...a/com/bakdata/conquery/sql/conversion/cqelement/concept/select/PrefixSelectConverter.java
@@ -0,0 +1,48 @@
+package com.bakdata.conquery.sql.conversion.cqelement.concept.select;
+
+import java.util.Collections;
+
+import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.PrefixSelect;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.CteStep;
+import com.bakdata.conquery.sql.conversion.model.select.ExtractingSqlSelect;
+import com.bakdata.conquery.sql.conversion.model.select.PrefixSqlSelect;
+import com.bakdata.conquery.sql.conversion.model.select.SqlSelects;
+import org.jooq.Field;
+
+public class PrefixSelectConverter implements SelectConverter<PrefixSelect> {
+
+	@Override
+	public SqlSelects convert(PrefixSelect prefixSelect, SelectContext context) {
+		ExtractingSqlSelect<String> rootSelect = new ExtractingSqlSelect<>(
+				context.getConceptTables().getPredecessorTableName(CteStep.PREPROCESSING),
+				prefixSelect.getColumn().getName(),
+				String.class
+		);
+
+		Field<Object> qualifiedRootSelect = context.getConceptTables().qualifyOnPredecessorTableName(CteStep.AGGREGATION_SELECT, rootSelect.aliased());
+		PrefixSqlSelect prefixGroupBy = new PrefixSqlSelect(
+			qualifiedRootSelect, 
+			prefixSelect.getPrefix(), 
+			context.getParentContext().getSqlDialect().getFunction(), 
+			prefixSelect.getName()
+		);
+
+		ExtractingSqlSelect<String> finalSelect = new ExtractingSqlSelect<>(
+				context.getConceptTables().getPredecessorTableName(CteStep.FINAL),
+				prefixGroupBy.aliased().getName(),
+				String.class
+		);
+
+		return SqlSelects.builder()
+						 .forPreprocessingStep(Collections.singletonList(rootSelect))
+						 .forAggregationSelectStep(Collections.singletonList(prefixGroupBy))
+						 .forFinalStep(Collections.singletonList(finalSelect))
+						 .build();
+	}
+
+	@Override
+	public Class<? extends PrefixSelect> getConversionClass() {
+		return PrefixSelect.class;
+	}
+
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java
@@ -21,12 +21,14 @@
 import com.bakdata.conquery.sql.conversion.cqelement.concept.filter.FilterConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.filter.MultiSelectFilterConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.filter.NumberFilterConverter;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.filter.PrefixTextFilterConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.filter.SingleSelectFilterConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.filter.SumFilterConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.select.CountSelectConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.select.DateDistanceSelectConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.select.ExistsSelectConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.select.FirstValueSelectConverter;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.select.PrefixSelectConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.select.SelectConversions;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.select.SelectConverter;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.select.SumSelectConverter;
@@ -87,7 +89,8 @@ default List<SelectConverter<? extends Select>> customizeSelectConverters(List<S
 				new SingleSelectFilterConverter(),
 				new NumberFilterConverter(),
 				new SumFilterConverter(),
-				new CountFilterConverter()
+				new CountFilterConverter(),
+				new PrefixTextFilterConverter()
 		);
 	}
 
@@ -97,7 +100,8 @@ default List<SelectConverter<? extends Select>> getDefaultSelectConverters() {
 				new DateDistanceSelectConverter(DEFAULT_DATE_NOW_SUPPLIER),
 				new ExistsSelectConverter(),
 				new SumSelectConverter(),
-				new CountSelectConverter()
+				new CountSelectConverter(),
+				new PrefixSelectConverter()
 		);
 	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java
@@ -71,4 +71,10 @@ default Field<Date> toDateField(String dateExpression) {
 		return DSL.toDate(dateExpression, DEFAULT_DATE_FORMAT);
 	}
 
+	default Field<Object> prefixStringAggregation(Field<Object> field, String prefix) {
+		Field<String> likePattern = DSL.inline(prefix + "%");
+		String sqlTemplate = "'[' || STRING_AGG(CASE WHEN {0} LIKE {1} THEN {0} ELSE NULL END, ', ') || ']'";
+		return DSL.field(DSL.sql(sqlTemplate, field, likePattern));
+	}
+
 }
diff --git a/...d/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/PrefixTextCondition.java b/...d/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/PrefixTextCondition.java
@@ -0,0 +1,23 @@
+package com.bakdata.conquery.sql.conversion.model.filter;
+
+import lombok.RequiredArgsConstructor;
+import org.jooq.Condition;
+import org.jooq.Field;
+
+@RequiredArgsConstructor
+public class PrefixTextCondition implements FilterCondition {
+
+	private final Field<Object> prefixTextColumn;
+	private final String value;
+
+	@Override
+	public Condition filterCondition() {
+		return prefixTextColumn.like(value + "%");
+	}
+
+	@Override
+	public FilterType type() {
+		return FilterType.EVENT;
+	}
+
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/PrefixSqlSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/PrefixSqlSelect.java
@@ -0,0 +1,34 @@
+package com.bakdata.conquery.sql.conversion.model.select;
+
+import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider;
+import lombok.EqualsAndHashCode;
+import lombok.RequiredArgsConstructor;
+import org.jooq.Field;
+import org.jooq.impl.DSL;
+
+@RequiredArgsConstructor
+@EqualsAndHashCode
+public class PrefixSqlSelect implements SqlSelect {
+
+	private final Field<Object> columnToPrefix;
+	private final String prefix;
+	private final SqlFunctionProvider functionProvider;
+	private final String alias;
+
+	@Override
+	public Field<Object> select() {
+		return functionProvider.prefixStringAggregation(columnToPrefix, prefix).as(alias);
+	}
+
+	@Override
+	public Field<?> aliased() {
+		return DSL.field(alias);
+	}
+
+	@Override
+	public String columnName() {
+		return columnToPrefix.getName();
+	}
+
+}
+
diff --git a/backend/src/test/resources/tests/sql/filter/prefix_text/content.csv b/backend/src/test/resources/tests/sql/filter/prefix_text/content.csv
@@ -0,0 +1,6 @@
+pid,value
+1,"fa"
+2,"fb"
+3,
+4,"f"
+5,"m"
diff --git a/backend/src/test/resources/tests/sql/filter/prefix_text/expected.csv b/backend/src/test/resources/tests/sql/filter/prefix_text/expected.csv
@@ -0,0 +1,4 @@
+result,prefix
+1
+2
+4
diff --git a/backend/src/test/resources/tests/sql/filter/prefix_text/prefix_text.json b/backend/src/test/resources/tests/sql/filter/prefix_text/prefix_text.json
@@ -0,0 +1,67 @@
+{
+  "label": "PREFIX_TEXT filter",
+  "type": "SQL_TEST",
+  "expectedCsv": "expected.csv",
+  "query": {
+    "type": "CONCEPT_QUERY",
+    "root": {
+      "type": "AND",
+      "children": [
+        {
+          "ids": [
+            "prefix"
+          ],
+          "type": "CONCEPT",
+          "label": "prefix",
+          "tables": [
+            {
+              "id": "prefix.prefix_connector",
+              "filters": [
+                {
+                  "filter": "prefix.prefix_connector.prefix_filter",
+                  "type": "SELECT",
+                  "value": "f"
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  },
+  "concepts": [
+    {
+      "label": "prefix",
+      "type": "TREE",
+      "connectors": [
+        {
+          "label": "prefix_connector",
+          "table": "table1",
+          "filters": {
+            "label": "prefix_filter",
+            "column": "table1.value",
+            "type": "PREFIX_TEXT"
+          }
+        }
+      ]
+    }
+  ],
+  "content": {
+    "tables": [
+      {
+        "csv": "tests/sql/filter/prefix_text/content.csv",
+        "name": "table1",
+        "primaryColumn": {
+          "name": "pid",
+          "type": "STRING"
+        },
+        "columns": [
+          {
+            "name": "value",
+            "type": "STRING"
+          }
+        ]
+      }
+    ]
+  }
+}
diff --git a/backend/src/test/resources/tests/sql/selects/prefix/content.csv b/backend/src/test/resources/tests/sql/selects/prefix/content.csv
@@ -0,0 +1,6 @@
+pid,prefix
+1,"fa"
+1,"fb"
+2,
+3,"f"
+4,"m"
diff --git a/backend/src/test/resources/tests/sql/selects/prefix/expected.csv b/backend/src/test/resources/tests/sql/selects/prefix/expected.csv
@@ -0,0 +1,5 @@
+result,prefix
+1,"[fa, fb]"
+2,
+3,[f]
+4,
diff --git a/backend/src/test/resources/tests/sql/selects/prefix/prefix.json b/backend/src/test/resources/tests/sql/selects/prefix/prefix.json
@@ -0,0 +1,64 @@
+{
+  "label": "PREFIX select",
+  "type": "SQL_TEST",
+  "expectedCsv": "expected.csv",
+  "query": {
+    "type": "CONCEPT_QUERY",
+    "root": {
+      "type": "AND",
+      "children": [
+        {
+          "ids": [
+            "prefix"
+          ],
+          "type": "CONCEPT",
+          "label": "prefix",
+          "tables": [
+            {
+              "id": "prefix.prefix_connector",
+              "selects": [
+                "prefix.prefix_connector.prefix_select"
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  },
+  "concepts": [
+    {
+      "label": "prefix",
+      "type": "TREE",
+      "connectors": [
+        {
+          "label": "prefix_connector",
+          "table": "table1",
+          "selects": {
+            "type": "PREFIX",
+            "name": "prefix_select",
+            "column": "table1.prefix",
+            "prefix": "f"
+          }
+        }
+      ]
+    }
+  ],
+  "content": {
+    "tables": [
+      {
+        "csv": "tests/sql/selects/prefix/content.csv",
+        "name": "table1",
+        "primaryColumn": {
+          "name": "pid",
+          "type": "STRING"
+        },
+        "columns": [
+          {
+            "name": "prefix",
+            "type": "STRING"
+          }
+        ]
+      }
+    ]
+  }
+}