From ecf0b1e50db572a6fc61b975c05e6068f582b5c6 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Wed, 19 Nov 2025 11:27:33 +0530 Subject: [PATCH 01/14] ArrayIdentifierExpression and JsonArrayIdentifierExpression to pass type info to flat collections --- .../impl/ArrayIdentifierExpression.java | 22 ++++++++ .../impl/JsonArrayIdentifierExpression.java | 50 +++++++++++++++++++ .../impl/JsonIdentifierExpression.java | 6 +-- 3 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/ArrayIdentifierExpression.java create mode 100644 document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/JsonArrayIdentifierExpression.java diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/ArrayIdentifierExpression.java b/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/ArrayIdentifierExpression.java new file mode 100644 index 00000000..e3e3d31f --- /dev/null +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/ArrayIdentifierExpression.java @@ -0,0 +1,22 @@ +package org.hypertrace.core.documentstore.expression.impl; + +import lombok.EqualsAndHashCode; + +/** + * Represents an identifier expression for array-typed fields. This allows parsers to apply + * array-specific logic (e.g., cardinality checks for EXISTS operators to exclude empty arrays). + * + *

Similar to {@link JsonIdentifierExpression}, this provides type information to parsers so they + * can generate appropriate database-specific queries for array operations. + */ +@EqualsAndHashCode(callSuper = true) +public class ArrayIdentifierExpression extends IdentifierExpression { + + public ArrayIdentifierExpression(String name) { + super(name); + } + + public static ArrayIdentifierExpression of(String name) { + return new ArrayIdentifierExpression(name); + } +} diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/JsonArrayIdentifierExpression.java b/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/JsonArrayIdentifierExpression.java new file mode 100644 index 00000000..efde92d8 --- /dev/null +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/JsonArrayIdentifierExpression.java @@ -0,0 +1,50 @@ +package org.hypertrace.core.documentstore.expression.impl; + +import java.util.List; +import lombok.EqualsAndHashCode; +import org.hypertrace.core.documentstore.postgres.utils.BasicPostgresSecurityValidator; + +/** + * Represents an identifier expression for array-typed fields inside JSONB columns. This allows + * parsers to apply array-specific logic (e.g., jsonb_array_length checks for EXISTS operators to + * exclude empty arrays). + * + *

Example: For a JSONB column "attributes" with a nested array field "tags": + * + * 

{"attributes": {"tags": ["value1", "value2"]}}
+ * + * Use: {@code JsonArrayIdentifierExpression.of("attributes", "tags")} + */ +@EqualsAndHashCode(callSuper = true) +public class JsonArrayIdentifierExpression extends JsonIdentifierExpression { + + public static JsonArrayIdentifierExpression of( + final String columnName, final String... pathElements) { + if (pathElements == null || pathElements.length == 0) { + throw new IllegalArgumentException("JSON path cannot be null or empty for array field"); + } + return of(columnName, List.of(pathElements)); + } + + public static JsonArrayIdentifierExpression of( + final String columnName, final List jsonPath) { + // Validate inputs + BasicPostgresSecurityValidator.getDefault().validateIdentifier(columnName); + + if (jsonPath == null || jsonPath.isEmpty()) { + throw new IllegalArgumentException("JSON path cannot be null or empty for array field"); + } + + BasicPostgresSecurityValidator.getDefault().validateJsonPath(jsonPath); + + List unmodifiablePath = List.copyOf(jsonPath); + + // Construct full name for compatibility: "customAttr.myAttribute" + String fullName = columnName + "." + String.join(".", unmodifiablePath); + return new JsonArrayIdentifierExpression(fullName, columnName, unmodifiablePath); + } + + private JsonArrayIdentifierExpression(String name, String columnName, List jsonPath) { + super(name, columnName, jsonPath); + } +} diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/JsonIdentifierExpression.java b/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/JsonIdentifierExpression.java index 277af0f2..a11ac38d 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/JsonIdentifierExpression.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/JsonIdentifierExpression.java @@ -2,7 +2,7 @@ import java.util.List; import lombok.EqualsAndHashCode; -import lombok.Value; +import lombok.Getter; import org.hypertrace.core.documentstore.parser.FieldTransformationVisitor; import org.hypertrace.core.documentstore.postgres.utils.BasicPostgresSecurityValidator; @@ -13,7 +13,7 @@ * *

This generates SQL like: customAttr -> 'myAttribute' -> 'nestedField' (returns JSON) */ -@Value +@Getter @EqualsAndHashCode(callSuper = true) public class JsonIdentifierExpression extends IdentifierExpression { @@ -44,7 +44,7 @@ public static JsonIdentifierExpression of(final String columnName, final List jsonPath) { + protected JsonIdentifierExpression(String name, String columnName, List jsonPath) { super(name); this.columnName = columnName; this.jsonPath = jsonPath; From aaf391b6c78e89fe16da59bb7b1b41ffc48089a8 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Wed, 19 Nov 2025 11:53:52 +0530 Subject: [PATCH 02/14] Added test cases --- .../impl/ArrayIdentifierExpressionTest.java | 72 +++++ .../JsonArrayIdentifierExpressionTest.java | 257 ++++++++++++++++++ 2 files changed, 329 insertions(+) create mode 100644 document-store/src/test/java/org/hypertrace/core/documentstore/expression/impl/ArrayIdentifierExpressionTest.java create mode 100644 document-store/src/test/java/org/hypertrace/core/documentstore/expression/impl/JsonArrayIdentifierExpressionTest.java diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/expression/impl/ArrayIdentifierExpressionTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/expression/impl/ArrayIdentifierExpressionTest.java new file mode 100644 index 00000000..98d5287b --- /dev/null +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/expression/impl/ArrayIdentifierExpressionTest.java @@ -0,0 +1,72 @@ +package org.hypertrace.core.documentstore.expression.impl; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import org.junit.jupiter.api.Test; + +class ArrayIdentifierExpressionTest { + + @Test + void testOfCreatesInstance() { + ArrayIdentifierExpression expression = ArrayIdentifierExpression.of("tags"); + + assertEquals("tags", expression.getName()); + } + + @Test + void testEqualsAndHashCode() { + ArrayIdentifierExpression expr1 = ArrayIdentifierExpression.of("tags"); + ArrayIdentifierExpression expr2 = ArrayIdentifierExpression.of("tags"); + + // Test equality - should be equal + assertEquals(expr1, expr2, "Expressions with same name should be equal"); + + // Test hashCode + assertEquals( + expr1.hashCode(), expr2.hashCode(), "Expressions with same name should have same hashCode"); + } + + @Test + void testNotEqualsWithDifferentName() { + ArrayIdentifierExpression expr1 = ArrayIdentifierExpression.of("tags"); + ArrayIdentifierExpression expr2 = ArrayIdentifierExpression.of("categories"); + + // Test inequality + assertNotEquals(expr1, expr2, "Expressions with different names should not be equal"); + } + + @Test + void testNotEqualsWithIdentifierExpression() { + ArrayIdentifierExpression arrayExpr = ArrayIdentifierExpression.of("tags"); + IdentifierExpression identExpr = IdentifierExpression.of("tags"); + + // Even though they have the same name, they are different types + assertNotEquals( + arrayExpr, identExpr, "ArrayIdentifierExpression should not equal IdentifierExpression"); + } + + @Test + void testInheritsFromIdentifierExpression() { + ArrayIdentifierExpression expression = ArrayIdentifierExpression.of("tags"); + + // Verify it's an instance of parent class + assertEquals( + IdentifierExpression.class, + expression.getClass().getSuperclass(), + "ArrayIdentifierExpression should extend IdentifierExpression"); + } + + @Test + void testMultipleInstancesWithSameNameAreEqual() { + ArrayIdentifierExpression expr1 = ArrayIdentifierExpression.of("categoryTags"); + ArrayIdentifierExpression expr2 = ArrayIdentifierExpression.of("categoryTags"); + ArrayIdentifierExpression expr3 = ArrayIdentifierExpression.of("categoryTags"); + + assertEquals(expr1, expr2); + assertEquals(expr2, expr3); + assertEquals(expr1, expr3); + assertEquals(expr1.hashCode(), expr2.hashCode()); + assertEquals(expr2.hashCode(), expr3.hashCode()); + } +} diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/expression/impl/JsonArrayIdentifierExpressionTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/expression/impl/JsonArrayIdentifierExpressionTest.java new file mode 100644 index 00000000..02a29b8a --- /dev/null +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/expression/impl/JsonArrayIdentifierExpressionTest.java @@ -0,0 +1,257 @@ +package org.hypertrace.core.documentstore.expression.impl; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.Test; + +class JsonArrayIdentifierExpressionTest { + + @Test + void testOfWithVarargs() { + JsonArrayIdentifierExpression expression = + JsonArrayIdentifierExpression.of("attributes", "tags"); + + assertEquals("attributes", expression.getColumnName()); + assertEquals(List.of("tags"), expression.getJsonPath()); + } + + @Test + void testOfWithMultiplePathElements() { + JsonArrayIdentifierExpression expression = + JsonArrayIdentifierExpression.of("attributes", "nested", "array", "field"); + + assertEquals("attributes", expression.getColumnName()); + assertEquals(List.of("nested", "array", "field"), expression.getJsonPath()); + } + + @Test + void testOfWithList() { + JsonArrayIdentifierExpression expression = + JsonArrayIdentifierExpression.of("attributes", List.of("certifications")); + + assertEquals("attributes", expression.getColumnName()); + assertEquals(List.of("certifications"), expression.getJsonPath()); + } + + @Test + void testOfWithNullPathElementsThrowsException() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> JsonArrayIdentifierExpression.of("attributes", (String[]) null)); + + assertEquals("JSON path cannot be null or empty for array field", exception.getMessage()); + } + + @Test + void testOfWithEmptyPathElementsThrowsException() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, () -> JsonArrayIdentifierExpression.of("attributes")); + + assertEquals("JSON path cannot be null or empty for array field", exception.getMessage()); + } + + @Test + void testOfWithNullListThrowsException() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> JsonArrayIdentifierExpression.of("attributes", (List) null)); + + assertEquals("JSON path cannot be null or empty", exception.getMessage()); + } + + @Test + void testOfWithEmptyListThrowsException() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> JsonArrayIdentifierExpression.of("attributes", Collections.emptyList())); + + assertEquals("JSON path cannot be null or empty", exception.getMessage()); + } + + @Test + void testEqualsAndHashCode() { + JsonArrayIdentifierExpression expr1 = JsonArrayIdentifierExpression.of("attributes", "tags"); + JsonArrayIdentifierExpression expr2 = JsonArrayIdentifierExpression.of("attributes", "tags"); + + // Test equality + assertEquals(expr1, expr2, "Expressions with same column and path should be equal"); + + // Test hashCode + assertEquals( + expr1.hashCode(), + expr2.hashCode(), + "Expressions with same column and path should have same hashCode"); + } + + @Test + void testNotEqualsWithDifferentPath() { + JsonArrayIdentifierExpression expr1 = JsonArrayIdentifierExpression.of("attributes", "tags"); + JsonArrayIdentifierExpression expr2 = + JsonArrayIdentifierExpression.of("attributes", "categories"); + + assertNotEquals(expr1, expr2, "Expressions with different paths should not be equal"); + } + + @Test + void testNotEqualsWithDifferentColumn() { + JsonArrayIdentifierExpression expr1 = JsonArrayIdentifierExpression.of("attributes", "tags"); + JsonArrayIdentifierExpression expr2 = JsonArrayIdentifierExpression.of("props", "tags"); + + assertNotEquals(expr1, expr2, "Expressions with different columns should not be equal"); + } + + @Test + void testNotEqualsWithJsonIdentifierExpression() { + JsonArrayIdentifierExpression arrayExpr = + JsonArrayIdentifierExpression.of("attributes", "tags"); + JsonIdentifierExpression scalarExpr = JsonIdentifierExpression.of("attributes", "tags"); + + // Even though they have the same column and path, they are different types + assertNotEquals( + arrayExpr, + scalarExpr, + "JsonArrayIdentifierExpression should not equal JsonIdentifierExpression"); + } + + @Test + void testInheritsFromJsonIdentifierExpression() { + JsonArrayIdentifierExpression expression = + JsonArrayIdentifierExpression.of("attributes", "tags"); + + // Verify it's an instance of parent class + assertEquals( + JsonIdentifierExpression.class, + expression.getClass().getSuperclass(), + "JsonArrayIdentifierExpression should extend JsonIdentifierExpression"); + } + + @Test + void testValidColumnNames() { + // Valid column names should not throw + assertDoesNotThrow(() -> JsonArrayIdentifierExpression.of("attributes", "tags")); + assertDoesNotThrow(() -> JsonArrayIdentifierExpression.of("_internal", "field")); + assertDoesNotThrow(() -> JsonArrayIdentifierExpression.of("customAttr", "array")); + assertDoesNotThrow(() -> JsonArrayIdentifierExpression.of("attr123", "field")); + } + + @Test + void testInvalidColumnNameWithSqlInjection() { + SecurityException exception = + assertThrows( + SecurityException.class, + () -> JsonArrayIdentifierExpression.of("attributes\"; DROP TABLE users; --", "tags")); + + assertTrue(exception.getMessage().contains("invalid")); + } + + @Test + void testInvalidColumnNameStartsWithNumber() { + SecurityException exception = + assertThrows( + SecurityException.class, () -> JsonArrayIdentifierExpression.of("123attr", "tags")); + + assertTrue(exception.getMessage().contains("Must start with a letter or underscore")); + } + + @Test + void testInvalidColumnNameWithSpace() { + SecurityException exception = + assertThrows( + SecurityException.class, () -> JsonArrayIdentifierExpression.of("my column", "tags")); + + assertTrue(exception.getMessage().contains("invalid")); + } + + @Test + void testValidJsonPathWithHyphen() { + assertDoesNotThrow(() -> JsonArrayIdentifierExpression.of("attributes", "user-tags")); + assertDoesNotThrow(() -> JsonArrayIdentifierExpression.of("attributes", "repo-urls")); + } + + @Test + void testValidJsonPathWithDot() { + assertDoesNotThrow(() -> JsonArrayIdentifierExpression.of("attributes", "field.name")); + assertDoesNotThrow( + () -> JsonArrayIdentifierExpression.of("attributes", "user.preferences.tags")); + } + + @Test + void testInvalidJsonPathWithSqlInjection() { + SecurityException exception = + assertThrows( + SecurityException.class, + () -> JsonArrayIdentifierExpression.of("attributes", "tags' OR '1'='1")); + + assertTrue(exception.getMessage().contains("invalid characters")); + } + + @Test + void testInvalidJsonPathWithSemicolon() { + SecurityException exception = + assertThrows( + SecurityException.class, + () -> JsonArrayIdentifierExpression.of("attributes", "field; DROP")); + + assertTrue(exception.getMessage().contains("invalid characters")); + } + + @Test + void testJsonPathMaxDepthExceeded() { + String[] deepPath = new String[101]; // Max is 100 + for (int i = 0; i < deepPath.length; i++) { + deepPath[i] = "level" + i; + } + + SecurityException exception = + assertThrows( + SecurityException.class, + () -> JsonArrayIdentifierExpression.of("attributes", deepPath)); + + assertTrue(exception.getMessage().contains("exceeds maximum depth")); + } + + @Test + void testJsonPathWithEmptyElement() { + SecurityException exception = + assertThrows( + SecurityException.class, + () -> JsonArrayIdentifierExpression.of("attributes", "nested", "", "field")); + + assertTrue(exception.getMessage().contains("null or empty")); + } + + @Test + void testMultipleInstancesWithSamePathAreEqual() { + JsonArrayIdentifierExpression expr1 = + JsonArrayIdentifierExpression.of("attributes", "certifications"); + JsonArrayIdentifierExpression expr2 = + JsonArrayIdentifierExpression.of("attributes", "certifications"); + JsonArrayIdentifierExpression expr3 = + JsonArrayIdentifierExpression.of("attributes", "certifications"); + + assertEquals(expr1, expr2); + assertEquals(expr2, expr3); + assertEquals(expr1, expr3); + assertEquals(expr1.hashCode(), expr2.hashCode()); + assertEquals(expr2.hashCode(), expr3.hashCode()); + } + + @Test + void testNestedArrayPath() { + JsonArrayIdentifierExpression expression = + JsonArrayIdentifierExpression.of("attributes", "nested", "deep", "arrays"); + + assertEquals("attributes", expression.getColumnName()); + assertEquals(List.of("nested", "deep", "arrays"), expression.getJsonPath()); + } +} From b01f3c502552cb9a8a94001d14597ef22d70297c Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Wed, 19 Nov 2025 12:00:16 +0530 Subject: [PATCH 03/14] Fixed failing test cases --- .../impl/JsonArrayIdentifierExpressionTest.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/expression/impl/JsonArrayIdentifierExpressionTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/expression/impl/JsonArrayIdentifierExpressionTest.java index 02a29b8a..250888a1 100644 --- a/document-store/src/test/java/org/hypertrace/core/documentstore/expression/impl/JsonArrayIdentifierExpressionTest.java +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/expression/impl/JsonArrayIdentifierExpressionTest.java @@ -55,7 +55,9 @@ void testOfWithEmptyPathElementsThrowsException() { assertThrows( IllegalArgumentException.class, () -> JsonArrayIdentifierExpression.of("attributes")); - assertEquals("JSON path cannot be null or empty for array field", exception.getMessage()); + assertEquals( + "JSON path cannot be null or empty. Use of(columnName, path...) instead.", + exception.getMessage()); } @Test @@ -65,7 +67,7 @@ void testOfWithNullListThrowsException() { IllegalArgumentException.class, () -> JsonArrayIdentifierExpression.of("attributes", (List) null)); - assertEquals("JSON path cannot be null or empty", exception.getMessage()); + assertEquals("JSON path cannot be null or empty for array field", exception.getMessage()); } @Test @@ -75,7 +77,7 @@ void testOfWithEmptyListThrowsException() { IllegalArgumentException.class, () -> JsonArrayIdentifierExpression.of("attributes", Collections.emptyList())); - assertEquals("JSON path cannot be null or empty", exception.getMessage()); + assertEquals("JSON path cannot be null or empty for array field", exception.getMessage()); } @Test From c07ee867d31e5e69e97da212422f32e78c4647a7 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Wed, 19 Nov 2025 12:03:28 +0530 Subject: [PATCH 04/14] Fixed failing test cases --- .../expression/impl/JsonIdentifierExpression.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/JsonIdentifierExpression.java b/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/JsonIdentifierExpression.java index a11ac38d..a9de5946 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/JsonIdentifierExpression.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/JsonIdentifierExpression.java @@ -20,6 +20,11 @@ public class JsonIdentifierExpression extends IdentifierExpression { String columnName; // e.g., "customAttr" (the top-level JSONB column) List jsonPath; // e.g., ["myAttribute", "nestedField"] + public static JsonIdentifierExpression of(final String columnName) { + throw new IllegalArgumentException( + "JSON path cannot be null or empty. Use of(columnName, path...) instead."); + } + public static JsonIdentifierExpression of(final String columnName, final String... pathElements) { if (pathElements == null || pathElements.length == 0) { // In this case, use IdentifierExpression From cbb08fd7bf1459cdf23ba37a82e470644df1c220 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Wed, 19 Nov 2025 16:02:38 +0530 Subject: [PATCH 05/14] WIP --- .../documentstore/DocStoreQueryV1Test.java | 201 +++++++++++++++++- .../query/pg_flat_collection_insert.json | 24 ++- .../PostgresExistsRelationalFilterParser.java | 30 ++- ...stgresNotExistsRelationalFilterParser.java | 30 ++- 4 files changed, 268 insertions(+), 17 deletions(-) diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java index 8a1c65ca..ba799b5e 100644 --- a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java +++ b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java @@ -62,17 +62,20 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import java.io.IOException; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Random; +import java.util.Set; import java.util.Spliterator; import java.util.UUID; import java.util.concurrent.Callable; @@ -85,10 +88,12 @@ import org.hypertrace.core.documentstore.commons.DocStoreConstants; import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.ArrayIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ArrayRelationalFilterExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.JsonArrayIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.JsonIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.KeyExpression; import org.hypertrace.core.documentstore.expression.impl.LogicalExpression; @@ -206,7 +211,8 @@ private static void createFlatCollectionSchema( + "\"sales\" JSONB," + "\"numbers\" INTEGER[]," + "\"scores\" DOUBLE PRECISION[]," - + "\"flags\" BOOLEAN[]" + + "\"flags\" BOOLEAN[]," + + "\"attributes\" JSONB" + ");", collectionName); @@ -4362,6 +4368,199 @@ void testJsonbNumericComparisonOperators(String dataStoreName) { } } + @Nested + class FlatCollectionArrayBehaviourTest { + + /** + * Test EXISTS filter on top-level arrays. It should only return arrays that are non-empty (have + * at-least one element) + */ + @ParameterizedTest + @ArgumentsSource(PostgresProvider.class) + void testExistsFilterOnArray(String dataStoreName) throws JsonProcessingException { + Datastore datastore = datastoreMap.get(dataStoreName); + Collection flatCollection = + datastore.getCollectionForType(FLAT_COLLECTION_NAME, DocumentType.FLAT); + + // Query using EXISTS on array field (simulating ArrayIdentifierExpression behavior) + // tags column has: NULL (row 9), empty '{}' (rows 10, 11, 13), non-empty (rows 1-8, 12, 14) + // Using EXISTS with 'null' parameter (matching entity-service pattern) + Query query = + Query.builder() + .addSelection(IdentifierExpression.of("item")) + .addSelection(IdentifierExpression.of("tags")) + .setFilter( + RelationalExpression.of( + ArrayIdentifierExpression.of("tags"), EXISTS, ConstantExpression.of("null"))) + .build(); + + Iterator results = flatCollection.find(query); + + int count = 0; + while (results.hasNext()) { + Document doc = results.next(); + JsonNode json = new ObjectMapper().readTree(doc.toJson()); + count++; + // Verify that ALL returned documents have non-empty arrays + JsonNode tags = json.get("tags"); + assertTrue( + tags.isArray() && !tags.isEmpty(), "tags should be non-empty array, but was: " + tags); + } + + // Should return only documents with non-empty arrays + // From test data: rows 1-8 have non-empty arrays (8 docs) + // Plus rows 12, 14 have non-empty arrays (2 docs) + // Total: 10 documents + assertEquals(10, count, "Should return a total of 10 docs that have non-empty tags"); + } + + /** + * Test NOT_EXISTS filter with ArrayIdentifierExpression. This validates that NOT_EXISTS on + * array fields returns both NULL and empty arrays, excluding only non-empty arrays. + */ + @ParameterizedTest + @ArgumentsSource(PostgresProvider.class) + void testNotExistsFilterOnArrays(String dataStoreName) throws JsonProcessingException { + Datastore datastore = datastoreMap.get(dataStoreName); + Collection flatCollection = + datastore.getCollectionForType(FLAT_COLLECTION_NAME, DocumentType.FLAT); + + // Query using NOT_EXISTS on array field (simulating ArrayIdentifierExpression behavior) + // Using NOT_EXISTS with 'null' parameter (matching entity-service pattern) + Query query = + Query.builder() + .addSelection(IdentifierExpression.of("item")) + .addSelection(IdentifierExpression.of("tags")) + .setFilter( + RelationalExpression.of( + ArrayIdentifierExpression.of("tags"), + NOT_EXISTS, + ConstantExpression.of("null"))) + .build(); + + Iterator results = flatCollection.find(query); + + int count = 0; + while (results.hasNext()) { + Document doc = results.next(); + JsonNode json = new ObjectMapper().readTree(doc.toJson()); + count++; + // Verify that ALL returned documents have NULL or empty arrays + JsonNode tags = json.get("tags"); + assertTrue( + tags == null || !tags.isArray() || tags.isEmpty(), + "tags should be NULL or empty array, but was: " + tags); + } + + // Should return documents with NULL or empty arrays + // From test data: row 9 (NULL), rows 10, 11, 13 (empty arrays) + // Total: 4 documents + assertEquals(4, count, "Should return at 4 documents with NULL or empty tags"); + } + + /** + * Test EXISTS filter on JSONB arrays. Should only return non-empty arrays (with at-least one + * element). + */ + @ParameterizedTest + @ArgumentsSource(PostgresProvider.class) + void testExistsFilterOnJsonArrays(String dataStoreName) throws JsonProcessingException { + Datastore datastore = datastoreMap.get(dataStoreName); + Collection flatCollection = + datastore.getCollectionForType(FLAT_COLLECTION_NAME, DocumentType.FLAT); + + // Query using EXISTS on JSONB array field + // attributes.certifications has: non-empty (row 1), empty (rows 2, 10, 11), NULL (rest) + Query query = + Query.builder() + .addSelection(IdentifierExpression.of("item")) + .addSelection(JsonIdentifierExpression.of("attributes", "certifications")) + .setFilter( + RelationalExpression.of( + JsonArrayIdentifierExpression.of("attributes", "certifications"), + EXISTS, + ConstantExpression.of("null"))) + .build(); + + Iterator results = flatCollection.find(query); + + int count = 0; + while (results.hasNext()) { + Document doc = results.next(); + JsonNode json = new ObjectMapper().readTree(doc.toJson()); + count++; + + // Verify that ALL returned documents have non-empty arrays in attributes.certifications + JsonNode attributes = json.get("attributes"); + assertTrue(attributes.isObject(), "attributes should be a JSON object"); + + JsonNode certifications = attributes.get("certifications"); + assertTrue( + certifications.isArray() && !certifications.isEmpty(), + "certifications should be non-empty array, but was: " + certifications); + } + + // Should return only row 1 which has non-empty certifications array + assertEquals(1, count, "Should return exactly 1 document with non-empty certifications"); + } + + /** + * Test NOT_EXISTS filter on JSONB arrays. This validates that NOT_EXISTS on array fields inside + * JSONB returns documents where the field is NULL, the parent object is NULL, or the array is + * empty. + */ + @ParameterizedTest + @ArgumentsSource(PostgresProvider.class) + void testNotExistsFilterOnJsonArrays(String dataStoreName) throws JsonProcessingException { + Datastore datastore = datastoreMap.get(dataStoreName); + Collection flatCollection = + datastore.getCollectionForType(FLAT_COLLECTION_NAME, DocumentType.FLAT); + + // Query using NOT_EXISTS on JSONB array field + // Test with attributes.colors field + Query query = + Query.builder() + .addSelection(IdentifierExpression.of("item")) + .addSelection(JsonIdentifierExpression.of("attributes", "colors")) + .setFilter( + RelationalExpression.of( + JsonArrayIdentifierExpression.of("attributes", "colors"), + NOT_EXISTS, + ConstantExpression.of("null"))) + .build(); + + Iterator results = flatCollection.find(query); + + int count = 0; + Set returnedItems = new HashSet<>(); + while (results.hasNext()) { + Document doc = results.next(); + JsonNode json = new ObjectMapper().readTree(doc.toJson()); + count++; + + String item = json.get("item").asText(); + returnedItems.add(item); + + // Verify that returned documents have NULL parent, missing field, or empty arrays + JsonNode attributes = json.get("attributes"); + if (attributes != null && attributes.isObject()) { + JsonNode colors = attributes.get("colors"); + assertTrue( + colors == null || !colors.isArray() || colors.isEmpty(), + "colors should be NULL or empty array for item: " + item + ", but was: " + colors); + } + // NULL attributes is also valid + } + + // Should include documents where attributes is NULL or attributes.colors is NULL/empty + // Row 11 (Pencil) and other rows with empty/NULL colors + assertTrue(count > 0, "Should return at least some documents"); + assertTrue( + returnedItems.contains("Pencil"), + "Should include Pencil (has empty colors array in attributes)"); + } + } + @Nested class BulkUpdateTest { diff --git a/document-store/src/integrationTest/resources/query/pg_flat_collection_insert.json b/document-store/src/integrationTest/resources/query/pg_flat_collection_insert.json index 050105fd..45e058da 100644 --- a/document-store/src/integrationTest/resources/query/pg_flat_collection_insert.json +++ b/document-store/src/integrationTest/resources/query/pg_flat_collection_insert.json @@ -1,14 +1,18 @@ { "statements": [ - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n1, 'Soap', 10, 2, '2014-03-01T08:00:00Z',\n'{\"hygiene\", \"personal-care\", \"premium\"}',\n'{\"Hygiene\", \"PersonalCare\"}',\n'{\"colors\": [\"Blue\", \"Green\"], \"brand\": \"Dettol\", \"size\": \"M\", \"seller\": {\"name\": \"Metro Chemicals Pvt. Ltd.\", \"address\": {\"city\": \"Mumbai\", \"pincode\": 400004}}}',\nNULL,\n'{1, 2, 3}',\n'{4.5, 9.2}',\n'{true, false}'\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n2, 'Mirror', 20, 1, '2014-03-01T09:00:00Z',\n'{\"home-decor\", \"reflective\", \"glass\"}',\n'{\"HomeDecor\"}',\nNULL,\nNULL,\n'{10, 20}',\nNULL,\nNULL\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n3, 'Shampoo', 5, 10, '2014-03-15T09:00:00Z',\n'{\"hair-care\", \"personal-care\", \"premium\", \"herbal\"}',\n'{\"HairCare\", \"PersonalCare\"}',\n'{\"colors\": [\"Black\"], \"brand\": \"Sunsilk\", \"size\": \"L\", \"seller\": {\"name\": \"Metro Chemicals Pvt. Ltd.\", \"address\": {\"city\": \"Mumbai\", \"pincode\": 400004}}}',\nNULL,\nNULL,\n'{3.14, 2.71}',\nNULL\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n4, 'Shampoo', 5, 20, '2014-04-04T11:21:39.736Z',\n'{\"hair-care\", \"budget\", \"bulk\"}',\n'{\"HairCare\"}',\nNULL,\nNULL,\nNULL,\nNULL,\n'{true, true}'\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n5, 'Soap', 20, 5, '2014-04-04T21:23:13.331Z',\n'{\"hygiene\", \"antibacterial\", \"family-pack\"}',\n'{\"Hygiene\"}',\n'{\"colors\": [\"Orange\", \"Blue\"], \"brand\": \"Lifebuoy\", \"size\": \"S\", \"seller\": {\"name\": \"Hans and Co.\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\nNULL,\nNULL,\nNULL\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n6, 'Comb', 7.5, 5, '2015-06-04T05:08:13Z',\n'{\"grooming\", \"plastic\", \"essential\"}',\n'{\"Grooming\"}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n7, 'Comb', 7.5, 10, '2015-09-10T08:43:00Z',\n'{\"grooming\", \"bulk\", \"wholesale\"}',\n'{\"Grooming\"}',\n'{\"colors\": [], \"seller\": {\"name\": \"Go Go Plastics\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\nNULL,\nNULL,\nNULL\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n8, 'Soap', 10, 5, '2016-02-06T20:20:13Z',\n'{\"hygiene\", \"budget\", \"basic\"}',\n'{\"Hygiene\"}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n9, 'Bottle', 15, 3, '2016-03-01T10:00:00Z',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n10, 'Cup', 8, 2, '2016-04-01T10:00:00Z',\n'{}',\n'{}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)" + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n1, 'Soap', 10, 2, '2014-03-01T08:00:00Z',\n'{\"hygiene\", \"personal-care\", \"premium\"}',\n'{\"Hygiene\", \"PersonalCare\"}',\n'{\"colors\": [\"Blue\", \"Green\"], \"brand\": \"Dettol\", \"size\": \"M\", \"seller\": {\"name\": \"Metro Chemicals Pvt. Ltd.\", \"address\": {\"city\": \"Mumbai\", \"pincode\": 400004}}}',\nNULL,\n'{1, 2, 3}',\n'{4.5, 9.2}',\n'{true, false}',\n'{\"name\": \"Premium Soap\", \"certifications\": [\"ISO9001\", \"FDA\"], \"origin\": \"India\"}'\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n2, 'Mirror', 20, 1, '2014-03-01T09:00:00Z',\n'{\"home-decor\", \"reflective\", \"glass\"}',\n'{\"HomeDecor\"}',\nNULL,\nNULL,\n'{10, 20}',\nNULL,\nNULL,\n'{\"category\": \"HomeDecor\", \"tags\": []}'\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n3, 'Shampoo', 5, 10, '2014-03-15T09:00:00Z',\n'{\"hair-care\", \"personal-care\", \"premium\", \"herbal\"}',\n'{\"HairCare\", \"PersonalCare\"}',\n'{\"colors\": [\"Black\"], \"brand\": \"Sunsilk\", \"size\": \"L\", \"seller\": {\"name\": \"Metro Chemicals Pvt. Ltd.\", \"address\": {\"city\": \"Mumbai\", \"pincode\": 400004}}}',\nNULL,\nNULL,\n'{3.14, 2.71}',\nNULL,\n'{\"type\": \"shampoo\", \"ingredients\": [\"herbal\", \"natural\"]}'\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n4, 'Shampoo', 5, 20, '2014-04-04T11:21:39.736Z',\n'{\"hair-care\", \"budget\", \"bulk\"}',\n'{\"HairCare\"}',\nNULL,\nNULL,\nNULL,\nNULL,\n'{true, true}',\n'{\"brand\": \"Generic\"}'\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n5, 'Soap', 20, 5, '2014-04-04T21:23:13.331Z',\n'{\"hygiene\", \"antibacterial\", \"family-pack\"}',\n'{\"Hygiene\"}',\n'{\"colors\": [\"Orange\", \"Blue\"], \"brand\": \"Lifebuoy\", \"size\": \"S\", \"seller\": {\"name\": \"Hans and Co.\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n6, 'Comb', 7.5, 5, '2015-06-04T05:08:13Z',\n'{\"grooming\", \"plastic\", \"essential\"}',\n'{\"Grooming\"}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n7, 'Comb', 7.5, 10, '2015-09-10T08:43:00Z',\n'{\"grooming\", \"bulk\", \"wholesale\"}',\n'{\"Grooming\"}',\n'{\"colors\": [], \"seller\": {\"name\": \"Go Go Plastics\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n8, 'Soap', 10, 5, '2016-02-06T20:20:13Z',\n'{\"hygiene\", \"budget\", \"basic\"}',\n'{\"Hygiene\"}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n9, 'Bottle', 15, 3, '2016-03-01T10:00:00Z',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n10, 'Cup', 8, 2, '2016-04-01T10:00:00Z',\n'{}',\n'{}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\n'{\"material\": \"ceramic\", \"features\": []}'\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n11, 'Pencil', 2, 50, '2016-05-01T10:00:00Z',\n'{}',\nNULL,\nNULL,\nNULL,\n'{}',\n'{}',\n'{}',\n'{\"grade\": \"2B\", \"colors\": []}'\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n12, 'Eraser', 1, 100, '2016-06-01T10:00:00Z',\n'{\"stationery\"}',\n'{}',\nNULL,\nNULL,\n'{}',\nNULL,\n'{}',\n'{\"type\": \"rubber\", \"useCases\": [\"drawing\", \"writing\"]}'\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n13, 'Notebook', 25, 10, '2016-07-01T10:00:00Z',\nNULL,\n'{}',\nNULL,\nNULL,\nNULL,\n'{}',\nNULL,\nNULL\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n14, 'Pen', 5, 20, '2016-08-01T10:00:00Z',\n'{\"stationery\", \"writing\"}',\n'{\"Stationery\"}',\nNULL,\nNULL,\n'{5, 10, 15}',\n'{}',\n'{false}',\n'{\"brand\": \"Parker\", \"colors\": [\"black\", \"blue\", \"red\"]}'\n)" ] } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java index 98d046c6..a29ed672 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java @@ -1,6 +1,8 @@ package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter; +import org.hypertrace.core.documentstore.expression.impl.ArrayIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; +import org.hypertrace.core.documentstore.expression.impl.JsonArrayIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; class PostgresExistsRelationalFilterParser implements PostgresRelationalFilterParser { @@ -9,8 +11,30 @@ public String parse( final RelationalExpression expression, final PostgresRelationalFilterContext context) { final String parsedLhs = expression.getLhs().accept(context.lhsParser()); final boolean parsedRhs = !ConstantExpression.of(false).equals(expression.getRhs()); - return parsedRhs - ? String.format("%s IS NOT NULL", parsedLhs) - : String.format("%s IS NULL", parsedLhs); + + // For array fields, EXISTS should check both NOT NULL and non-empty + boolean isArrayField = expression.getLhs() instanceof ArrayIdentifierExpression; + boolean isJsonArrayField = expression.getLhs() instanceof JsonArrayIdentifierExpression; + + if (isArrayField) { + // First-class PostgreSQL array columns (text[], int[], etc.) + return parsedRhs + ? String.format("(%s IS NOT NULL AND cardinality(%s) > 0)", parsedLhs, parsedLhs) + : String.format("(%s IS NULL OR cardinality(%s) = 0)", parsedLhs, parsedLhs); + } else if (isJsonArrayField) { + // Arrays inside JSONB columns + return parsedRhs + ? String.format( + "(%s IS NOT NULL AND jsonb_typeof(%s) = 'array' AND jsonb_array_length(%s) > 0)", + parsedLhs, parsedLhs, parsedLhs) + : String.format( + "(%s IS NULL OR jsonb_typeof(%s) != 'array' OR jsonb_array_length(%s) = 0)", + parsedLhs, parsedLhs, parsedLhs); + } else { + // Regular scalar fields + return parsedRhs + ? String.format("%s IS NOT NULL", parsedLhs) + : String.format("%s IS NULL", parsedLhs); + } } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java index d365d196..ab1ca02d 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java @@ -1,6 +1,8 @@ package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter; +import org.hypertrace.core.documentstore.expression.impl.ArrayIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; +import org.hypertrace.core.documentstore.expression.impl.JsonArrayIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; class PostgresNotExistsRelationalFilterParser implements PostgresRelationalFilterParser { @@ -9,8 +11,30 @@ public String parse( final RelationalExpression expression, final PostgresRelationalFilterContext context) { final String parsedLhs = expression.getLhs().accept(context.lhsParser()); final boolean parsedRhs = ConstantExpression.of(false).equals(expression.getRhs()); - return parsedRhs - ? String.format("%s IS NOT NULL", parsedLhs) - : String.format("%s IS NULL", parsedLhs); + + // For array fields, NOT_EXISTS should check NULL or empty + boolean isArrayField = expression.getLhs() instanceof ArrayIdentifierExpression; + boolean isJsonArrayField = expression.getLhs() instanceof JsonArrayIdentifierExpression; + + if (isArrayField) { + // First-class PostgreSQL array columns (text[], int[], etc.) + return parsedRhs + ? String.format("(%s IS NOT NULL AND cardinality(%s) > 0)", parsedLhs, parsedLhs) + : String.format("(%s IS NULL OR cardinality(%s) = 0)", parsedLhs, parsedLhs); + } else if (isJsonArrayField) { + // Arrays inside JSONB columns + return parsedRhs + ? String.format( + "(%s IS NOT NULL AND jsonb_typeof(%s) = 'array' AND jsonb_array_length(%s) > 0)", + parsedLhs, parsedLhs, parsedLhs) + : String.format( + "(%s IS NULL OR jsonb_typeof(%s) != 'array' OR jsonb_array_length(%s) = 0)", + parsedLhs, parsedLhs, parsedLhs); + } else { + // Regular scalar fields + return parsedRhs + ? String.format("%s IS NOT NULL", parsedLhs) + : String.format("%s IS NULL", parsedLhs); + } } } From ce28574cf0ab439025a5ec6089d06c3fbb4a02e8 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Wed, 19 Nov 2025 23:09:50 +0530 Subject: [PATCH 06/14] Fixed failing test cases --- .../documentstore/DocStoreQueryV1Test.java | 29 ++++++----- .../resources/query/collection_data.json | 42 +++++++++++++++ .../flat_integer_array_filter_response.json | 6 ++- .../flat_jsonb_brand_selection_response.json | 6 ++- .../flat_jsonb_city_selection_response.json | 15 +++--- .../flat_jsonb_colors_selection_response.json | 6 ++- ...at_jsonb_group_by_brand_test_response.json | 2 +- .../flat_unnest_mixed_case_response.json | 31 ++++++++--- ...t_not_preserving_empty_array_response.json | 2 +- ...nnest_preserving_empty_array_response.json | 2 +- .../query/flat_unnest_tags_response.json | 52 +++++++++++-------- .../PostgresExistsRelationalFilterParser.java | 9 ++++ ...stgresNotExistsRelationalFilterParser.java | 9 ++++ 13 files changed, 156 insertions(+), 55 deletions(-) diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java index ba799b5e..e42cfd68 100644 --- a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java +++ b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java @@ -68,6 +68,7 @@ import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -3219,7 +3220,7 @@ void testFlatPostgresCollectionFindAll(String dataStoreName) throws IOException iterator.close(); // Should have 8 documents from the INSERT statements - assertEquals(10, count); + assertEquals(14, count); } @ParameterizedTest @@ -3260,7 +3261,7 @@ void testFlatPostgresCollectionCount(String dataStoreName) { // Test count method - all documents long totalCount = flatCollection.count(Query.builder().build()); - assertEquals(10, totalCount); + assertEquals(14, totalCount); // Test count with filter - soap documents only Query soapQuery = @@ -3357,8 +3358,8 @@ void testFlatVsNestedCollectionConsistency(String dataStoreName) throws IOExcept Query countAllQuery = Query.builder().build(); long nestedCount = nestedCollection.count(countAllQuery); long flatCount = flatCollection.count(countAllQuery); - assertEquals(8, nestedCount, "Nested collection should have 8 documents"); - assertEquals(10, flatCount, "Flat collection should have 10 documents"); + assertEquals(14, nestedCount, "Nested collection should have 14 documents"); + assertEquals(14, flatCount, "Flat collection should have 14 documents"); // Test 2: Filter by top-level field - item Query itemFilterQuery = @@ -3385,16 +3386,16 @@ void testFlatVsNestedCollectionConsistency(String dataStoreName) throws IOExcept long nestedPriceCount = nestedCollection.count(priceFilterQuery); long flatPriceCount = flatCollection.count(priceFilterQuery); - assertEquals(2, nestedPriceCount, "Nested should have 2 docs with price > 10"); - assertEquals(3, flatPriceCount, "Flat should have 3 docs with price > 10"); + assertEquals(4, nestedPriceCount, "Nested should have 4 docs with price > 10"); + assertEquals(4, flatPriceCount, "Flat should have 4 docs with price > 10"); // Test 4: Compare actual document content for same filter CloseableIterator nestedIterator = nestedCollection.find(itemFilterQuery); CloseableIterator flatIterator = flatCollection.find(itemFilterQuery); // Collect documents from both collections - java.util.List nestedDocs = new java.util.ArrayList<>(); - java.util.List flatDocs = new java.util.ArrayList<>(); + List nestedDocs = new ArrayList<>(); + List flatDocs = new ArrayList<>(); while (nestedIterator.hasNext()) { nestedDocs.add(nestedIterator.next().toJson()); @@ -3503,7 +3504,7 @@ void testFlatPostgresCollectionUnnestTags(String dataStoreName) throws IOExcepti Iterator resultIterator = flatCollection.aggregate(unnestQuery); assertDocsAndSizeEqualWithoutOrder( - dataStoreName, resultIterator, "query/flat_unnest_tags_response.json", 17); + dataStoreName, resultIterator, "query/flat_unnest_tags_response.json", 19); } /** @@ -3771,7 +3772,7 @@ void testFlatPostgresCollectionUnnestMixedCaseField(String dataStoreName) throws Iterator resultIterator = flatCollection.aggregate(unnestQuery); // Expected categories: Hygiene(3), PersonalCare(2), HairCare(2), HomeDecor(1), Grooming(2) assertDocsAndSizeEqualWithoutOrder( - dataStoreName, resultIterator, "query/flat_unnest_mixed_case_response.json", 5); + dataStoreName, resultIterator, "query/flat_unnest_mixed_case_response.json", 6); } @ParameterizedTest @@ -3798,7 +3799,7 @@ void testFlatPostgresCollectionIntegerArrayFilter(String dataStoreName) throws I Iterator resultIterator = flatCollection.find(integerArrayQuery); assertDocsAndSizeEqualWithoutOrder( - dataStoreName, resultIterator, "query/flat_integer_array_filter_response.json", 1); + dataStoreName, resultIterator, "query/flat_integer_array_filter_response.json", 2); } @ParameterizedTest @@ -3873,7 +3874,7 @@ void testFlatCollectionNestedJsonSelections(String dataStoreName) throws IOExcep Iterator brandIterator = flatCollection.find(brandSelectionQuery); assertDocsAndSizeEqualWithoutOrder( - dataStoreName, brandIterator, "query/flat_jsonb_brand_selection_response.json", 10); + dataStoreName, brandIterator, "query/flat_jsonb_brand_selection_response.json", 14); // Test 2: Select deeply nested STRING field from JSONB column (props.seller.address.city) Query citySelectionQuery = @@ -3883,7 +3884,7 @@ void testFlatCollectionNestedJsonSelections(String dataStoreName) throws IOExcep Iterator cityIterator = flatCollection.find(citySelectionQuery); assertDocsAndSizeEqualWithoutOrder( - dataStoreName, cityIterator, "query/flat_jsonb_city_selection_response.json", 10); + dataStoreName, cityIterator, "query/flat_jsonb_city_selection_response.json", 14); // Test 3: Select STRING_ARRAY field from JSONB column (props.colors) Query colorsSelectionQuery = @@ -3891,7 +3892,7 @@ void testFlatCollectionNestedJsonSelections(String dataStoreName) throws IOExcep Iterator colorsIterator = flatCollection.find(colorsSelectionQuery); assertDocsAndSizeEqualWithoutOrder( - dataStoreName, colorsIterator, "query/flat_jsonb_colors_selection_response.json", 10); + dataStoreName, colorsIterator, "query/flat_jsonb_colors_selection_response.json", 14); } @ParameterizedTest diff --git a/document-store/src/integrationTest/resources/query/collection_data.json b/document-store/src/integrationTest/resources/query/collection_data.json index 3a389f38..2f7227fe 100644 --- a/document-store/src/integrationTest/resources/query/collection_data.json +++ b/document-store/src/integrationTest/resources/query/collection_data.json @@ -183,5 +183,47 @@ "price": 10, "quantity": 5, "date": "2016-02-06T20:20:13Z" + }, + { + "_id": 9, + "item": "Bottle", + "price": 15, + "quantity": 3, + "date": "2016-03-01T10:00:00Z" + }, + { + "_id": 10, + "item": "Cup", + "price": 8, + "quantity": 2, + "date": "2016-04-01T10:00:00Z" + }, + { + "_id": 11, + "item": "Pencil", + "price": 2, + "quantity": 50, + "date": "2016-05-01T10:00:00Z" + }, + { + "_id": 12, + "item": "Eraser", + "price": 1, + "quantity": 100, + "date": "2016-06-01T10:00:00Z" + }, + { + "_id": 13, + "item": "Notebook", + "price": 25, + "quantity": 10, + "date": "2016-07-01T10:00:00Z" + }, + { + "_id": 14, + "item": "Pen", + "price": 5, + "quantity": 20, + "date": "2016-08-01T10:00:00Z" } ] diff --git a/document-store/src/integrationTest/resources/query/flat_integer_array_filter_response.json b/document-store/src/integrationTest/resources/query/flat_integer_array_filter_response.json index 9eab7526..b43f3728 100644 --- a/document-store/src/integrationTest/resources/query/flat_integer_array_filter_response.json +++ b/document-store/src/integrationTest/resources/query/flat_integer_array_filter_response.json @@ -2,5 +2,9 @@ { "item": "Mirror", "price": 20 + }, + { + "item": "Pen", + "price": 5 } -] +] \ No newline at end of file diff --git a/document-store/src/integrationTest/resources/query/flat_jsonb_brand_selection_response.json b/document-store/src/integrationTest/resources/query/flat_jsonb_brand_selection_response.json index 112c14e2..b8e639a6 100644 --- a/document-store/src/integrationTest/resources/query/flat_jsonb_brand_selection_response.json +++ b/document-store/src/integrationTest/resources/query/flat_jsonb_brand_selection_response.json @@ -20,5 +20,9 @@ {}, {}, {}, + {}, + {}, + {}, + {}, {} -] +] \ No newline at end of file diff --git a/document-store/src/integrationTest/resources/query/flat_jsonb_city_selection_response.json b/document-store/src/integrationTest/resources/query/flat_jsonb_city_selection_response.json index 0ea7fdd6..8e32e22a 100644 --- a/document-store/src/integrationTest/resources/query/flat_jsonb_city_selection_response.json +++ b/document-store/src/integrationTest/resources/query/flat_jsonb_city_selection_response.json @@ -8,8 +8,7 @@ } } }, - { - }, + {}, { "props": { "seller": { @@ -19,8 +18,7 @@ } } }, - { - }, + {}, { "props": { "seller": { @@ -30,8 +28,7 @@ } } }, - { - }, + {}, { "props": { "seller": { @@ -43,5 +40,9 @@ }, {}, {}, + {}, + {}, + {}, + {}, {} -] +] \ No newline at end of file diff --git a/document-store/src/integrationTest/resources/query/flat_jsonb_colors_selection_response.json b/document-store/src/integrationTest/resources/query/flat_jsonb_colors_selection_response.json index c74918b3..740f57bf 100644 --- a/document-store/src/integrationTest/resources/query/flat_jsonb_colors_selection_response.json +++ b/document-store/src/integrationTest/resources/query/flat_jsonb_colors_selection_response.json @@ -7,6 +7,7 @@ ] } }, + {}, { "props": { "colors": [ @@ -14,6 +15,7 @@ ] } }, + {}, { "props": { "colors": [ @@ -22,6 +24,7 @@ ] } }, + {}, { "props": { "colors": [] @@ -32,5 +35,6 @@ {}, {}, {}, + {}, {} -] +] \ No newline at end of file diff --git a/document-store/src/integrationTest/resources/query/flat_jsonb_group_by_brand_test_response.json b/document-store/src/integrationTest/resources/query/flat_jsonb_group_by_brand_test_response.json index 0538f1e0..77965209 100644 --- a/document-store/src/integrationTest/resources/query/flat_jsonb_group_by_brand_test_response.json +++ b/document-store/src/integrationTest/resources/query/flat_jsonb_group_by_brand_test_response.json @@ -1,6 +1,6 @@ [ { - "count": 7 + "count": 11 }, { "props": { diff --git a/document-store/src/integrationTest/resources/query/flat_unnest_mixed_case_response.json b/document-store/src/integrationTest/resources/query/flat_unnest_mixed_case_response.json index 73d8e91d..85e24fcf 100644 --- a/document-store/src/integrationTest/resources/query/flat_unnest_mixed_case_response.json +++ b/document-store/src/integrationTest/resources/query/flat_unnest_mixed_case_response.json @@ -1,7 +1,26 @@ [ - {"categoryTags": "Hygiene", "count": 3}, - {"categoryTags": "PersonalCare", "count": 2}, - {"categoryTags": "HairCare", "count": 2}, - {"categoryTags": "HomeDecor", "count": 1}, - {"categoryTags": "Grooming", "count": 2} -] + { + "categoryTags": "PersonalCare", + "count": 2 + }, + { + "categoryTags": "HomeDecor", + "count": 1 + }, + { + "categoryTags": "Hygiene", + "count": 3 + }, + { + "categoryTags": "Stationery", + "count": 1 + }, + { + "categoryTags": "HairCare", + "count": 2 + }, + { + "categoryTags": "Grooming", + "count": 2 + } +] \ No newline at end of file diff --git a/document-store/src/integrationTest/resources/query/flat_unnest_not_preserving_empty_array_response.json b/document-store/src/integrationTest/resources/query/flat_unnest_not_preserving_empty_array_response.json index 832f51e4..9541182f 100644 --- a/document-store/src/integrationTest/resources/query/flat_unnest_not_preserving_empty_array_response.json +++ b/document-store/src/integrationTest/resources/query/flat_unnest_not_preserving_empty_array_response.json @@ -1,5 +1,5 @@ [ { - "count": 25 + "count": 28 } ] diff --git a/document-store/src/integrationTest/resources/query/flat_unnest_preserving_empty_array_response.json b/document-store/src/integrationTest/resources/query/flat_unnest_preserving_empty_array_response.json index 3a53e637..14e2cf07 100644 --- a/document-store/src/integrationTest/resources/query/flat_unnest_preserving_empty_array_response.json +++ b/document-store/src/integrationTest/resources/query/flat_unnest_preserving_empty_array_response.json @@ -1,5 +1,5 @@ [ { - "count": 27 + "count": 32 } ] diff --git a/document-store/src/integrationTest/resources/query/flat_unnest_tags_response.json b/document-store/src/integrationTest/resources/query/flat_unnest_tags_response.json index 5aac4063..2771ac60 100644 --- a/document-store/src/integrationTest/resources/query/flat_unnest_tags_response.json +++ b/document-store/src/integrationTest/resources/query/flat_unnest_tags_response.json @@ -1,18 +1,22 @@ [ { - "tags": "hygiene", - "count": 3 + "tags": "wholesale", + "count": 1 }, { - "tags": "personal-care", + "tags": "stationery", "count": 2 }, { - "tags": "premium", + "tags": "grooming", "count": 2 }, { - "tags": "home-decor", + "tags": "family-pack", + "count": 1 + }, + { + "tags": "glass", "count": 1 }, { @@ -20,51 +24,55 @@ "count": 1 }, { - "tags": "glass", + "tags": "home-decor", "count": 1 }, { - "tags": "hair-care", + "tags": "writing", + "count": 1 + }, + { + "tags": "premium", "count": 2 }, { - "tags": "herbal", + "tags": "plastic", "count": 1 }, { - "tags": "budget", - "count": 2 + "tags": "basic", + "count": 1 }, { - "tags": "bulk", + "tags": "personal-care", "count": 2 }, { - "tags": "antibacterial", + "tags": "essential", "count": 1 }, { - "tags": "family-pack", - "count": 1 + "tags": "hygiene", + "count": 3 }, { - "tags": "grooming", + "tags": "budget", "count": 2 }, { - "tags": "plastic", + "tags": "antibacterial", "count": 1 }, { - "tags": "essential", + "tags": "herbal", "count": 1 }, { - "tags": "wholesale", - "count": 1 + "tags": "bulk", + "count": 2 }, { - "tags": "basic", - "count": 1 + "tags": "hair-care", + "count": 2 } -] +] \ No newline at end of file diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java index a29ed672..c55fae39 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java @@ -6,10 +6,19 @@ import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; class PostgresExistsRelationalFilterParser implements PostgresRelationalFilterParser { + @Override public String parse( final RelationalExpression expression, final PostgresRelationalFilterContext context) { final String parsedLhs = expression.getLhs().accept(context.lhsParser()); + // If true: + // Regular fields -> IS NOT NULL + // Arrays -> IS NOT NULL and cardinality(...) > 0, + // JSONB arrays: IS NOT NULL and jsonb_array_length(...) > 0 + // If false: + // Regular fields -> IS NULL + // Arrays -> IS NULL OR cardinality(...) = 0, + // JSONB arrays: IS NULL OR jsonb_array_length(...) > 0 final boolean parsedRhs = !ConstantExpression.of(false).equals(expression.getRhs()); // For array fields, EXISTS should check both NOT NULL and non-empty diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java index ab1ca02d..ca35ef40 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java @@ -6,10 +6,19 @@ import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; class PostgresNotExistsRelationalFilterParser implements PostgresRelationalFilterParser { + @Override public String parse( final RelationalExpression expression, final PostgresRelationalFilterContext context) { final String parsedLhs = expression.getLhs().accept(context.lhsParser()); + // If true (RHS = false): + // Regular fields -> IS NOT NULL + // Arrays -> IS NOT NULL AND cardinality(...) > 0 + // JSONB arrays: IS NOT NULL AND jsonb_typeof(%s) = 'array' AND jsonb_array_length(...) > 0 + // If false (RHS = true or other): + // Regular fields -> IS NULL + // Arrays -> IS NULL OR cardinality(...) = 0 + // JSONB arrays: IS NULL OR jsonb_typeof(%s) != 'array' OR jsonb_array_length(...) = 0 final boolean parsedRhs = ConstantExpression.of(false).equals(expression.getRhs()); // For array fields, NOT_EXISTS should check NULL or empty From ca758837849b8201489a86b608952162186ddd24 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Mon, 24 Nov 2025 11:37:59 +0530 Subject: [PATCH 07/14] WIP --- .../documentstore/DocStoreQueryV1Test.java | 47 +++++----- .../query/pg_flat_collection_insert.json | 24 +++-- .../filter/PostgresArrayFieldDetector.java | 90 +++++++++++++++++++ .../PostgresExistsRelationalFilterParser.java | 53 +++++------ ...stgresNotExistsRelationalFilterParser.java | 53 +++++------ 5 files changed, 178 insertions(+), 89 deletions(-) create mode 100644 document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresArrayFieldDetector.java diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java index 042f9c24..f5dd7058 100644 --- a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java +++ b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java @@ -94,7 +94,6 @@ import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression; -import org.hypertrace.core.documentstore.expression.impl.JsonArrayIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.JsonFieldType; import org.hypertrace.core.documentstore.expression.impl.JsonIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.KeyExpression; @@ -212,8 +211,7 @@ private static void createFlatCollectionSchema( + "\"sales\" JSONB," + "\"numbers\" INTEGER[]," + "\"scores\" DOUBLE PRECISION[]," - + "\"flags\" BOOLEAN[]," - + "\"attributes\" JSONB" + + "\"flags\" BOOLEAN[]" + ");", collectionName); @@ -4690,14 +4688,14 @@ void testExistsFilterOnJsonArrays(String dataStoreName) throws JsonProcessingExc datastore.getCollectionForType(FLAT_COLLECTION_NAME, DocumentType.FLAT); // Query using EXISTS on JSONB array field - // attributes.certifications has: non-empty (row 1), empty (rows 2, 10, 11), NULL (rest) + // props.colors has: non-empty (rows 1, 3, 5), empty (row 7), NULL (rest) Query query = Query.builder() .addSelection(IdentifierExpression.of("item")) - .addSelection(JsonIdentifierExpression.of("attributes", "certifications")) + .addSelection(JsonIdentifierExpression.of("props", "colors")) .setFilter( RelationalExpression.of( - JsonArrayIdentifierExpression.of("attributes", "certifications"), + JsonIdentifierExpression.of("props", JsonFieldType.STRING_ARRAY, "colors"), EXISTS, ConstantExpression.of("null"))) .build(); @@ -4710,18 +4708,18 @@ void testExistsFilterOnJsonArrays(String dataStoreName) throws JsonProcessingExc JsonNode json = new ObjectMapper().readTree(doc.toJson()); count++; - // Verify that ALL returned documents have non-empty arrays in attributes.certifications - JsonNode attributes = json.get("attributes"); - assertTrue(attributes.isObject(), "attributes should be a JSON object"); + // Verify that ALL returned documents have non-empty arrays in props.colors + JsonNode props = json.get("props"); + assertTrue(props.isObject(), "props should be a JSON object"); - JsonNode certifications = attributes.get("certifications"); + JsonNode colors = props.get("colors"); assertTrue( - certifications.isArray() && !certifications.isEmpty(), - "certifications should be non-empty array, but was: " + certifications); + colors.isArray() && !colors.isEmpty(), + "colors should be non-empty array, but was: " + colors); } - // Should return only row 1 which has non-empty certifications array - assertEquals(1, count, "Should return exactly 1 document with non-empty certifications"); + // Should return rows 1, 3, 5 which have non-empty colors arrays + assertEquals(3, count, "Should return exactly 3 documents with non-empty colors"); } /** @@ -4737,14 +4735,14 @@ void testNotExistsFilterOnJsonArrays(String dataStoreName) throws JsonProcessing datastore.getCollectionForType(FLAT_COLLECTION_NAME, DocumentType.FLAT); // Query using NOT_EXISTS on JSONB array field - // Test with attributes.colors field + // Test with props.colors field Query query = Query.builder() .addSelection(IdentifierExpression.of("item")) - .addSelection(JsonIdentifierExpression.of("attributes", "colors")) + .addSelection(JsonIdentifierExpression.of("props", "colors")) .setFilter( RelationalExpression.of( - JsonArrayIdentifierExpression.of("attributes", "colors"), + JsonIdentifierExpression.of("props", JsonFieldType.STRING_ARRAY, "colors"), NOT_EXISTS, ConstantExpression.of("null"))) .build(); @@ -4762,22 +4760,21 @@ void testNotExistsFilterOnJsonArrays(String dataStoreName) throws JsonProcessing returnedItems.add(item); // Verify that returned documents have NULL parent, missing field, or empty arrays - JsonNode attributes = json.get("attributes"); - if (attributes != null && attributes.isObject()) { - JsonNode colors = attributes.get("colors"); + JsonNode props = json.get("props"); + if (props != null && props.isObject()) { + JsonNode colors = props.get("colors"); assertTrue( colors == null || !colors.isArray() || colors.isEmpty(), "colors should be NULL or empty array for item: " + item + ", but was: " + colors); } - // NULL attributes is also valid + // NULL props is also valid } - // Should include documents where attributes is NULL or attributes.colors is NULL/empty - // Row 11 (Pencil) and other rows with empty/NULL colors + // Should include documents where props is NULL or props.colors is NULL/empty + // Row 7 (Comb) has empty colors array, rows 2,4,6,8,9,10 have NULL props assertTrue(count > 0, "Should return at least some documents"); assertTrue( - returnedItems.contains("Pencil"), - "Should include Pencil (has empty colors array in attributes)"); + returnedItems.contains("Comb"), "Should include Comb (has empty colors array in props)"); } } diff --git a/document-store/src/integrationTest/resources/query/pg_flat_collection_insert.json b/document-store/src/integrationTest/resources/query/pg_flat_collection_insert.json index 45e058da..050105fd 100644 --- a/document-store/src/integrationTest/resources/query/pg_flat_collection_insert.json +++ b/document-store/src/integrationTest/resources/query/pg_flat_collection_insert.json @@ -1,18 +1,14 @@ { "statements": [ - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n1, 'Soap', 10, 2, '2014-03-01T08:00:00Z',\n'{\"hygiene\", \"personal-care\", \"premium\"}',\n'{\"Hygiene\", \"PersonalCare\"}',\n'{\"colors\": [\"Blue\", \"Green\"], \"brand\": \"Dettol\", \"size\": \"M\", \"seller\": {\"name\": \"Metro Chemicals Pvt. Ltd.\", \"address\": {\"city\": \"Mumbai\", \"pincode\": 400004}}}',\nNULL,\n'{1, 2, 3}',\n'{4.5, 9.2}',\n'{true, false}',\n'{\"name\": \"Premium Soap\", \"certifications\": [\"ISO9001\", \"FDA\"], \"origin\": \"India\"}'\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n2, 'Mirror', 20, 1, '2014-03-01T09:00:00Z',\n'{\"home-decor\", \"reflective\", \"glass\"}',\n'{\"HomeDecor\"}',\nNULL,\nNULL,\n'{10, 20}',\nNULL,\nNULL,\n'{\"category\": \"HomeDecor\", \"tags\": []}'\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n3, 'Shampoo', 5, 10, '2014-03-15T09:00:00Z',\n'{\"hair-care\", \"personal-care\", \"premium\", \"herbal\"}',\n'{\"HairCare\", \"PersonalCare\"}',\n'{\"colors\": [\"Black\"], \"brand\": \"Sunsilk\", \"size\": \"L\", \"seller\": {\"name\": \"Metro Chemicals Pvt. Ltd.\", \"address\": {\"city\": \"Mumbai\", \"pincode\": 400004}}}',\nNULL,\nNULL,\n'{3.14, 2.71}',\nNULL,\n'{\"type\": \"shampoo\", \"ingredients\": [\"herbal\", \"natural\"]}'\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n4, 'Shampoo', 5, 20, '2014-04-04T11:21:39.736Z',\n'{\"hair-care\", \"budget\", \"bulk\"}',\n'{\"HairCare\"}',\nNULL,\nNULL,\nNULL,\nNULL,\n'{true, true}',\n'{\"brand\": \"Generic\"}'\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n5, 'Soap', 20, 5, '2014-04-04T21:23:13.331Z',\n'{\"hygiene\", \"antibacterial\", \"family-pack\"}',\n'{\"Hygiene\"}',\n'{\"colors\": [\"Orange\", \"Blue\"], \"brand\": \"Lifebuoy\", \"size\": \"S\", \"seller\": {\"name\": \"Hans and Co.\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n6, 'Comb', 7.5, 5, '2015-06-04T05:08:13Z',\n'{\"grooming\", \"plastic\", \"essential\"}',\n'{\"Grooming\"}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n7, 'Comb', 7.5, 10, '2015-09-10T08:43:00Z',\n'{\"grooming\", \"bulk\", \"wholesale\"}',\n'{\"Grooming\"}',\n'{\"colors\": [], \"seller\": {\"name\": \"Go Go Plastics\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n8, 'Soap', 10, 5, '2016-02-06T20:20:13Z',\n'{\"hygiene\", \"budget\", \"basic\"}',\n'{\"Hygiene\"}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n9, 'Bottle', 15, 3, '2016-03-01T10:00:00Z',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n10, 'Cup', 8, 2, '2016-04-01T10:00:00Z',\n'{}',\n'{}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\n'{\"material\": \"ceramic\", \"features\": []}'\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n11, 'Pencil', 2, 50, '2016-05-01T10:00:00Z',\n'{}',\nNULL,\nNULL,\nNULL,\n'{}',\n'{}',\n'{}',\n'{\"grade\": \"2B\", \"colors\": []}'\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n12, 'Eraser', 1, 100, '2016-06-01T10:00:00Z',\n'{\"stationery\"}',\n'{}',\nNULL,\nNULL,\n'{}',\nNULL,\n'{}',\n'{\"type\": \"rubber\", \"useCases\": [\"drawing\", \"writing\"]}'\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n13, 'Notebook', 25, 10, '2016-07-01T10:00:00Z',\nNULL,\n'{}',\nNULL,\nNULL,\nNULL,\n'{}',\nNULL,\nNULL\n)", - "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\", \"attributes\"\n) VALUES (\n14, 'Pen', 5, 20, '2016-08-01T10:00:00Z',\n'{\"stationery\", \"writing\"}',\n'{\"Stationery\"}',\nNULL,\nNULL,\n'{5, 10, 15}',\n'{}',\n'{false}',\n'{\"brand\": \"Parker\", \"colors\": [\"black\", \"blue\", \"red\"]}'\n)" + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n1, 'Soap', 10, 2, '2014-03-01T08:00:00Z',\n'{\"hygiene\", \"personal-care\", \"premium\"}',\n'{\"Hygiene\", \"PersonalCare\"}',\n'{\"colors\": [\"Blue\", \"Green\"], \"brand\": \"Dettol\", \"size\": \"M\", \"seller\": {\"name\": \"Metro Chemicals Pvt. Ltd.\", \"address\": {\"city\": \"Mumbai\", \"pincode\": 400004}}}',\nNULL,\n'{1, 2, 3}',\n'{4.5, 9.2}',\n'{true, false}'\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n2, 'Mirror', 20, 1, '2014-03-01T09:00:00Z',\n'{\"home-decor\", \"reflective\", \"glass\"}',\n'{\"HomeDecor\"}',\nNULL,\nNULL,\n'{10, 20}',\nNULL,\nNULL\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n3, 'Shampoo', 5, 10, '2014-03-15T09:00:00Z',\n'{\"hair-care\", \"personal-care\", \"premium\", \"herbal\"}',\n'{\"HairCare\", \"PersonalCare\"}',\n'{\"colors\": [\"Black\"], \"brand\": \"Sunsilk\", \"size\": \"L\", \"seller\": {\"name\": \"Metro Chemicals Pvt. Ltd.\", \"address\": {\"city\": \"Mumbai\", \"pincode\": 400004}}}',\nNULL,\nNULL,\n'{3.14, 2.71}',\nNULL\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n4, 'Shampoo', 5, 20, '2014-04-04T11:21:39.736Z',\n'{\"hair-care\", \"budget\", \"bulk\"}',\n'{\"HairCare\"}',\nNULL,\nNULL,\nNULL,\nNULL,\n'{true, true}'\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n5, 'Soap', 20, 5, '2014-04-04T21:23:13.331Z',\n'{\"hygiene\", \"antibacterial\", \"family-pack\"}',\n'{\"Hygiene\"}',\n'{\"colors\": [\"Orange\", \"Blue\"], \"brand\": \"Lifebuoy\", \"size\": \"S\", \"seller\": {\"name\": \"Hans and Co.\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\nNULL,\nNULL,\nNULL\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n6, 'Comb', 7.5, 5, '2015-06-04T05:08:13Z',\n'{\"grooming\", \"plastic\", \"essential\"}',\n'{\"Grooming\"}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n7, 'Comb', 7.5, 10, '2015-09-10T08:43:00Z',\n'{\"grooming\", \"bulk\", \"wholesale\"}',\n'{\"Grooming\"}',\n'{\"colors\": [], \"seller\": {\"name\": \"Go Go Plastics\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\nNULL,\nNULL,\nNULL\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n8, 'Soap', 10, 5, '2016-02-06T20:20:13Z',\n'{\"hygiene\", \"budget\", \"basic\"}',\n'{\"Hygiene\"}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n9, 'Bottle', 15, 3, '2016-03-01T10:00:00Z',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)", + "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n10, 'Cup', 8, 2, '2016-04-01T10:00:00Z',\n'{}',\n'{}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)" ] } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresArrayFieldDetector.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresArrayFieldDetector.java new file mode 100644 index 00000000..46c2f820 --- /dev/null +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresArrayFieldDetector.java @@ -0,0 +1,90 @@ +package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter; + +import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.ArrayIdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; +import org.hypertrace.core.documentstore.expression.impl.ConstantExpression.DocumentConstantExpression; +import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; +import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.JsonFieldType; +import org.hypertrace.core.documentstore.expression.impl.JsonIdentifierExpression; +import org.hypertrace.core.documentstore.parser.SelectTypeExpressionVisitor; + +/** + * Visitor to detect the category of a field expression for array-aware SQL generation. + * + *

Categorizes fields into three types: + * + *

    + *
  • SCALAR: Regular fields and JSON primitives (strings, numbers, booleans, objects) + *
  • POSTGRES_ARRAY: Native PostgreSQL arrays (text[], integer[], boolean[], etc.) + *
  • JSONB_ARRAY: Arrays inside JSONB columns with JsonFieldType annotation + *
+ * + *

This categorization is used by EXISTS/NOT_EXISTS parsers to generate appropriate SQL: + * + *

    + *
  • SCALAR: {@code IS NOT NULL / IS NULL} + *
  • POSTGRES_ARRAY: {@code IS NOT NULL AND cardinality(...) > 0} + *
  • JSONB_ARRAY: {@code IS NOT NULL AND jsonb_array_length(...) > 0} + *
+ */ +class PostgresArrayFieldDetector implements SelectTypeExpressionVisitor { + + /** Field category for determining appropriate SQL generation strategy */ + enum FieldCategory { + SCALAR, // Regular fields and JSON primitives + ARRAY, // Native PostgreSQL arrays (text[], int[], etc.) + JSONB_ARRAY // Arrays inside JSONB columns + } + + @Override + public FieldCategory visit(ArrayIdentifierExpression expression) { + return FieldCategory.ARRAY; + } + + @Override + public FieldCategory visit(JsonIdentifierExpression expression) { + return expression + .getFieldType() + .filter( + type -> + type == JsonFieldType.STRING_ARRAY + || type == JsonFieldType.NUMBER_ARRAY + || type == JsonFieldType.BOOLEAN_ARRAY + || type == JsonFieldType.OBJECT_ARRAY) + .map(type -> FieldCategory.JSONB_ARRAY) + .orElse(FieldCategory.SCALAR); + } + + @Override + public FieldCategory visit(IdentifierExpression expression) { + return FieldCategory.SCALAR; + } + + @Override + public FieldCategory visit(AggregateExpression expression) { + return FieldCategory.SCALAR; + } + + @Override + public FieldCategory visit(ConstantExpression expression) { + return FieldCategory.SCALAR; + } + + @Override + public FieldCategory visit(DocumentConstantExpression expression) { + return FieldCategory.SCALAR; + } + + @Override + public FieldCategory visit(FunctionExpression expression) { + return FieldCategory.SCALAR; + } + + @Override + public FieldCategory visit(AliasedIdentifierExpression expression) { + return FieldCategory.SCALAR; + } +} diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java index c55fae39..3ab6c03d 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java @@ -1,12 +1,13 @@ package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter; -import org.hypertrace.core.documentstore.expression.impl.ArrayIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; -import org.hypertrace.core.documentstore.expression.impl.JsonArrayIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; +import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresArrayFieldDetector.FieldCategory; class PostgresExistsRelationalFilterParser implements PostgresRelationalFilterParser { + private static final PostgresArrayFieldDetector ARRAY_DETECTOR = new PostgresArrayFieldDetector(); + @Override public String parse( final RelationalExpression expression, final PostgresRelationalFilterContext context) { @@ -21,29 +22,31 @@ public String parse( // JSONB arrays: IS NULL OR jsonb_array_length(...) > 0 final boolean parsedRhs = !ConstantExpression.of(false).equals(expression.getRhs()); - // For array fields, EXISTS should check both NOT NULL and non-empty - boolean isArrayField = expression.getLhs() instanceof ArrayIdentifierExpression; - boolean isJsonArrayField = expression.getLhs() instanceof JsonArrayIdentifierExpression; - - if (isArrayField) { - // First-class PostgreSQL array columns (text[], int[], etc.) - return parsedRhs - ? String.format("(%s IS NOT NULL AND cardinality(%s) > 0)", parsedLhs, parsedLhs) - : String.format("(%s IS NULL OR cardinality(%s) = 0)", parsedLhs, parsedLhs); - } else if (isJsonArrayField) { - // Arrays inside JSONB columns - return parsedRhs - ? String.format( - "(%s IS NOT NULL AND jsonb_typeof(%s) = 'array' AND jsonb_array_length(%s) > 0)", - parsedLhs, parsedLhs, parsedLhs) - : String.format( - "(%s IS NULL OR jsonb_typeof(%s) != 'array' OR jsonb_array_length(%s) = 0)", - parsedLhs, parsedLhs, parsedLhs); - } else { - // Regular scalar fields - return parsedRhs - ? String.format("%s IS NOT NULL", parsedLhs) - : String.format("%s IS NULL", parsedLhs); + FieldCategory category = expression.getLhs().accept(ARRAY_DETECTOR); + + switch (category) { + case ARRAY: + // First-class PostgreSQL array columns (text[], int[], etc.) + return parsedRhs + ? String.format("(%s IS NOT NULL AND cardinality(%s) > 0)", parsedLhs, parsedLhs) + : String.format("(%s IS NULL OR cardinality(%s) = 0)", parsedLhs, parsedLhs); + + case JSONB_ARRAY: + // Arrays inside JSONB columns + return parsedRhs + ? String.format( + "(%s IS NOT NULL AND jsonb_typeof(%s) = 'array' AND jsonb_array_length(%s) > 0)", + parsedLhs, parsedLhs, parsedLhs) + : String.format( + "(%s IS NULL OR jsonb_typeof(%s) != 'array' OR jsonb_array_length(%s) = 0)", + parsedLhs, parsedLhs, parsedLhs); + + case SCALAR: + default: + // Regular scalar fields + return parsedRhs + ? String.format("%s IS NOT NULL", parsedLhs) + : String.format("%s IS NULL", parsedLhs); } } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java index ca35ef40..caa919c4 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java @@ -1,12 +1,13 @@ package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter; -import org.hypertrace.core.documentstore.expression.impl.ArrayIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; -import org.hypertrace.core.documentstore.expression.impl.JsonArrayIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; +import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresArrayFieldDetector.FieldCategory; class PostgresNotExistsRelationalFilterParser implements PostgresRelationalFilterParser { + private static final PostgresArrayFieldDetector ARRAY_DETECTOR = new PostgresArrayFieldDetector(); + @Override public String parse( final RelationalExpression expression, final PostgresRelationalFilterContext context) { @@ -21,29 +22,31 @@ public String parse( // JSONB arrays: IS NULL OR jsonb_typeof(%s) != 'array' OR jsonb_array_length(...) = 0 final boolean parsedRhs = ConstantExpression.of(false).equals(expression.getRhs()); - // For array fields, NOT_EXISTS should check NULL or empty - boolean isArrayField = expression.getLhs() instanceof ArrayIdentifierExpression; - boolean isJsonArrayField = expression.getLhs() instanceof JsonArrayIdentifierExpression; - - if (isArrayField) { - // First-class PostgreSQL array columns (text[], int[], etc.) - return parsedRhs - ? String.format("(%s IS NOT NULL AND cardinality(%s) > 0)", parsedLhs, parsedLhs) - : String.format("(%s IS NULL OR cardinality(%s) = 0)", parsedLhs, parsedLhs); - } else if (isJsonArrayField) { - // Arrays inside JSONB columns - return parsedRhs - ? String.format( - "(%s IS NOT NULL AND jsonb_typeof(%s) = 'array' AND jsonb_array_length(%s) > 0)", - parsedLhs, parsedLhs, parsedLhs) - : String.format( - "(%s IS NULL OR jsonb_typeof(%s) != 'array' OR jsonb_array_length(%s) = 0)", - parsedLhs, parsedLhs, parsedLhs); - } else { - // Regular scalar fields - return parsedRhs - ? String.format("%s IS NOT NULL", parsedLhs) - : String.format("%s IS NULL", parsedLhs); + FieldCategory category = expression.getLhs().accept(ARRAY_DETECTOR); + + switch (category) { + case ARRAY: + // First-class PostgreSQL array columns (text[], int[], etc.) + return parsedRhs + ? String.format("(%s IS NOT NULL AND cardinality(%s) > 0)", parsedLhs, parsedLhs) + : String.format("(%s IS NULL OR cardinality(%s) = 0)", parsedLhs, parsedLhs); + + case JSONB_ARRAY: + // Arrays inside JSONB columns + return parsedRhs + ? String.format( + "(%s IS NOT NULL AND jsonb_typeof(%s) = 'array' AND jsonb_array_length(%s) > 0)", + parsedLhs, parsedLhs, parsedLhs) + : String.format( + "(%s IS NULL OR jsonb_typeof(%s) != 'array' OR jsonb_array_length(%s) = 0)", + parsedLhs, parsedLhs, parsedLhs); + + case SCALAR: + default: + // Regular scalar fields + return parsedRhs + ? String.format("%s IS NOT NULL", parsedLhs) + : String.format("%s IS NULL", parsedLhs); } } } From 8826b1f01fcd7106a6c8ec67b5aa089e7adbd81d Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Mon, 24 Nov 2025 11:38:28 +0530 Subject: [PATCH 08/14] Revert "Fixed failing test cases" This reverts commit ce28574cf0ab439025a5ec6089d06c3fbb4a02e8. --- .../documentstore/DocStoreQueryV1Test.java | 17 +++--- .../resources/query/collection_data.json | 42 --------------- .../flat_integer_array_filter_response.json | 6 +-- .../flat_jsonb_brand_selection_response.json | 6 +-- .../flat_jsonb_city_selection_response.json | 15 +++--- .../flat_jsonb_colors_selection_response.json | 6 +-- ...at_jsonb_group_by_brand_test_response.json | 2 +- .../flat_unnest_mixed_case_response.json | 31 +++-------- ...t_not_preserving_empty_array_response.json | 2 +- ...nnest_preserving_empty_array_response.json | 2 +- .../query/flat_unnest_tags_response.json | 52 ++++++++----------- 11 files changed, 49 insertions(+), 132 deletions(-) diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java index f5dd7058..691bbdf5 100644 --- a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java +++ b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java @@ -68,7 +68,6 @@ import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import java.io.IOException; -import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -3277,7 +3276,7 @@ void testFlatPostgresCollectionCount(String dataStoreName) { // Test count method - all documents long totalCount = flatCollection.count(Query.builder().build()); - assertEquals(14, totalCount); + assertEquals(10, totalCount); // Test count with filter - soap documents only Query soapQuery = @@ -3570,8 +3569,8 @@ void testFlatVsNestedCollectionConsistency(String dataStoreName) throws IOExcept Query countAllQuery = Query.builder().build(); long nestedCount = nestedCollection.count(countAllQuery); long flatCount = flatCollection.count(countAllQuery); - assertEquals(14, nestedCount, "Nested collection should have 14 documents"); - assertEquals(14, flatCount, "Flat collection should have 14 documents"); + assertEquals(8, nestedCount, "Nested collection should have 8 documents"); + assertEquals(10, flatCount, "Flat collection should have 10 documents"); // Test 2: Filter by top-level field - item Query itemFilterQuery = @@ -3598,16 +3597,16 @@ void testFlatVsNestedCollectionConsistency(String dataStoreName) throws IOExcept long nestedPriceCount = nestedCollection.count(priceFilterQuery); long flatPriceCount = flatCollection.count(priceFilterQuery); - assertEquals(4, nestedPriceCount, "Nested should have 4 docs with price > 10"); - assertEquals(4, flatPriceCount, "Flat should have 4 docs with price > 10"); + assertEquals(2, nestedPriceCount, "Nested should have 2 docs with price > 10"); + assertEquals(3, flatPriceCount, "Flat should have 3 docs with price > 10"); // Test 4: Compare actual document content for same filter CloseableIterator nestedIterator = nestedCollection.find(itemFilterQuery); CloseableIterator flatIterator = flatCollection.find(itemFilterQuery); // Collect documents from both collections - List nestedDocs = new ArrayList<>(); - List flatDocs = new ArrayList<>(); + java.util.List nestedDocs = new java.util.ArrayList<>(); + java.util.List flatDocs = new java.util.ArrayList<>(); while (nestedIterator.hasNext()) { nestedDocs.add(nestedIterator.next().toJson()); @@ -3716,7 +3715,7 @@ void testFlatPostgresCollectionUnnestTags(String dataStoreName) throws IOExcepti Iterator resultIterator = flatCollection.aggregate(unnestQuery); assertDocsAndSizeEqualWithoutOrder( - dataStoreName, resultIterator, "query/flat_unnest_tags_response.json", 19); + dataStoreName, resultIterator, "query/flat_unnest_tags_response.json", 17); } /** diff --git a/document-store/src/integrationTest/resources/query/collection_data.json b/document-store/src/integrationTest/resources/query/collection_data.json index 2f7227fe..3a389f38 100644 --- a/document-store/src/integrationTest/resources/query/collection_data.json +++ b/document-store/src/integrationTest/resources/query/collection_data.json @@ -183,47 +183,5 @@ "price": 10, "quantity": 5, "date": "2016-02-06T20:20:13Z" - }, - { - "_id": 9, - "item": "Bottle", - "price": 15, - "quantity": 3, - "date": "2016-03-01T10:00:00Z" - }, - { - "_id": 10, - "item": "Cup", - "price": 8, - "quantity": 2, - "date": "2016-04-01T10:00:00Z" - }, - { - "_id": 11, - "item": "Pencil", - "price": 2, - "quantity": 50, - "date": "2016-05-01T10:00:00Z" - }, - { - "_id": 12, - "item": "Eraser", - "price": 1, - "quantity": 100, - "date": "2016-06-01T10:00:00Z" - }, - { - "_id": 13, - "item": "Notebook", - "price": 25, - "quantity": 10, - "date": "2016-07-01T10:00:00Z" - }, - { - "_id": 14, - "item": "Pen", - "price": 5, - "quantity": 20, - "date": "2016-08-01T10:00:00Z" } ] diff --git a/document-store/src/integrationTest/resources/query/flat_integer_array_filter_response.json b/document-store/src/integrationTest/resources/query/flat_integer_array_filter_response.json index b43f3728..9eab7526 100644 --- a/document-store/src/integrationTest/resources/query/flat_integer_array_filter_response.json +++ b/document-store/src/integrationTest/resources/query/flat_integer_array_filter_response.json @@ -2,9 +2,5 @@ { "item": "Mirror", "price": 20 - }, - { - "item": "Pen", - "price": 5 } -] \ No newline at end of file +] diff --git a/document-store/src/integrationTest/resources/query/flat_jsonb_brand_selection_response.json b/document-store/src/integrationTest/resources/query/flat_jsonb_brand_selection_response.json index b8e639a6..112c14e2 100644 --- a/document-store/src/integrationTest/resources/query/flat_jsonb_brand_selection_response.json +++ b/document-store/src/integrationTest/resources/query/flat_jsonb_brand_selection_response.json @@ -20,9 +20,5 @@ {}, {}, {}, - {}, - {}, - {}, - {}, {} -] \ No newline at end of file +] diff --git a/document-store/src/integrationTest/resources/query/flat_jsonb_city_selection_response.json b/document-store/src/integrationTest/resources/query/flat_jsonb_city_selection_response.json index 8e32e22a..0ea7fdd6 100644 --- a/document-store/src/integrationTest/resources/query/flat_jsonb_city_selection_response.json +++ b/document-store/src/integrationTest/resources/query/flat_jsonb_city_selection_response.json @@ -8,7 +8,8 @@ } } }, - {}, + { + }, { "props": { "seller": { @@ -18,7 +19,8 @@ } } }, - {}, + { + }, { "props": { "seller": { @@ -28,7 +30,8 @@ } } }, - {}, + { + }, { "props": { "seller": { @@ -40,9 +43,5 @@ }, {}, {}, - {}, - {}, - {}, - {}, {} -] \ No newline at end of file +] diff --git a/document-store/src/integrationTest/resources/query/flat_jsonb_colors_selection_response.json b/document-store/src/integrationTest/resources/query/flat_jsonb_colors_selection_response.json index 740f57bf..c74918b3 100644 --- a/document-store/src/integrationTest/resources/query/flat_jsonb_colors_selection_response.json +++ b/document-store/src/integrationTest/resources/query/flat_jsonb_colors_selection_response.json @@ -7,7 +7,6 @@ ] } }, - {}, { "props": { "colors": [ @@ -15,7 +14,6 @@ ] } }, - {}, { "props": { "colors": [ @@ -24,7 +22,6 @@ ] } }, - {}, { "props": { "colors": [] @@ -35,6 +32,5 @@ {}, {}, {}, - {}, {} -] \ No newline at end of file +] diff --git a/document-store/src/integrationTest/resources/query/flat_jsonb_group_by_brand_test_response.json b/document-store/src/integrationTest/resources/query/flat_jsonb_group_by_brand_test_response.json index 77965209..0538f1e0 100644 --- a/document-store/src/integrationTest/resources/query/flat_jsonb_group_by_brand_test_response.json +++ b/document-store/src/integrationTest/resources/query/flat_jsonb_group_by_brand_test_response.json @@ -1,6 +1,6 @@ [ { - "count": 11 + "count": 7 }, { "props": { diff --git a/document-store/src/integrationTest/resources/query/flat_unnest_mixed_case_response.json b/document-store/src/integrationTest/resources/query/flat_unnest_mixed_case_response.json index 85e24fcf..73d8e91d 100644 --- a/document-store/src/integrationTest/resources/query/flat_unnest_mixed_case_response.json +++ b/document-store/src/integrationTest/resources/query/flat_unnest_mixed_case_response.json @@ -1,26 +1,7 @@ [ - { - "categoryTags": "PersonalCare", - "count": 2 - }, - { - "categoryTags": "HomeDecor", - "count": 1 - }, - { - "categoryTags": "Hygiene", - "count": 3 - }, - { - "categoryTags": "Stationery", - "count": 1 - }, - { - "categoryTags": "HairCare", - "count": 2 - }, - { - "categoryTags": "Grooming", - "count": 2 - } -] \ No newline at end of file + {"categoryTags": "Hygiene", "count": 3}, + {"categoryTags": "PersonalCare", "count": 2}, + {"categoryTags": "HairCare", "count": 2}, + {"categoryTags": "HomeDecor", "count": 1}, + {"categoryTags": "Grooming", "count": 2} +] diff --git a/document-store/src/integrationTest/resources/query/flat_unnest_not_preserving_empty_array_response.json b/document-store/src/integrationTest/resources/query/flat_unnest_not_preserving_empty_array_response.json index 9541182f..832f51e4 100644 --- a/document-store/src/integrationTest/resources/query/flat_unnest_not_preserving_empty_array_response.json +++ b/document-store/src/integrationTest/resources/query/flat_unnest_not_preserving_empty_array_response.json @@ -1,5 +1,5 @@ [ { - "count": 28 + "count": 25 } ] diff --git a/document-store/src/integrationTest/resources/query/flat_unnest_preserving_empty_array_response.json b/document-store/src/integrationTest/resources/query/flat_unnest_preserving_empty_array_response.json index 14e2cf07..3a53e637 100644 --- a/document-store/src/integrationTest/resources/query/flat_unnest_preserving_empty_array_response.json +++ b/document-store/src/integrationTest/resources/query/flat_unnest_preserving_empty_array_response.json @@ -1,5 +1,5 @@ [ { - "count": 32 + "count": 27 } ] diff --git a/document-store/src/integrationTest/resources/query/flat_unnest_tags_response.json b/document-store/src/integrationTest/resources/query/flat_unnest_tags_response.json index 2771ac60..5aac4063 100644 --- a/document-store/src/integrationTest/resources/query/flat_unnest_tags_response.json +++ b/document-store/src/integrationTest/resources/query/flat_unnest_tags_response.json @@ -1,22 +1,18 @@ [ { - "tags": "wholesale", - "count": 1 + "tags": "hygiene", + "count": 3 }, { - "tags": "stationery", + "tags": "personal-care", "count": 2 }, { - "tags": "grooming", + "tags": "premium", "count": 2 }, { - "tags": "family-pack", - "count": 1 - }, - { - "tags": "glass", + "tags": "home-decor", "count": 1 }, { @@ -24,55 +20,51 @@ "count": 1 }, { - "tags": "home-decor", - "count": 1 - }, - { - "tags": "writing", + "tags": "glass", "count": 1 }, { - "tags": "premium", + "tags": "hair-care", "count": 2 }, { - "tags": "plastic", + "tags": "herbal", "count": 1 }, { - "tags": "basic", - "count": 1 + "tags": "budget", + "count": 2 }, { - "tags": "personal-care", + "tags": "bulk", "count": 2 }, { - "tags": "essential", + "tags": "antibacterial", "count": 1 }, { - "tags": "hygiene", - "count": 3 + "tags": "family-pack", + "count": 1 }, { - "tags": "budget", + "tags": "grooming", "count": 2 }, { - "tags": "antibacterial", + "tags": "plastic", "count": 1 }, { - "tags": "herbal", + "tags": "essential", "count": 1 }, { - "tags": "bulk", - "count": 2 + "tags": "wholesale", + "count": 1 }, { - "tags": "hair-care", - "count": 2 + "tags": "basic", + "count": 1 } -] \ No newline at end of file +] From d5b4b3833ce7a43d11582c1c3e3641ba5a880866 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Mon, 24 Nov 2025 11:56:53 +0530 Subject: [PATCH 09/14] Fixed failing test cases --- .../core/documentstore/DocStoreQueryV1Test.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java index 691bbdf5..0c3991c5 100644 --- a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java +++ b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java @@ -4626,14 +4626,14 @@ void testExistsFilterOnArray(String dataStoreName) throws JsonProcessingExceptio // Should return only documents with non-empty arrays // From test data: rows 1-8 have non-empty arrays (8 docs) - // Plus rows 12, 14 have non-empty arrays (2 docs) + // Plus rows 9, 10 have non-empty arrays (2 docs) // Total: 10 documents - assertEquals(10, count, "Should return a total of 10 docs that have non-empty tags"); + assertEquals(8, count, "Should return a total of 10 docs that have non-empty tags"); } /** - * Test NOT_EXISTS filter with ArrayIdentifierExpression. This validates that NOT_EXISTS on - * array fields returns both NULL and empty arrays, excluding only non-empty arrays. + * Test NOT_EXISTS filter on top-level arrays. This validates that NOT_EXISTS on array fields + * returns both NULL and empty arrays, excluding only non-empty arrays. */ @ParameterizedTest @ArgumentsSource(PostgresProvider.class) @@ -4672,7 +4672,7 @@ void testNotExistsFilterOnArrays(String dataStoreName) throws JsonProcessingExce // Should return documents with NULL or empty arrays // From test data: row 9 (NULL), rows 10, 11, 13 (empty arrays) // Total: 4 documents - assertEquals(4, count, "Should return at 4 documents with NULL or empty tags"); + assertEquals(2, count, "Should return at 4 documents with NULL or empty tags"); } /** @@ -4717,7 +4717,7 @@ void testExistsFilterOnJsonArrays(String dataStoreName) throws JsonProcessingExc "colors should be non-empty array, but was: " + colors); } - // Should return rows 1, 3, 5 which have non-empty colors arrays + // Should return rows 1, 2, 3 which have non-empty colors arrays assertEquals(3, count, "Should return exactly 3 documents with non-empty colors"); } @@ -4766,11 +4766,10 @@ void testNotExistsFilterOnJsonArrays(String dataStoreName) throws JsonProcessing colors == null || !colors.isArray() || colors.isEmpty(), "colors should be NULL or empty array for item: " + item + ", but was: " + colors); } - // NULL props is also valid + // NULL props is also valid (if props is null, then props->colours is null too) } // Should include documents where props is NULL or props.colors is NULL/empty - // Row 7 (Comb) has empty colors array, rows 2,4,6,8,9,10 have NULL props assertTrue(count > 0, "Should return at least some documents"); assertTrue( returnedItems.contains("Comb"), "Should include Comb (has empty colors array in props)"); From 6911569bb7f5733523b1a93f9ddd8875fba82148 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Mon, 24 Nov 2025 12:15:04 +0530 Subject: [PATCH 10/14] Add UTs for coverage --- .../PostgresExistsRelationalFilterParser.java | 10 +- ...or.java => PostgresFieldTypeDetector.java} | 2 +- ...stgresNotExistsRelationalFilterParser.java | 15 +- .../filter/PostgresFieldTypeDetectorTest.java | 156 ++++++++++++++++++ 4 files changed, 168 insertions(+), 15 deletions(-) rename document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/{PostgresArrayFieldDetector.java => PostgresFieldTypeDetector.java} (97%) create mode 100644 document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetectorTest.java diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java index 3ab6c03d..80dfd2ba 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java @@ -2,12 +2,10 @@ import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; -import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresArrayFieldDetector.FieldCategory; +import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresFieldTypeDetector.FieldCategory; class PostgresExistsRelationalFilterParser implements PostgresRelationalFilterParser { - private static final PostgresArrayFieldDetector ARRAY_DETECTOR = new PostgresArrayFieldDetector(); - @Override public String parse( final RelationalExpression expression, final PostgresRelationalFilterContext context) { @@ -19,10 +17,10 @@ public String parse( // If false: // Regular fields -> IS NULL // Arrays -> IS NULL OR cardinality(...) = 0, - // JSONB arrays: IS NULL OR jsonb_array_length(...) > 0 + // JSONB arrays: IS NULL OR (jsonb_typeof(%s) = 'array' AND jsonb_array_length(...) = 0) final boolean parsedRhs = !ConstantExpression.of(false).equals(expression.getRhs()); - FieldCategory category = expression.getLhs().accept(ARRAY_DETECTOR); + FieldCategory category = expression.getLhs().accept(new PostgresFieldTypeDetector()); switch (category) { case ARRAY: @@ -38,7 +36,7 @@ public String parse( "(%s IS NOT NULL AND jsonb_typeof(%s) = 'array' AND jsonb_array_length(%s) > 0)", parsedLhs, parsedLhs, parsedLhs) : String.format( - "(%s IS NULL OR jsonb_typeof(%s) != 'array' OR jsonb_array_length(%s) = 0)", + "(%s IS NULL OR (jsonb_typeof(%s) = 'array' AND jsonb_array_length(%s) = 0))", parsedLhs, parsedLhs, parsedLhs); case SCALAR: diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresArrayFieldDetector.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetector.java similarity index 97% rename from document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresArrayFieldDetector.java rename to document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetector.java index 46c2f820..c17d5cac 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresArrayFieldDetector.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetector.java @@ -30,7 +30,7 @@ *
  • JSONB_ARRAY: {@code IS NOT NULL AND jsonb_array_length(...) > 0} * */ -class PostgresArrayFieldDetector implements SelectTypeExpressionVisitor { +class PostgresFieldTypeDetector implements SelectTypeExpressionVisitor { /** Field category for determining appropriate SQL generation strategy */ enum FieldCategory { diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java index caa919c4..c22bc0c2 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java @@ -2,12 +2,10 @@ import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; -import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresArrayFieldDetector.FieldCategory; +import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresFieldTypeDetector.FieldCategory; class PostgresNotExistsRelationalFilterParser implements PostgresRelationalFilterParser { - private static final PostgresArrayFieldDetector ARRAY_DETECTOR = new PostgresArrayFieldDetector(); - @Override public String parse( final RelationalExpression expression, final PostgresRelationalFilterContext context) { @@ -19,26 +17,27 @@ public String parse( // If false (RHS = true or other): // Regular fields -> IS NULL // Arrays -> IS NULL OR cardinality(...) = 0 - // JSONB arrays: IS NULL OR jsonb_typeof(%s) != 'array' OR jsonb_array_length(...) = 0 + // JSONB arrays: IS NULL OR (jsonb_typeof(%s) = 'array' AND jsonb_array_length(...) = 0) final boolean parsedRhs = ConstantExpression.of(false).equals(expression.getRhs()); - FieldCategory category = expression.getLhs().accept(ARRAY_DETECTOR); + FieldCategory category = expression.getLhs().accept(new PostgresFieldTypeDetector()); switch (category) { case ARRAY: - // First-class PostgreSQL array columns (text[], int[], etc.) + // For first-class array fields, only return those arrays that are not null and have + // at-least 1 element in it (so exclude NULL or empty arrays). This is to match Mongo's + // behavior return parsedRhs ? String.format("(%s IS NOT NULL AND cardinality(%s) > 0)", parsedLhs, parsedLhs) : String.format("(%s IS NULL OR cardinality(%s) = 0)", parsedLhs, parsedLhs); case JSONB_ARRAY: - // Arrays inside JSONB columns return parsedRhs ? String.format( "(%s IS NOT NULL AND jsonb_typeof(%s) = 'array' AND jsonb_array_length(%s) > 0)", parsedLhs, parsedLhs, parsedLhs) : String.format( - "(%s IS NULL OR jsonb_typeof(%s) != 'array' OR jsonb_array_length(%s) = 0)", + "(%s IS NULL OR (jsonb_typeof(%s) = 'array' AND jsonb_array_length(%s) = 0))", parsedLhs, parsedLhs, parsedLhs); case SCALAR: diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetectorTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetectorTest.java new file mode 100644 index 00000000..f5013416 --- /dev/null +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetectorTest.java @@ -0,0 +1,156 @@ +package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.ArrayIdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.ArrayType; +import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; +import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; +import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.JsonFieldType; +import org.hypertrace.core.documentstore.expression.impl.JsonIdentifierExpression; +import org.hypertrace.core.documentstore.expression.operators.AggregationOperator; +import org.hypertrace.core.documentstore.expression.operators.FunctionOperator; +import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresFieldTypeDetector.FieldCategory; +import org.junit.jupiter.api.Test; + +class PostgresFieldTypeDetectorTest { + + private final PostgresFieldTypeDetector detector = new PostgresFieldTypeDetector(); + + @Test + void testVisitArrayIdentifierExpression_returnsArray() { + ArrayIdentifierExpression expr = ArrayIdentifierExpression.of("tags", ArrayType.TEXT); + FieldCategory result = detector.visit(expr); + assertNotNull(result); + assertEquals(FieldCategory.ARRAY, result, "ArrayIdentifierExpression should return ARRAY"); + } + + @Test + void testVisitJsonIdentifierExpression_stringArray_returnsJsonbArray() { + JsonIdentifierExpression expr = + JsonIdentifierExpression.of("props", JsonFieldType.STRING_ARRAY, "colors"); + FieldCategory result = detector.visit(expr); + assertNotNull(result); + assertEquals( + FieldCategory.JSONB_ARRAY, + result, + "JsonIdentifierExpression with STRING_ARRAY should return JSONB_ARRAY"); + } + + @Test + void testVisitJsonIdentifierExpression_numberArray_returnsJsonbArray() { + JsonIdentifierExpression expr = + JsonIdentifierExpression.of("props", JsonFieldType.NUMBER_ARRAY, "scores"); + FieldCategory result = detector.visit(expr); + assertNotNull(result); + assertEquals( + FieldCategory.JSONB_ARRAY, + result, + "JsonIdentifierExpression with NUMBER_ARRAY should return JSONB_ARRAY"); + } + + @Test + void testVisitJsonIdentifierExpression_booleanArray_returnsJsonbArray() { + JsonIdentifierExpression expr = + JsonIdentifierExpression.of("props", JsonFieldType.BOOLEAN_ARRAY, "flags"); + FieldCategory result = detector.visit(expr); + assertNotNull(result); + assertEquals( + FieldCategory.JSONB_ARRAY, + result, + "JsonIdentifierExpression with BOOLEAN_ARRAY should return JSONB_ARRAY"); + } + + @Test + void testVisitJsonIdentifierExpression_objectArray_returnsJsonbArray() { + JsonIdentifierExpression expr = + JsonIdentifierExpression.of("props", JsonFieldType.OBJECT_ARRAY, "items"); + FieldCategory result = detector.visit(expr); + assertNotNull(result); + assertEquals( + FieldCategory.JSONB_ARRAY, + result, + "JsonIdentifierExpression with OBJECT_ARRAY should return JSONB_ARRAY"); + } + + @Test + void testVisitJsonIdentifierExpression_stringScalar_returnsScalar() { + JsonIdentifierExpression expr = + JsonIdentifierExpression.of("props", JsonFieldType.STRING, "brand"); + FieldCategory result = detector.visit(expr); + assertNotNull(result); + assertEquals( + FieldCategory.SCALAR, result, "JsonIdentifierExpression with STRING should return SCALAR"); + } + + @Test + void testVisitJsonIdentifierExpression_noFieldType_returnsScalar() { + JsonIdentifierExpression expr = JsonIdentifierExpression.of("props", "field"); + FieldCategory result = detector.visit(expr); + assertNotNull(result); + assertEquals( + FieldCategory.SCALAR, + result, + "JsonIdentifierExpression without field type should return SCALAR"); + } + + @Test + void testVisitIdentifierExpression_returnsScalar() { + IdentifierExpression expr = IdentifierExpression.of("item"); + FieldCategory result = detector.visit(expr); + assertNotNull(result); + assertEquals(FieldCategory.SCALAR, result, "IdentifierExpression should return SCALAR"); + } + + @Test + void testVisitAggregateExpression_returnsScalar() { + AggregateExpression expr = + AggregateExpression.of(AggregationOperator.COUNT, IdentifierExpression.of("item")); + FieldCategory result = detector.visit(expr); + assertNotNull(result); + assertEquals(FieldCategory.SCALAR, result, "AggregateExpression should return SCALAR"); + } + + @Test + void testVisitConstantExpression_returnsScalar() { + ConstantExpression expr = ConstantExpression.of("test"); + FieldCategory result = detector.visit(expr); + assertNotNull(result); + assertEquals(FieldCategory.SCALAR, result, "ConstantExpression should return SCALAR"); + } + + @Test + void testVisitDocumentConstantExpression_returnsScalar() { + ConstantExpression.DocumentConstantExpression expr = + (ConstantExpression.DocumentConstantExpression) + ConstantExpression.of((org.hypertrace.core.documentstore.Document) null); + FieldCategory result = detector.visit(expr); + assertNotNull(result); + assertEquals(FieldCategory.SCALAR, result, "DocumentConstantExpression should return SCALAR"); + } + + @Test + void testVisitFunctionExpression_returnsScalar() { + FunctionExpression expr = + FunctionExpression.builder() + .operator(FunctionOperator.LENGTH) + .operand(IdentifierExpression.of("item")) + .build(); + FieldCategory result = detector.visit(expr); + assertNotNull(result); + assertEquals(FieldCategory.SCALAR, result, "FunctionExpression should return SCALAR"); + } + + @Test + void testVisitAliasedIdentifierExpression_returnsScalar() { + AliasedIdentifierExpression expr = + AliasedIdentifierExpression.builder().name("item").contextAlias("i").build(); + FieldCategory result = detector.visit(expr); + assertNotNull(result); + assertEquals(FieldCategory.SCALAR, result, "AliasedIdentifierExpression should return SCALAR"); + } +} From e1a3671c7634683b33462162c856133c3c133007 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Mon, 24 Nov 2025 12:38:25 +0530 Subject: [PATCH 11/14] Add UTs for coverage --- ...tgresExistsRelationalFilterParserTest.java | 159 ++++++++++++++++++ ...esNotExistsRelationalFilterParserTest.java | 159 ++++++++++++++++++ 2 files changed, 318 insertions(+) create mode 100644 document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParserTest.java create mode 100644 document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParserTest.java diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParserTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParserTest.java new file mode 100644 index 00000000..060df64d --- /dev/null +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParserTest.java @@ -0,0 +1,159 @@ +package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter; + +import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.EXISTS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.hypertrace.core.documentstore.expression.impl.ArrayIdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.ArrayType; +import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; +import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.JsonFieldType; +import org.hypertrace.core.documentstore.expression.impl.JsonIdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; +import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresRelationalFilterParser.PostgresRelationalFilterContext; +import org.hypertrace.core.documentstore.postgres.query.v1.vistors.PostgresSelectTypeExpressionVisitor; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class PostgresExistsRelationalFilterParserTest { + + private PostgresExistsRelationalFilterParser parser; + private PostgresRelationalFilterContext context; + private PostgresSelectTypeExpressionVisitor lhsParser; + + @BeforeEach + void setUp() { + parser = new PostgresExistsRelationalFilterParser(); + context = mock(PostgresRelationalFilterContext.class); + lhsParser = mock(PostgresSelectTypeExpressionVisitor.class); + when(context.lhsParser()).thenReturn(lhsParser); + } + + @Test + void testParse_arrayField_rhsTrue() { + // Test EXISTS on array with RHS = true + ArrayIdentifierExpression lhs = ArrayIdentifierExpression.of("tags", ArrayType.TEXT); + ConstantExpression rhs = ConstantExpression.of("null"); // Any non-false value + RelationalExpression expression = RelationalExpression.of(lhs, EXISTS, rhs); + + when(lhsParser.visit(any(ArrayIdentifierExpression.class))).thenReturn("\"tags\""); + + String result = parser.parse(expression, context); + + assertEquals( + "(\"tags\" IS NOT NULL AND cardinality(\"tags\") > 0)", + result, + "EXISTS with RHS=true on ARRAY should check IS NOT NULL AND cardinality > 0"); + } + + @Test + void testParse_arrayField_rhsFalse() { + // Test EXISTS on array with RHS = false + ArrayIdentifierExpression lhs = ArrayIdentifierExpression.of("tags", ArrayType.TEXT); + ConstantExpression rhs = ConstantExpression.of(false); + RelationalExpression expression = RelationalExpression.of(lhs, EXISTS, rhs); + + when(lhsParser.visit(any(ArrayIdentifierExpression.class))).thenReturn("\"tags\""); + + String result = parser.parse(expression, context); + + assertEquals( + "(\"tags\" IS NULL OR cardinality(\"tags\") = 0)", + result, + "EXISTS with RHS=false on ARRAY should check IS NULL OR cardinality = 0"); + } + + @Test + void testParse_jsonbArrayField_rhsTrue() { + // Test EXISTS on JSONB array with RHS = true + JsonIdentifierExpression lhs = + JsonIdentifierExpression.of("props", JsonFieldType.STRING_ARRAY, "colors"); + ConstantExpression rhs = ConstantExpression.of("null"); + RelationalExpression expression = RelationalExpression.of(lhs, EXISTS, rhs); + + when(lhsParser.visit(any(JsonIdentifierExpression.class))) + .thenReturn("document->'props'->'colors'"); + + String result = parser.parse(expression, context); + + assertEquals( + "(document->'props'->'colors' IS NOT NULL AND jsonb_typeof(document->'props'->'colors') = 'array' AND jsonb_array_length(document->'props'->'colors') > 0)", + result, + "EXISTS with RHS=true on JSONB_ARRAY should check IS NOT NULL AND typeof = 'array' AND length > 0"); + } + + @Test + void testParse_jsonbArrayField_rhsFalse() { + // Test EXISTS on JSONB array with RHS = false + JsonIdentifierExpression lhs = + JsonIdentifierExpression.of("props", JsonFieldType.NUMBER_ARRAY, "scores"); + ConstantExpression rhs = ConstantExpression.of(false); + RelationalExpression expression = RelationalExpression.of(lhs, EXISTS, rhs); + + when(lhsParser.visit(any(JsonIdentifierExpression.class))) + .thenReturn("document->'props'->'scores'"); + + String result = parser.parse(expression, context); + + assertEquals( + "(document->'props'->'scores' IS NULL OR (jsonb_typeof(document->'props'->'scores') = 'array' AND jsonb_array_length(document->'props'->'scores') = 0))", + result, + "EXISTS with RHS=false on JSONB_ARRAY should check IS NULL OR (typeof = 'array' AND length = 0)"); + } + + @Test + void testParse_scalarField_rhsTrue() { + // Test EXISTS on scalar field with RHS = true + IdentifierExpression lhs = IdentifierExpression.of("item"); + ConstantExpression rhs = ConstantExpression.of("null"); + RelationalExpression expression = RelationalExpression.of(lhs, EXISTS, rhs); + + when(lhsParser.visit(any(IdentifierExpression.class))).thenReturn("document->>'item'"); + + String result = parser.parse(expression, context); + + assertEquals( + "document->>'item' IS NOT NULL", + result, + "EXISTS with RHS=true on SCALAR should check IS NOT NULL"); + } + + @Test + void testParse_scalarField_rhsFalse() { + // Test EXISTS on scalar field with RHS = false + IdentifierExpression lhs = IdentifierExpression.of("item"); + ConstantExpression rhs = ConstantExpression.of(false); + RelationalExpression expression = RelationalExpression.of(lhs, EXISTS, rhs); + + when(lhsParser.visit(any(IdentifierExpression.class))).thenReturn("document->>'item'"); + + String result = parser.parse(expression, context); + + assertEquals( + "document->>'item' IS NULL", + result, + "EXISTS with RHS=false on SCALAR should check IS NULL"); + } + + @Test + void testParse_jsonScalarField_rhsTrue() { + // Test EXISTS on JSON scalar (non-array) field with RHS = true + JsonIdentifierExpression lhs = + JsonIdentifierExpression.of("props", JsonFieldType.STRING, "brand"); + ConstantExpression rhs = ConstantExpression.of("null"); + RelationalExpression expression = RelationalExpression.of(lhs, EXISTS, rhs); + + when(lhsParser.visit(any(JsonIdentifierExpression.class))) + .thenReturn("document->'props'->>'brand'"); + + String result = parser.parse(expression, context); + + assertEquals( + "document->'props'->>'brand' IS NOT NULL", + result, + "EXISTS with RHS=true on JSON scalar should check IS NOT NULL"); + } +} diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParserTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParserTest.java new file mode 100644 index 00000000..9b4fa7fc --- /dev/null +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParserTest.java @@ -0,0 +1,159 @@ +package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter; + +import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.NOT_EXISTS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.hypertrace.core.documentstore.expression.impl.ArrayIdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.ArrayType; +import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; +import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.JsonFieldType; +import org.hypertrace.core.documentstore.expression.impl.JsonIdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; +import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresRelationalFilterParser.PostgresRelationalFilterContext; +import org.hypertrace.core.documentstore.postgres.query.v1.vistors.PostgresSelectTypeExpressionVisitor; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class PostgresNotExistsRelationalFilterParserTest { + + private PostgresNotExistsRelationalFilterParser parser; + private PostgresRelationalFilterContext context; + private PostgresSelectTypeExpressionVisitor lhsParser; + + @BeforeEach + void setUp() { + parser = new PostgresNotExistsRelationalFilterParser(); + context = mock(PostgresRelationalFilterContext.class); + lhsParser = mock(PostgresSelectTypeExpressionVisitor.class); + when(context.lhsParser()).thenReturn(lhsParser); + } + + @Test + void testParse_arrayField_rhsFalse() { + // Test NOT_EXISTS on array with RHS = false (means NOT_EXISTS should be true) + ArrayIdentifierExpression lhs = ArrayIdentifierExpression.of("tags", ArrayType.TEXT); + ConstantExpression rhs = ConstantExpression.of(false); + RelationalExpression expression = RelationalExpression.of(lhs, NOT_EXISTS, rhs); + + when(lhsParser.visit(any(ArrayIdentifierExpression.class))).thenReturn("\"tags\""); + + String result = parser.parse(expression, context); + + assertEquals( + "(\"tags\" IS NOT NULL AND cardinality(\"tags\") > 0)", + result, + "NOT_EXISTS with RHS=false on ARRAY should check IS NOT NULL AND cardinality > 0"); + } + + @Test + void testParse_arrayField_rhsTrue() { + // Test NOT_EXISTS on array with RHS = true (means NOT_EXISTS should be false) + ArrayIdentifierExpression lhs = ArrayIdentifierExpression.of("tags", ArrayType.TEXT); + ConstantExpression rhs = ConstantExpression.of("null"); // Any non-false value + RelationalExpression expression = RelationalExpression.of(lhs, NOT_EXISTS, rhs); + + when(lhsParser.visit(any(ArrayIdentifierExpression.class))).thenReturn("\"tags\""); + + String result = parser.parse(expression, context); + + assertEquals( + "(\"tags\" IS NULL OR cardinality(\"tags\") = 0)", + result, + "NOT_EXISTS with RHS=true on ARRAY should check IS NULL OR cardinality = 0"); + } + + @Test + void testParse_jsonbArrayField_rhsFalse() { + // Test NOT_EXISTS on JSONB array with RHS = false + JsonIdentifierExpression lhs = + JsonIdentifierExpression.of("props", JsonFieldType.STRING_ARRAY, "colors"); + ConstantExpression rhs = ConstantExpression.of(false); + RelationalExpression expression = RelationalExpression.of(lhs, NOT_EXISTS, rhs); + + when(lhsParser.visit(any(JsonIdentifierExpression.class))) + .thenReturn("document->'props'->'colors'"); + + String result = parser.parse(expression, context); + + assertEquals( + "(document->'props'->'colors' IS NOT NULL AND jsonb_typeof(document->'props'->'colors') = 'array' AND jsonb_array_length(document->'props'->'colors') > 0)", + result, + "NOT_EXISTS with RHS=false on JSONB_ARRAY should check IS NOT NULL AND typeof = 'array' AND length > 0"); + } + + @Test + void testParse_jsonbArrayField_rhsTrue() { + // Test NOT_EXISTS on JSONB array with RHS = true + JsonIdentifierExpression lhs = + JsonIdentifierExpression.of("props", JsonFieldType.BOOLEAN_ARRAY, "flags"); + ConstantExpression rhs = ConstantExpression.of("null"); + RelationalExpression expression = RelationalExpression.of(lhs, NOT_EXISTS, rhs); + + when(lhsParser.visit(any(JsonIdentifierExpression.class))) + .thenReturn("document->'props'->'flags'"); + + String result = parser.parse(expression, context); + + assertEquals( + "(document->'props'->'flags' IS NULL OR (jsonb_typeof(document->'props'->'flags') = 'array' AND jsonb_array_length(document->'props'->'flags') = 0))", + result, + "NOT_EXISTS with RHS=true on JSONB_ARRAY should check IS NULL OR (typeof = 'array' AND length = 0)"); + } + + @Test + void testParse_scalarField_rhsFalse() { + // Test NOT_EXISTS on scalar field with RHS = false + IdentifierExpression lhs = IdentifierExpression.of("item"); + ConstantExpression rhs = ConstantExpression.of(false); + RelationalExpression expression = RelationalExpression.of(lhs, NOT_EXISTS, rhs); + + when(lhsParser.visit(any(IdentifierExpression.class))).thenReturn("document->>'item'"); + + String result = parser.parse(expression, context); + + assertEquals( + "document->>'item' IS NOT NULL", + result, + "NOT_EXISTS with RHS=false on SCALAR should check IS NOT NULL"); + } + + @Test + void testParse_scalarField_rhsTrue() { + // Test NOT_EXISTS on scalar field with RHS = true + IdentifierExpression lhs = IdentifierExpression.of("item"); + ConstantExpression rhs = ConstantExpression.of("null"); + RelationalExpression expression = RelationalExpression.of(lhs, NOT_EXISTS, rhs); + + when(lhsParser.visit(any(IdentifierExpression.class))).thenReturn("document->>'item'"); + + String result = parser.parse(expression, context); + + assertEquals( + "document->>'item' IS NULL", + result, + "NOT_EXISTS with RHS=true on SCALAR should check IS NULL"); + } + + @Test + void testParse_jsonScalarField_rhsFalse() { + // Test NOT_EXISTS on JSON scalar (non-array) field with RHS = false + JsonIdentifierExpression lhs = + JsonIdentifierExpression.of("props", JsonFieldType.STRING, "brand"); + ConstantExpression rhs = ConstantExpression.of(false); + RelationalExpression expression = RelationalExpression.of(lhs, NOT_EXISTS, rhs); + + when(lhsParser.visit(any(JsonIdentifierExpression.class))) + .thenReturn("document->'props'->>'brand'"); + + String result = parser.parse(expression, context); + + assertEquals( + "document->'props'->>'brand' IS NOT NULL", + result, + "NOT_EXISTS with RHS=false on JSON scalar should check IS NOT NULL"); + } +} From 828b73c79fc3e45d1ab554f66a8f3e9b79408550 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Mon, 24 Nov 2025 23:39:05 +0530 Subject: [PATCH 12/14] Optimise queries for indices --- .../PostgresExistsRelationalFilterParser.java | 51 +++-- .../filter/PostgresFieldTypeDetector.java | 18 +- ...stgresNotExistsRelationalFilterParser.java | 53 ++++- .../query/v1/PostgresQueryParserTest.java | 210 ++++++++++++++++++ ...tgresExistsRelationalFilterParserTest.java | 43 +++- ...esNotExistsRelationalFilterParserTest.java | 43 +++- 6 files changed, 362 insertions(+), 56 deletions(-) diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java index 80dfd2ba..224704c4 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java @@ -1,6 +1,7 @@ package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; +import org.hypertrace.core.documentstore.expression.impl.JsonIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresFieldTypeDetector.FieldCategory; @@ -13,11 +14,11 @@ public String parse( // If true: // Regular fields -> IS NOT NULL // Arrays -> IS NOT NULL and cardinality(...) > 0, - // JSONB arrays: IS NOT NULL and jsonb_array_length(...) > 0 + // JSONB arrays: Optimized GIN index query with containment check // If false: // Regular fields -> IS NULL // Arrays -> IS NULL OR cardinality(...) = 0, - // JSONB arrays: IS NULL OR (jsonb_typeof(%s) = 'array' AND jsonb_array_length(...) = 0) + // JSONB arrays: COALESCE with array length check final boolean parsedRhs = !ConstantExpression.of(false).equals(expression.getRhs()); FieldCategory category = expression.getLhs().accept(new PostgresFieldTypeDetector()); @@ -26,25 +27,49 @@ public String parse( case ARRAY: // First-class PostgreSQL array columns (text[], int[], etc.) return parsedRhs - ? String.format("(%s IS NOT NULL AND cardinality(%s) > 0)", parsedLhs, parsedLhs) - : String.format("(%s IS NULL OR cardinality(%s) = 0)", parsedLhs, parsedLhs); + // We don't need to check that LHS is NOT NULL because WHERE cardinality(NULL) will not + // be included in the result set + ? String.format("(cardinality(%s) > 0)", parsedLhs) + : String.format("COALESCE(cardinality(%s), 0) = 0", parsedLhs); case JSONB_ARRAY: - // Arrays inside JSONB columns - return parsedRhs - ? String.format( - "(%s IS NOT NULL AND jsonb_typeof(%s) = 'array' AND jsonb_array_length(%s) > 0)", - parsedLhs, parsedLhs, parsedLhs) - : String.format( - "(%s IS NULL OR (jsonb_typeof(%s) = 'array' AND jsonb_array_length(%s) = 0))", - parsedLhs, parsedLhs, parsedLhs); + { + JsonIdentifierExpression jsonExpr = (JsonIdentifierExpression) expression.getLhs(); + String baseColumn = wrapWithDoubleQuotes(jsonExpr.getColumnName()); + String nestedPath = String.join(".", jsonExpr.getJsonPath()); + return parsedRhs + // This is type-safe and will use the GIN index on parent JSONB col + ? String.format( + "(%s @> '{\"" + nestedPath + "\": []}' AND jsonb_array_length(%s) > 0)", + baseColumn, + parsedLhs) + // Return the number of elements in a JSONB array, default value of 0 if the array is + // NULL + : String.format("COALESCE(jsonb_array_length(%s), 0) = 0", parsedLhs); + } + + case JSONB_SCALAR: + { + // JSONB scalar fields - use ? operator for GIN index optimization + JsonIdentifierExpression jsonExpr = (JsonIdentifierExpression) expression.getLhs(); + String baseColumn = wrapWithDoubleQuotes(jsonExpr.getColumnName()); + String nestedPath = String.join(".", jsonExpr.getJsonPath()); + + return parsedRhs + ? String.format("%s ? '%s'", baseColumn, nestedPath) + : String.format("NOT (%s ? '%s')", baseColumn, nestedPath); + } case SCALAR: default: - // Regular scalar fields + // Regular scalar fields - use standard NULL checks return parsedRhs ? String.format("%s IS NOT NULL", parsedLhs) : String.format("%s IS NULL", parsedLhs); } } + + private String wrapWithDoubleQuotes(String identifier) { + return "\"" + identifier + "\""; + } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetector.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetector.java index c17d5cac..3a54cc1f 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetector.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetector.java @@ -14,11 +14,13 @@ /** * Visitor to detect the category of a field expression for array-aware SQL generation. * - *

    Categorizes fields into three types: + *

    Categorizes fields into four types: * *

      - *
    • SCALAR: Regular fields and JSON primitives (strings, numbers, booleans, objects) - *
    • POSTGRES_ARRAY: Native PostgreSQL arrays (text[], integer[], boolean[], etc.) + *
    • SCALAR: Regular non-JSON fields + *
    • ARRAY: Native PostgreSQL arrays (text[], integer[], boolean[], etc.) + *
    • JSONB_SCALAR: Scalar fields inside JSONB columns (strings, numbers, booleans, + * objects) *
    • JSONB_ARRAY: Arrays inside JSONB columns with JsonFieldType annotation *
    * @@ -26,16 +28,18 @@ * *
      *
    • SCALAR: {@code IS NOT NULL / IS NULL} - *
    • POSTGRES_ARRAY: {@code IS NOT NULL AND cardinality(...) > 0} - *
    • JSONB_ARRAY: {@code IS NOT NULL AND jsonb_array_length(...) > 0} + *
    • ARRAY: {@code IS NOT NULL AND cardinality(...) > 0} + *
    • JSONB_SCALAR: {@code "col" ? 'field'} (uses GIN index) + *
    • JSONB_ARRAY: {@code "col" @> '{field:[]}' AND jsonb_array_length(...) > 0} (uses GIN index) *
    */ class PostgresFieldTypeDetector implements SelectTypeExpressionVisitor { /** Field category for determining appropriate SQL generation strategy */ enum FieldCategory { - SCALAR, // Regular fields and JSON primitives + SCALAR, // Regular non-JSON fields ARRAY, // Native PostgreSQL arrays (text[], int[], etc.) + JSONB_SCALAR, // Scalar fields inside JSONB columns JSONB_ARRAY // Arrays inside JSONB columns } @@ -55,7 +59,7 @@ public FieldCategory visit(JsonIdentifierExpression expression) { || type == JsonFieldType.BOOLEAN_ARRAY || type == JsonFieldType.OBJECT_ARRAY) .map(type -> FieldCategory.JSONB_ARRAY) - .orElse(FieldCategory.SCALAR); + .orElse(FieldCategory.JSONB_SCALAR); } @Override diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java index c22bc0c2..2d558a02 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParser.java @@ -1,6 +1,7 @@ package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; +import org.hypertrace.core.documentstore.expression.impl.JsonIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresFieldTypeDetector.FieldCategory; @@ -13,11 +14,11 @@ public String parse( // If true (RHS = false): // Regular fields -> IS NOT NULL // Arrays -> IS NOT NULL AND cardinality(...) > 0 - // JSONB arrays: IS NOT NULL AND jsonb_typeof(%s) = 'array' AND jsonb_array_length(...) > 0 + // JSONB arrays: Optimized GIN index query with containment check // If false (RHS = true or other): // Regular fields -> IS NULL // Arrays -> IS NULL OR cardinality(...) = 0 - // JSONB arrays: IS NULL OR (jsonb_typeof(%s) = 'array' AND jsonb_array_length(...) = 0) + // JSONB arrays: COALESCE with array length check final boolean parsedRhs = ConstantExpression.of(false).equals(expression.getRhs()); FieldCategory category = expression.getLhs().accept(new PostgresFieldTypeDetector()); @@ -28,24 +29,52 @@ public String parse( // at-least 1 element in it (so exclude NULL or empty arrays). This is to match Mongo's // behavior return parsedRhs - ? String.format("(%s IS NOT NULL AND cardinality(%s) > 0)", parsedLhs, parsedLhs) - : String.format("(%s IS NULL OR cardinality(%s) = 0)", parsedLhs, parsedLhs); + ? String.format("(cardinality(%s) > 0)", parsedLhs) + // More efficient than: %s IS NULL OR cardinality(%s) = 0)? as we can create + // an index on the COALESCE function itself which will return in a single + // index seek rather than two index seeks in the OR query + : String.format("COALESCE(cardinality(%s), 0) = 0", parsedLhs); case JSONB_ARRAY: - return parsedRhs - ? String.format( - "(%s IS NOT NULL AND jsonb_typeof(%s) = 'array' AND jsonb_array_length(%s) > 0)", - parsedLhs, parsedLhs, parsedLhs) - : String.format( - "(%s IS NULL OR (jsonb_typeof(%s) = 'array' AND jsonb_array_length(%s) = 0))", - parsedLhs, parsedLhs, parsedLhs); + { + // Arrays inside JSONB columns - use optimized GIN index queries + JsonIdentifierExpression jsonExpr = (JsonIdentifierExpression) expression.getLhs(); + String baseColumn = wrapWithDoubleQuotes(jsonExpr.getColumnName()); + String nestedPath = String.join(".", jsonExpr.getJsonPath()); + + return parsedRhs + ? String.format( + "(%s @> '{\"" + nestedPath + "\": []}' AND jsonb_array_length(%s) > 0)", + baseColumn, + parsedLhs) + : String.format("COALESCE(jsonb_array_length(%s), 0) = 0", parsedLhs); + } + + case JSONB_SCALAR: + { + // JSONB scalar fields - use ? operator for GIN index optimization + JsonIdentifierExpression jsonExpr = (JsonIdentifierExpression) expression.getLhs(); + String baseColumn = wrapWithDoubleQuotes(jsonExpr.getColumnName()); + String nestedPath = String.join(".", jsonExpr.getJsonPath()); + + return parsedRhs + // Uses the GIN index on the parent JSONB col + ? String.format("%s ? '%s'", baseColumn, nestedPath) + // Does not use the GIN index but is more computationally efficient than doing a IS + // NULL check + : String.format("NOT (%s ? '%s')", baseColumn, nestedPath); + } case SCALAR: default: - // Regular scalar fields + // Regular scalar fields - use standard NULL checks return parsedRhs ? String.format("%s IS NOT NULL", parsedLhs) : String.format("%s IS NULL", parsedLhs); } } + + private String wrapWithDoubleQuotes(String identifier) { + return "\"" + identifier + "\""; + } } diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java index 0bdac88e..0c41a793 100644 --- a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java @@ -15,12 +15,14 @@ import static org.hypertrace.core.documentstore.expression.operators.LogicalOperator.OR; import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.CONTAINS; import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.EQ; +import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.EXISTS; import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.GT; import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.GTE; import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.IN; import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.LTE; import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.NEQ; import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.NOT_CONTAINS; +import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.NOT_EXISTS; import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.NOT_IN; import static org.hypertrace.core.documentstore.expression.operators.SortOrder.ASC; import static org.hypertrace.core.documentstore.expression.operators.SortOrder.DESC; @@ -31,9 +33,13 @@ import org.hypertrace.core.documentstore.JSONDocument; import org.hypertrace.core.documentstore.SingleValueKey; import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.ArrayIdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.ArrayType; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.JsonFieldType; +import org.hypertrace.core.documentstore.expression.impl.JsonIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.KeyExpression; import org.hypertrace.core.documentstore.expression.impl.LogicalExpression; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; @@ -50,6 +56,7 @@ import org.hypertrace.core.documentstore.query.SelectionSpec; import org.hypertrace.core.documentstore.query.Sort; import org.hypertrace.core.documentstore.query.SortingSpec; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; public class PostgresQueryParserTest { @@ -1551,4 +1558,207 @@ void testContainsWithFlatCollectionNonJsonField() { assertEquals(1, params.getObjectParams().size()); assertEquals(List.of("java"), params.getObjectParams().get(1)); } + + @Nested + class FlatCollectionExistsNotExistsParserTest { + + @Test + void testExistsOnTopLevelScalarField() { + Query query = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("status"), EXISTS, ConstantExpression.of("null"))) + .build(); + + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser( + TEST_TABLE, + PostgresQueryTransformer.transform(query), + new FlatPostgresFieldTransformer()); + + String sql = postgresQueryParser.parse(); + assertEquals("SELECT * FROM \"testCollection\" WHERE \"status\" IS NOT NULL", sql); + + Params params = postgresQueryParser.getParamsBuilder().build(); + assertEquals(0, params.getObjectParams().size()); + } + + @Test + void testExistsOnTopLevelArrayField() { + // Query on IS_NOT_EMPTY, returns arrays with length > 0 + Query query = + Query.builder() + .setFilter( + RelationalExpression.of( + ArrayIdentifierExpression.of("tags", ArrayType.TEXT), + EXISTS, + ConstantExpression.of("null"))) + .build(); + + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser( + TEST_TABLE, + PostgresQueryTransformer.transform(query), + new FlatPostgresFieldTransformer()); + + String sql = postgresQueryParser.parse(); + assertEquals("SELECT * FROM \"testCollection\" WHERE (cardinality(\"tags\") > 0)", sql); + + Params params = postgresQueryParser.getParamsBuilder().build(); + assertEquals(0, params.getObjectParams().size()); + } + + @Test + void testExistsOnJsonbScalarField() { + Query query = + Query.builder() + .setFilter( + RelationalExpression.of( + JsonIdentifierExpression.of("customAttribute", JsonFieldType.STRING, "brand"), + EXISTS, + ConstantExpression.of("null"))) + .build(); + + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser( + TEST_TABLE, + PostgresQueryTransformer.transform(query), + new FlatPostgresFieldTransformer()); + + String sql = postgresQueryParser.parse(); + assertEquals("SELECT * FROM \"testCollection\" WHERE \"customAttribute\" ? 'brand'", sql); + + Params params = postgresQueryParser.getParamsBuilder().build(); + assertEquals(0, params.getObjectParams().size()); + } + + @Test + void testExistsOnJsonbArrayField() { + // Query on IS_NOT_EMPTY, returns arrays with length > 0 + Query query = + Query.builder() + .setFilter( + RelationalExpression.of( + JsonIdentifierExpression.of("props", JsonFieldType.STRING_ARRAY, "colors"), + EXISTS, + ConstantExpression.of("null"))) + .build(); + + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser( + TEST_TABLE, + PostgresQueryTransformer.transform(query), + new FlatPostgresFieldTransformer()); + + String sql = postgresQueryParser.parse(); + assertEquals( + "SELECT * FROM \"testCollection\" WHERE (\"props\" @> '{\"colors\": []}' AND jsonb_array_length(\"props\"->'colors') > 0)", + sql); + + Params params = postgresQueryParser.getParamsBuilder().build(); + assertEquals(0, params.getObjectParams().size()); + } + + @Test + void testNotExistsOnScalarField() { + Query query = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("status"), NOT_EXISTS, ConstantExpression.of("null"))) + .build(); + + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser( + TEST_TABLE, + PostgresQueryTransformer.transform(query), + new FlatPostgresFieldTransformer()); + + String sql = postgresQueryParser.parse(); + assertEquals("SELECT * FROM \"testCollection\" WHERE \"status\" IS NULL", sql); + + Params params = postgresQueryParser.getParamsBuilder().build(); + assertEquals(0, params.getObjectParams().size()); + } + + @Test + void testNotExistsOnArrayField() { + // Query on IS_NOT_EMPTY, returns arrays with length > 0 + Query query = + Query.builder() + .setFilter( + RelationalExpression.of( + ArrayIdentifierExpression.of("tags", ArrayType.TEXT), + NOT_EXISTS, + ConstantExpression.of("null"))) + .build(); + + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser( + TEST_TABLE, + PostgresQueryTransformer.transform(query), + new FlatPostgresFieldTransformer()); + + String sql = postgresQueryParser.parse(); + assertEquals( + "SELECT * FROM \"testCollection\" WHERE COALESCE(cardinality(\"tags\"), 0) = 0", sql); + + Params params = postgresQueryParser.getParamsBuilder().build(); + assertEquals(0, params.getObjectParams().size()); + } + + @Test + void testNotExistsOnJsonbScalarField() { + Query query = + Query.builder() + .setFilter( + RelationalExpression.of( + JsonIdentifierExpression.of("customAttribute", JsonFieldType.STRING, "brand"), + NOT_EXISTS, + ConstantExpression.of("null"))) + .build(); + + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser( + TEST_TABLE, + PostgresQueryTransformer.transform(query), + new FlatPostgresFieldTransformer()); + + String sql = postgresQueryParser.parse(); + assertEquals( + "SELECT * FROM \"testCollection\" WHERE NOT (\"customAttribute\" ? 'brand')", sql); + + Params params = postgresQueryParser.getParamsBuilder().build(); + assertEquals(0, params.getObjectParams().size()); + } + + @Test + void testNotExistsOnJsonbArrayField() { + // this is a IS_EMPTY query in a nested jsonb array. Returns arrays that are either NULL or + // empty arrays + Query query = + Query.builder() + .setFilter( + RelationalExpression.of( + JsonIdentifierExpression.of("props", JsonFieldType.STRING_ARRAY, "colors"), + NOT_EXISTS, + ConstantExpression.of("null"))) + .build(); + + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser( + TEST_TABLE, + PostgresQueryTransformer.transform(query), + new FlatPostgresFieldTransformer()); + + String sql = postgresQueryParser.parse(); + assertEquals( + "SELECT * FROM \"testCollection\" WHERE COALESCE(jsonb_array_length(\"props\"->'colors'), 0) = 0", + sql); + + Params params = postgresQueryParser.getParamsBuilder().build(); + assertEquals(0, params.getObjectParams().size()); + } + } } diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParserTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParserTest.java index 060df64d..2fbade12 100644 --- a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParserTest.java +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParserTest.java @@ -44,9 +44,9 @@ void testParse_arrayField_rhsTrue() { String result = parser.parse(expression, context); assertEquals( - "(\"tags\" IS NOT NULL AND cardinality(\"tags\") > 0)", + "(cardinality(\"tags\") > 0)", result, - "EXISTS with RHS=true on ARRAY should check IS NOT NULL AND cardinality > 0"); + "EXISTS with RHS=true on ARRAY should check cardinality > 0"); } @Test @@ -61,9 +61,9 @@ void testParse_arrayField_rhsFalse() { String result = parser.parse(expression, context); assertEquals( - "(\"tags\" IS NULL OR cardinality(\"tags\") = 0)", + "COALESCE(cardinality(\"tags\"), 0) = 0", result, - "EXISTS with RHS=false on ARRAY should check IS NULL OR cardinality = 0"); + "EXISTS with RHS=false on ARRAY should use COALESCE for NULL or empty check"); } @Test @@ -80,9 +80,9 @@ void testParse_jsonbArrayField_rhsTrue() { String result = parser.parse(expression, context); assertEquals( - "(document->'props'->'colors' IS NOT NULL AND jsonb_typeof(document->'props'->'colors') = 'array' AND jsonb_array_length(document->'props'->'colors') > 0)", + "(\"props\" @> '{\"colors\": []}' AND jsonb_array_length(document->'props'->'colors') > 0)", result, - "EXISTS with RHS=true on JSONB_ARRAY should check IS NOT NULL AND typeof = 'array' AND length > 0"); + "EXISTS with RHS=true on JSONB_ARRAY should use optimized GIN index containment query"); } @Test @@ -99,9 +99,9 @@ void testParse_jsonbArrayField_rhsFalse() { String result = parser.parse(expression, context); assertEquals( - "(document->'props'->'scores' IS NULL OR (jsonb_typeof(document->'props'->'scores') = 'array' AND jsonb_array_length(document->'props'->'scores') = 0))", + "COALESCE(jsonb_array_length(document->'props'->'scores'), 0) = 0", result, - "EXISTS with RHS=false on JSONB_ARRAY should check IS NULL OR (typeof = 'array' AND length = 0)"); + "EXISTS with RHS=false on JSONB_ARRAY should use COALESCE for NULL or empty arrays"); } @Test @@ -142,18 +142,37 @@ void testParse_scalarField_rhsFalse() { void testParse_jsonScalarField_rhsTrue() { // Test EXISTS on JSON scalar (non-array) field with RHS = true JsonIdentifierExpression lhs = - JsonIdentifierExpression.of("props", JsonFieldType.STRING, "brand"); + JsonIdentifierExpression.of("customAttribute", JsonFieldType.STRING, "brand"); ConstantExpression rhs = ConstantExpression.of("null"); RelationalExpression expression = RelationalExpression.of(lhs, EXISTS, rhs); when(lhsParser.visit(any(JsonIdentifierExpression.class))) - .thenReturn("document->'props'->>'brand'"); + .thenReturn("\"customAttribute\"->>'brand'"); String result = parser.parse(expression, context); assertEquals( - "document->'props'->>'brand' IS NOT NULL", + "\"customAttribute\" ? 'brand'", result, - "EXISTS with RHS=true on JSON scalar should check IS NOT NULL"); + "EXISTS with RHS=true on JSON scalar should use ? operator for GIN index"); + } + + @Test + void testParse_jsonScalarField_rhsFalse() { + // Test EXISTS on JSON scalar (non-array) field with RHS = false + JsonIdentifierExpression lhs = + JsonIdentifierExpression.of("customAttribute", JsonFieldType.STRING, "brand"); + ConstantExpression rhs = ConstantExpression.of(false); + RelationalExpression expression = RelationalExpression.of(lhs, EXISTS, rhs); + + when(lhsParser.visit(any(JsonIdentifierExpression.class))) + .thenReturn("\"customAttribute\"->>'brand'"); + + String result = parser.parse(expression, context); + + assertEquals( + "NOT (\"customAttribute\" ? 'brand')", + result, + "EXISTS with RHS=false on JSON scalar should use negated ? operator"); } } diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParserTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParserTest.java index 9b4fa7fc..db1b6701 100644 --- a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParserTest.java +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresNotExistsRelationalFilterParserTest.java @@ -44,9 +44,9 @@ void testParse_arrayField_rhsFalse() { String result = parser.parse(expression, context); assertEquals( - "(\"tags\" IS NOT NULL AND cardinality(\"tags\") > 0)", + "(cardinality(\"tags\") > 0)", result, - "NOT_EXISTS with RHS=false on ARRAY should check IS NOT NULL AND cardinality > 0"); + "NOT_EXISTS with RHS=false on ARRAY should check cardinality > 0"); } @Test @@ -61,9 +61,9 @@ void testParse_arrayField_rhsTrue() { String result = parser.parse(expression, context); assertEquals( - "(\"tags\" IS NULL OR cardinality(\"tags\") = 0)", + "COALESCE(cardinality(\"tags\"), 0) = 0", result, - "NOT_EXISTS with RHS=true on ARRAY should check IS NULL OR cardinality = 0"); + "NOT_EXISTS with RHS=true on ARRAY should use COALESCE for NULL or empty check"); } @Test @@ -80,9 +80,9 @@ void testParse_jsonbArrayField_rhsFalse() { String result = parser.parse(expression, context); assertEquals( - "(document->'props'->'colors' IS NOT NULL AND jsonb_typeof(document->'props'->'colors') = 'array' AND jsonb_array_length(document->'props'->'colors') > 0)", + "(\"props\" @> '{\"colors\": []}' AND jsonb_array_length(document->'props'->'colors') > 0)", result, - "NOT_EXISTS with RHS=false on JSONB_ARRAY should check IS NOT NULL AND typeof = 'array' AND length > 0"); + "NOT_EXISTS with RHS=false on JSONB_ARRAY should use optimized GIN index containment query"); } @Test @@ -99,9 +99,9 @@ void testParse_jsonbArrayField_rhsTrue() { String result = parser.parse(expression, context); assertEquals( - "(document->'props'->'flags' IS NULL OR (jsonb_typeof(document->'props'->'flags') = 'array' AND jsonb_array_length(document->'props'->'flags') = 0))", + "COALESCE(jsonb_array_length(document->'props'->'flags'), 0) = 0", result, - "NOT_EXISTS with RHS=true on JSONB_ARRAY should check IS NULL OR (typeof = 'array' AND length = 0)"); + "NOT_EXISTS with RHS=true on JSONB_ARRAY should use COALESCE for NULL or empty arrays"); } @Test @@ -142,18 +142,37 @@ void testParse_scalarField_rhsTrue() { void testParse_jsonScalarField_rhsFalse() { // Test NOT_EXISTS on JSON scalar (non-array) field with RHS = false JsonIdentifierExpression lhs = - JsonIdentifierExpression.of("props", JsonFieldType.STRING, "brand"); + JsonIdentifierExpression.of("customAttribute", JsonFieldType.STRING, "brand"); ConstantExpression rhs = ConstantExpression.of(false); RelationalExpression expression = RelationalExpression.of(lhs, NOT_EXISTS, rhs); when(lhsParser.visit(any(JsonIdentifierExpression.class))) - .thenReturn("document->'props'->>'brand'"); + .thenReturn("\"customAttribute\"->>'brand'"); String result = parser.parse(expression, context); assertEquals( - "document->'props'->>'brand' IS NOT NULL", + "\"customAttribute\" ? 'brand'", result, - "NOT_EXISTS with RHS=false on JSON scalar should check IS NOT NULL"); + "NOT_EXISTS with RHS=false on JSON scalar should use ? operator for GIN index"); + } + + @Test + void testParse_jsonScalarField_rhsTrue() { + // Test NOT_EXISTS on JSON scalar (non-array) field with RHS = true + JsonIdentifierExpression lhs = + JsonIdentifierExpression.of("customAttribute", JsonFieldType.STRING, "brand"); + ConstantExpression rhs = ConstantExpression.of("null"); + RelationalExpression expression = RelationalExpression.of(lhs, NOT_EXISTS, rhs); + + when(lhsParser.visit(any(JsonIdentifierExpression.class))) + .thenReturn("\"customAttribute\"->>'brand'"); + + String result = parser.parse(expression, context); + + assertEquals( + "NOT (\"customAttribute\" ? 'brand')", + result, + "NOT_EXISTS with RHS=true on JSON scalar should use negated ? operator"); } } From 1ea8dee404ff79108692e9bfc4dc200f4ebf556f Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Mon, 24 Nov 2025 23:43:18 +0530 Subject: [PATCH 13/14] Fix failing test cases --- .../v1/parser/filter/PostgresFieldTypeDetectorTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetectorTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetectorTest.java index f5013416..78317b00 100644 --- a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetectorTest.java +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetectorTest.java @@ -84,7 +84,8 @@ void testVisitJsonIdentifierExpression_stringScalar_returnsScalar() { FieldCategory result = detector.visit(expr); assertNotNull(result); assertEquals( - FieldCategory.SCALAR, result, "JsonIdentifierExpression with STRING should return SCALAR"); + FieldCategory.JSONB_SCALAR, result, + "JsonIdentifierExpression with STRING should return SCALAR"); } @Test @@ -93,7 +94,7 @@ void testVisitJsonIdentifierExpression_noFieldType_returnsScalar() { FieldCategory result = detector.visit(expr); assertNotNull(result); assertEquals( - FieldCategory.SCALAR, + FieldCategory.JSONB_SCALAR, result, "JsonIdentifierExpression without field type should return SCALAR"); } From 1b2cc0a8ccc2c7bbd8f9adeabb31fc8b8e6ff08c Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Tue, 25 Nov 2025 02:58:51 +0530 Subject: [PATCH 14/14] Spotless --- .../query/v1/parser/filter/PostgresFieldTypeDetectorTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetectorTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetectorTest.java index 78317b00..69ebc1c1 100644 --- a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetectorTest.java +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresFieldTypeDetectorTest.java @@ -84,7 +84,8 @@ void testVisitJsonIdentifierExpression_stringScalar_returnsScalar() { FieldCategory result = detector.visit(expr); assertNotNull(result); assertEquals( - FieldCategory.JSONB_SCALAR, result, + FieldCategory.JSONB_SCALAR, + result, "JsonIdentifierExpression with STRING should return SCALAR"); }