Postgres Query Parser Bugfixes

document-store/src/integrationTest/resources/query/flat_boolean_array_filter_response.json

@@ -1,10 +1,17 @@
 [
   {
-    "item": "Soap",
-    "price": 10
+    "item": "Soap"
   },
   {
-    "item": "Shampoo",
-    "price": 5
+    "item": "Shampoo"
+  },
+  {
+    "item": "Shampoo"
+  },
+  {
+    "item": "Comb"
+  },
+  {
+    "item": "Soap"
   }
 ]

document-store/src/integrationTest/resources/query/flat_double_array_filter_response.json

@@ -1,6 +1,5 @@
 [
   {
-    "item": "Shampoo",
-    "price": 5
+    "item": "Shampoo"
   }
 ]

document-store/src/integrationTest/resources/query/flat_integer_array_filter_response.json

@@ -1,6 +1,14 @@
 [
   {
-    "item": "Mirror",
-    "price": 20
+    "item": "Mirror"
+  },
+  {
+    "item": "Shampoo"
+  },
+  {
+    "item": "Comb"
+  },
+  {
+    "item": "Soap"
   }
 ]

document-store/src/integrationTest/resources/query/pg_flat_collection_insert.json

@@ -1,14 +1,14 @@
 {
   "statements": [
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n1, 'Soap', 10, 2, '2014-03-01T08:00:00Z',\n'{\"hygiene\", \"personal-care\", \"premium\"}',\n'{\"Hygiene\", \"PersonalCare\"}',\n'{\"colors\": [\"Blue\", \"Green\"], \"brand\": \"Dettol\", \"size\": \"M\", \"seller\": {\"name\": \"Metro Chemicals Pvt. Ltd.\", \"address\": {\"city\": \"Mumbai\", \"pincode\": 400004}}}',\nNULL,\n'{1, 2, 3}',\n'{4.5, 9.2}',\n'{true, false}'\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n2, 'Mirror', 20, 1, '2014-03-01T09:00:00Z',\n'{\"home-decor\", \"reflective\", \"glass\"}',\n'{\"HomeDecor\"}',\nNULL,\nNULL,\n'{10, 20}',\nNULL,\nNULL\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n3, 'Shampoo', 5, 10, '2014-03-15T09:00:00Z',\n'{\"hair-care\", \"personal-care\", \"premium\", \"herbal\"}',\n'{\"HairCare\", \"PersonalCare\"}',\n'{\"colors\": [\"Black\"], \"brand\": \"Sunsilk\", \"size\": \"L\", \"seller\": {\"name\": \"Metro Chemicals Pvt. Ltd.\", \"address\": {\"city\": \"Mumbai\", \"pincode\": 400004}}}',\nNULL,\nNULL,\n'{3.14, 2.71}',\nNULL\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n4, 'Shampoo', 5, 20, '2014-04-04T11:21:39.736Z',\n'{\"hair-care\", \"budget\", \"bulk\"}',\n'{\"HairCare\"}',\nNULL,\nNULL,\nNULL,\nNULL,\n'{true, true}'\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n5, 'Soap', 20, 5, '2014-04-04T21:23:13.331Z',\n'{\"hygiene\", \"antibacterial\", \"family-pack\"}',\n'{\"Hygiene\"}',\n'{\"colors\": [\"Orange\", \"Blue\"], \"brand\": \"Lifebuoy\", \"size\": \"S\", \"seller\": {\"name\": \"Hans and Co.\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\nNULL,\nNULL,\nNULL\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n6, 'Comb', 7.5, 5, '2015-06-04T05:08:13Z',\n'{\"grooming\", \"plastic\", \"essential\"}',\n'{\"Grooming\"}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n7, 'Comb', 7.5, 10, '2015-09-10T08:43:00Z',\n'{\"grooming\", \"bulk\", \"wholesale\"}',\n'{\"Grooming\"}',\n'{\"colors\": [], \"seller\": {\"name\": \"Go Go Plastics\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\nNULL,\nNULL,\nNULL\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n8, 'Soap', 10, 5, '2016-02-06T20:20:13Z',\n'{\"hygiene\", \"budget\", \"basic\"}',\n'{\"Hygiene\"}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n9, 'Bottle', 15, 3, '2016-03-01T10:00:00Z',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)",
-    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n10, 'Cup', 8, 2, '2016-04-01T10:00:00Z',\n'{}',\n'{}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)"
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n1, 'Soap', 10, 2, '2014-03-01T08:00:00Z', true,\n'{\"hygiene\", \"personal-care\", \"premium\"}',\n'{\"Hygiene\", \"PersonalCare\"}',\n'{\"colors\": [\"Blue\", \"Green\"], \"brand\": \"Dettol\", \"size\": \"M\", \"product-code\": \"SOAP-DET-001\", \"source-loc\": [\"warehouse-A\", \"store-1\"], \"seller\": {\"name\": \"Metro Chemicals Pvt. Ltd.\", \"address\": {\"city\": \"Mumbai\", \"pincode\": 400004}}}',\nNULL,\n'{1, 2, 3}',\n'{4.5, 9.2}',\n'{true, false}'\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n2, 'Mirror', 20, 1, '2014-03-01T09:00:00Z', true,\n'{\"home-decor\", \"reflective\", \"glass\"}',\n'{\"HomeDecor\"}',\nNULL,\nNULL,\n'{10, 20}',\n'{1.5, 2.5, 3.5}',\n'{false, false}'\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n3, 'Shampoo', 5, 10, '2014-03-15T09:00:00Z', true,\n'{\"hair-care\", \"personal-care\", \"premium\", \"herbal\"}',\n'{\"HairCare\", \"PersonalCare\"}',\n'{\"colors\": [\"Black\"], \"brand\": \"Sunsilk\", \"size\": \"L\", \"product-code\": \"SHAMP-SUN-003\", \"source-loc\": [\"warehouse-B\", \"store-2\", \"online\"], \"seller\": {\"name\": \"Metro Chemicals Pvt. Ltd.\", \"address\": {\"city\": \"Mumbai\", \"pincode\": 400004}}}',\nNULL,\n'{5, 10, 15}',\n'{3.14, 2.71}',\n'{true, false, true}'\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n4, 'Shampoo', 5, 20, '2014-04-04T11:21:39.736Z', false,\n'{\"hair-care\", \"budget\", \"bulk\"}',\n'{\"HairCare\"}',\nNULL,\nNULL,\n'{1, 2}',\n'{5.0, 10.0}',\n'{true, true}'\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n5, 'Soap', 20, 5, '2014-04-04T21:23:13.331Z', true,\n'{\"hygiene\", \"antibacterial\", \"family-pack\"}',\n'{\"Hygiene\"}',\n'{\"colors\": [\"Orange\", \"Blue\"], \"brand\": \"Lifebuoy\", \"size\": \"S\", \"product-code\": \"SOAP-LIF-005\", \"source-loc\": [\"warehouse-C\"], \"seller\": {\"name\": \"Hans and Co.\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\n'{3, 6, 9}',\n'{7.5}',\n'{false}'\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n6, 'Comb', 7.5, 5, '2015-06-04T05:08:13Z', true,\n'{\"grooming\", \"plastic\", \"essential\"}',\n'{\"Grooming\"}',\nNULL,\nNULL,\n'{20, 30}',\n'{6.0, 8.0}',\n'{true, false}'\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n7, 'Comb', 7.5, 10, '2015-09-10T08:43:00Z', false,\n'{\"grooming\", \"bulk\", \"wholesale\"}',\n'{\"Grooming\"}',\n'{\"colors\": [], \"product-code\": null, \"source-loc\": [], \"seller\": {\"name\": \"Go Go Plastics\", \"address\": {\"city\": \"Kolkata\", \"pincode\": 700007}}}',\nNULL,\n'{10}',\n'{3.0}',\n'{false, false, false}'\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n8, 'Soap', 10, 5, '2016-02-06T20:20:13Z', true,\n'{\"hygiene\", \"budget\", \"basic\"}',\n'{\"Hygiene\"}',\nNULL,\nNULL,\n'{1, 10, 20}',\n'{2.5, 5.0}',\n'{true}'\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n9, 'Bottle', 15, 3, '2016-03-01T10:00:00Z', false,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)",
+    "INSERT INTO \"myTestFlat\" (\n\"_id\", \"item\", \"price\", \"quantity\", \"date\", \"in_stock\", \"tags\", \"categoryTags\", \"props\", \"sales\", \"numbers\", \"scores\", \"flags\"\n) VALUES (\n10, 'Cup', 8, 2, '2016-04-01T10:00:00Z', true,\n'{}',\n'{}',\nNULL,\nNULL,\nNULL,\nNULL,\nNULL\n)"
   ]
 }

document-store/src/main/java/org/hypertrace/core/documentstore/postgres/PostgresCollection.java

@@ -1395,9 +1395,9 @@ private void addColumnToJsonNode(
           break;
 
         case "_text": // text array
-          Array array = resultSet.getArray(columnIndex);
-          if (array != null) {
-            String[] stringArray = (String[]) array.getArray();
+          Array textArray = resultSet.getArray(columnIndex);
+          if (textArray != null) {
+            String[] stringArray = (String[]) textArray.getArray();
             ArrayNode arrayNode = MAPPER.createArrayNode();
             for (String item : stringArray) {
               arrayNode.add(item);
@@ -1406,6 +1406,52 @@ private void addColumnToJsonNode(
           }
           break;
 
+        case "_int4": // integer array
+        case "_int8": // bigint array
+          Array intArray = resultSet.getArray(columnIndex);
+          if (intArray != null) {
+            Object[] intObjectArray = (Object[]) intArray.getArray();
+            ArrayNode intArrayNode = MAPPER.createArrayNode();
+            for (Object item : intObjectArray) {
+              if (item instanceof Integer) {
+                intArrayNode.add((Integer) item);
+              } else if (item instanceof Long) {
+                intArrayNode.add((Long) item);
+              }
+            }
+            jsonNode.set(columnName, intArrayNode);
+          }
+          break;
+
+        case "_float8": // double precision array
+        case "_float4": // real/float array
+          Array doubleArray = resultSet.getArray(columnIndex);
+          if (doubleArray != null) {
+            Object[] doubleObjectArray = (Object[]) doubleArray.getArray();
+            ArrayNode doubleArrayNode = MAPPER.createArrayNode();
+            for (Object item : doubleObjectArray) {
+              if (item instanceof Double) {
+                doubleArrayNode.add((Double) item);
+              } else if (item instanceof Float) {
+                doubleArrayNode.add((Float) item);
+              }
+            }
+            jsonNode.set(columnName, doubleArrayNode);
+          }
+          break;
+
+        case "_bool": // boolean array
+          Array boolArray = resultSet.getArray(columnIndex);
+          if (boolArray != null) {
+            Boolean[] boolObjectArray = (Boolean[]) boolArray.getArray();
+            ArrayNode boolArrayNode = MAPPER.createArrayNode();
+            for (Boolean item : boolObjectArray) {
+              boolArrayNode.add(item);
+            }
+            jsonNode.set(columnName, boolArrayNode);
+          }
+          break;
+
         case "jsonb":
         case "json":
           String jsonString = resultSet.getString(columnIndex);

document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresExistsRelationalFilterParser.java

@@ -1,5 +1,6 @@
 package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter;
 
+import org.hypertrace.core.documentstore.expression.impl.ArrayIdentifierExpression;
 import org.hypertrace.core.documentstore.expression.impl.ConstantExpression;
 import org.hypertrace.core.documentstore.expression.impl.JsonIdentifierExpression;
 import org.hypertrace.core.documentstore.expression.impl.RelationalExpression;
@@ -25,16 +26,37 @@ public String parse(
 
     switch (category) {
       case ARRAY:
-        // First-class PostgreSQL array columns (text[], int[], etc.)
-        return parsedRhs
-            // We don't need to check that LHS is NOT NULL because WHERE cardinality(NULL) will not
-            // be included in the result set
-            ? String.format("(cardinality(%s) > 0)", parsedLhs)
-            : String.format("COALESCE(cardinality(%s), 0) = 0", parsedLhs);
+        {
+          // First-class PostgreSQL array columns (text[], int[], etc.)
+          // Check if this field has been unnested - if so, treat it as a scalar (because the
+          // unnested array col is not longer an array, but a scalar col)
+          ArrayIdentifierExpression arrayExpr = (ArrayIdentifierExpression) expression.getLhs();
+          String arrayFieldName = arrayExpr.getName();
+          if (context.getPgColumnNames().containsKey(arrayFieldName)) {
+            // Field is unnested - each element is now a scalar, not an array
+            // Use simple NULL checks instead of cardinality
+            return getScalarExpr(parsedRhs, parsedLhs);
+          }
+
+          // Field is NOT unnested - apply cardinality logic
+          return parsedRhs
+              // We don't need to check that LHS is NOT NULL because WHERE cardinality(NULL) will
+              // not be included in the result set
+              ? String.format("(cardinality(%s) > 0)", parsedLhs)
+              : String.format("COALESCE(cardinality(%s), 0) = 0", parsedLhs);
+        }
 
       case JSONB_ARRAY:
         {
           JsonIdentifierExpression jsonExpr = (JsonIdentifierExpression) expression.getLhs();
+          // Check if this field has been unnested - if so, treat it as a scalar
+          String fieldName = jsonExpr.getName();
+          if (context.getPgColumnNames().containsKey(fieldName)) {
+            // Field is unnested - each element is now a scalar. Treat how we treated the array case
+            return getScalarExpr(parsedRhs, parsedLhs);
+          }
+
+          // Field is NOT unnested - apply array length logic
           String baseColumn = wrapWithDoubleQuotes(jsonExpr.getColumnName());
           String nestedPath = String.join(".", jsonExpr.getJsonPath());
           return parsedRhs
@@ -49,26 +71,18 @@ public String parse(
         }
 
       case JSONB_SCALAR:
-        {
-          // JSONB scalar fields - use ? operator for GIN index optimization
-          JsonIdentifierExpression jsonExpr = (JsonIdentifierExpression) expression.getLhs();
-          String baseColumn = wrapWithDoubleQuotes(jsonExpr.getColumnName());
-          String nestedPath = String.join(".", jsonExpr.getJsonPath());
-
-          return parsedRhs
-              ? String.format("%s ? '%s'", baseColumn, nestedPath)
-              : String.format("NOT (%s ? '%s')", baseColumn, nestedPath);
-        }
-
       case SCALAR:
       default:
-        // Regular scalar fields - use standard NULL checks
-        return parsedRhs
-            ? String.format("%s IS NOT NULL", parsedLhs)
-            : String.format("%s IS NULL", parsedLhs);
+        return getScalarExpr(parsedRhs, parsedLhs);
     }
   }
 
+  private String getScalarExpr(boolean parsedRhs, String parsedLhs) {
+    return parsedRhs
+        ? String.format("%s IS NOT NULL", parsedLhs)
+        : String.format("%s IS NULL", parsedLhs);
+  }
+
   private String wrapWithDoubleQuotes(String identifier) {
     return "\"" + identifier + "\"";
   }

document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonArray.java

@@ -22,6 +22,9 @@
  *
  * <p>This checks if the JSON array contains ANY of the provided values, using efficient JSONB
  * containment instead of defensive type checking.
+ *
+ * <p>Special case: If the JSONB array field has been unnested, each row contains a scalar value
+ * (not an array), so we use scalar IN syntax instead of the @> containment operator.
  */
 public class PostgresInRelationalFilterParserJsonArray
     implements PostgresInRelationalFilterParserInterface {
@@ -42,11 +45,74 @@ public String parse(
                     new IllegalStateException(
                         "JsonFieldType must be present - this should have been caught by the selector"));
 
-    return prepareFilterStringForInOperator(
+    // Check if this field has been unnested - if so, treat it as a scalar
+    String fieldName = jsonExpr.getName();
+    if (context.getPgColumnNames().containsKey(fieldName)) {
+      // Field is unnested - each element is now a scalar, not an array
+      // Use scalar IN operator instead of JSONB containment
+      return prepareFilterStringForScalarInOperator(
+          parsedLhs, parsedRhs, context.getParamsBuilder());
+    }
+
+    // Field is NOT unnested - use JSONB containment logic
+    return prepareFilterStringForArrayInOperator(
         parsedLhs, parsedRhs, fieldType, context.getParamsBuilder());
   }
 
-  private String prepareFilterStringForInOperator(
+  /**
+   * Generates SQL for scalar IN operator (used when JSONB array field has been unnested). Example:
+   * "props_dot_source-loc" IN (?::jsonb, ?::jsonb)
+   *
+   * <p>Note: After unnesting with jsonb_array_elements(), each row contains a JSONB scalar value.
+   * We cast the parameters to jsonb for direct JSONB-to-JSONB comparison, which works for all JSONB
+   * types (strings, numbers, booleans, objects).
+   */
+  private String prepareFilterStringForScalarInOperator(
+      final String parsedLhs,
+      final Iterable<Object> parsedRhs,
+      final Params.Builder paramsBuilder) {
+
+    String placeholders =
+        StreamSupport.stream(parsedRhs.spliterator(), false)
+            .map(
+                value -> {
+                  // Add the value as a JSONB-formatted string
+                  // For strings, this needs to be JSON-quoted (e.g., "warehouse-A" becomes
+                  // "\"warehouse-A\"")
+                  String jsonValue = convertToJsonString(value);
+                  paramsBuilder.addObjectParam(jsonValue);
+                  return "?::jsonb";
+                })
+            .collect(Collectors.joining(", "));
+
+    // Direct JSONB comparison - no text conversion needed
+    return String.format("%s IN (%s)", parsedLhs, placeholders);
+  }
+
+  /**
+   * Converts a Java value to its JSON string representation for JSONB casting. Strings are quoted,
+   * numbers/booleans are not.
+   */
+  private String convertToJsonString(Object value) {
+    if (value == null) {
+      return "null";
+    } else if (value instanceof String) {
+      // JSON strings must be quoted
+      return "\"" + value.toString().replace("\"", "\\\"") + "\"";
+    } else if (value instanceof Number || value instanceof Boolean) {
+      // Numbers and booleans are not quoted in JSON
+      return value.toString();
+    } else {
+      // For other types, assume they're already JSON-formatted or treat as string
+      return "\"" + value.toString().replace("\"", "\\\"") + "\"";
+    }
+  }
+
+  /**
+   * Generates SQL for JSONB containment operator (used for non-unnested JSONB array fields).
+   * Example: document->'tags' @> jsonb_build_array(?::text)
+   */
+  private String prepareFilterStringForArrayInOperator(
       final String parsedLhs,
       final Iterable<Object> parsedRhs,
       final JsonFieldType fieldType,