Add missing filters to RecordsQuery (#578)

* add published and dataCid to RecordQuery filters

* datePublished filter for RecordsQuery

* dateUpdated filter for RecordsQuery

* non-owner and anonymous case for published filters

* additional test cases

* dissallow both published to be set to false and a publishedDate filter to be present

* add comments and move tests, add additional edge tests to existing

* Apply suggestions from code review

Co-authored-by: Henry Tsai <henrytsai@outlook.com>

* update unauthorized tests

* clear up confusion around published/unpublished querying

* additional comment

* address review comments

---------

Co-authored-by: Henry Tsai <henrytsai@outlook.com>

json-schemas/interface-methods/records-filter.json

@@ -29,12 +29,18 @@
     "parentId": {
       "type": "string"
     },
+    "published": {
+      "type": "boolean"
+    },
     "dataFormat": {
       "type": "string"
     },
     "dataSize": {
       "$ref": "https://identity.foundation/dwn/json-schemas/number-range-filter.json"
     },
+    "dataCid": {
+      "type": "string"
+    },
     "dateCreated": {
       "type": "object",
       "minProperties": 1,
@@ -47,6 +53,51 @@
           "$ref": "https://identity.foundation/dwn/json-schemas/defs.json#/definitions/date-time"
         }
       }
+    },
+    "datePublished": {
+      "type": "object",
+      "minProperties": 1,
+      "additionalProperties": false,
+      "properties": {
+        "from": {
+          "$ref": "https://identity.foundation/dwn/json-schemas/defs.json#/definitions/date-time"
+        },
+        "to": {
+          "$ref": "https://identity.foundation/dwn/json-schemas/defs.json#/definitions/date-time"
+        }
+      }
+    },
+    "dateUpdated": {
+      "type": "object",
+      "minProperties": 1,
+      "additionalProperties": false,
+      "properties": {
+        "from": {
+          "$ref": "https://identity.foundation/dwn/json-schemas/defs.json#/definitions/date-time"
+        },
+        "to": {
+          "$ref": "https://identity.foundation/dwn/json-schemas/defs.json#/definitions/date-time"
+        }
+      }
+    }
+  },
+  "dependencies": {
+    "datePublished": {
+      "oneOf": [
+        {
+          "properties": {
+            "published": {
+              "enum": [true]
+            }
+          },
+          "required": ["published"]
+        },
+        {
+          "not": {
+            "required": ["published"]
+          }
+        }
+      ]
     }
   }
 }

src/handlers/records-query.ts

@@ -30,8 +30,8 @@ export class RecordsQueryHandler implements MethodHandler {
 
     let recordsWrites: RecordsWriteMessageWithOptionalEncodedData[];
     let paginationMessageCid: string|undefined;
-    // if this is an anonymous query, query only published records
-    if (recordsQuery.author === undefined) {
+    // if this is an anonymous query and the filter supports published records, query only published records
+    if (RecordsQueryHandler.filterIncludesPublishedRecords(recordsQuery) && recordsQuery.author === undefined) {
       const results = await this.fetchPublishedRecords(tenant, recordsQuery);
       recordsWrites = results.messages as RecordsWriteMessageWithOptionalEncodedData[];
       paginationMessageCid = results.paginationMessageCid;
@@ -110,27 +110,45 @@ export class RecordsQueryHandler implements MethodHandler {
   }
 
   /**
-   * Fetches the records as a non-owner, return only:
-   * 1. published records; and
-   * 2. unpublished records intended for the query author (where `recipient` is the query author)
+   * Fetches the records as a non-owner.
+   *
+   * Filters can support returning both published and unpublished records,
+   * as well as explicitly only published or only unpublished records.
+   *
+   * A) BOTH published and unpublished:
+   *    1. published records; and
+   *    2. unpublished records intended for the query author (where `recipient` is the query author); and
+   *    3. unpublished records authorized by a protocol rule.
+   *
+   * B) PUBLISHED:
+   *    1. only published records;
+   *
+   * C) UNPUBLISHED:
+   *    1. unpublished records intended for the query author (where `recipient` is the query author); and
+   *    2. unpublished records authorized by a protocol rule.
+   *
    */
   private async fetchRecordsAsNonOwner(
     tenant: string, recordsQuery: RecordsQuery
   ): Promise<{ messages: GenericMessage[], paginationMessageCid?: string }> {
     const { dateSort, pagination } = recordsQuery.message.descriptor;
+    const filters = [];
 
-    const filters = [
-      RecordsQueryHandler.buildPublishedRecordsFilter(recordsQuery),
-      RecordsQueryHandler.buildUnpublishedRecordsByQueryAuthorFilter(recordsQuery),
-    ];
-
-    const recipientFilter = recordsQuery.message.descriptor.filter.recipient;
-    if (recipientFilter === undefined || recipientFilter === recordsQuery.author) {
-      filters.push(RecordsQueryHandler.buildUnpublishedRecordsForQueryAuthorFilter(recordsQuery));
+    if (RecordsQueryHandler.filterIncludesPublishedRecords(recordsQuery)) {
+      filters.push(RecordsQueryHandler.buildPublishedRecordsFilter(recordsQuery));
     }
 
-    if (RecordsQueryHandler.shouldProtocolAuthorizeQuery(recordsQuery)) {
-      filters.push(RecordsQueryHandler.buildUnpublishedProtocolAuthorizedRecordsFilter(recordsQuery));
+    if (RecordsQueryHandler.filterIncludesUnpublishedRecords(recordsQuery)) {
+      filters.push(RecordsQueryHandler.buildUnpublishedRecordsByQueryAuthorFilter(recordsQuery));
+
+      const recipientFilter = recordsQuery.message.descriptor.filter.recipient;
+      if (recipientFilter === undefined || recipientFilter === recordsQuery.author) {
+        filters.push(RecordsQueryHandler.buildUnpublishedRecordsForQueryAuthorFilter(recordsQuery));
+      }
+
+      if (RecordsQueryHandler.shouldProtocolAuthorizeQuery(recordsQuery)) {
+        filters.push(RecordsQueryHandler.buildUnpublishedProtocolAuthorizedRecordsFilter(recordsQuery));
+      }
     }
 
     const messageSort = this.convertDateSort(dateSort);
@@ -210,4 +228,25 @@ export class RecordsQueryHandler implements MethodHandler {
   private static shouldProtocolAuthorizeQuery(recordsQuery: RecordsQuery): boolean {
     return recordsQuery.signerSignaturePayload!.protocolRole !== undefined;
   }
+
+  /**
+   * Checks if the recordQuery filter supports returning published records.
+   */
+  private static filterIncludesPublishedRecords(recordsQuery: RecordsQuery): boolean {
+    const { filter } = recordsQuery.message.descriptor;
+    // When `published` and `datePublished` range are both undefined, published records can be returned.
+    return filter.datePublished !== undefined || filter.published !== false;
+  }
+
+  /**
+   * Checks if the recordQuery filter supports returning unpublished records.
+   */
+  private static filterIncludesUnpublishedRecords(recordsQuery: RecordsQuery): boolean {
+    const { filter } = recordsQuery.message.descriptor;
+    // When `published` and `datePublished` range are both undefined, unpublished records can be returned.
+    if (filter.datePublished === undefined && filter.published === undefined) {
+      return true;
+    }
+    return filter.published === false;
+  }
 }

src/types/records-types.ts

@@ -96,13 +96,17 @@ export type RecordsFilter = {
   recipient?: string;
   protocol?: string;
   protocolPath?: string;
+  published?: boolean;
   contextId?: string;
   schema?: string;
   recordId?: string;
   parentId?: string;
   dataFormat?: string;
   dataSize?: RangeFilter;
+  dataCid?: string;
   dateCreated?: RangeCriterion;
+  datePublished?: RangeCriterion;
+  dateUpdated?: RangeCriterion;
 };
 
 export type RangeCriterion = {

src/utils/records.ts

@@ -1,7 +1,7 @@
 import type { DerivedPrivateJwk } from './hd-key.js';
 import type { Readable } from 'readable-stream';
 import type { Filter, RangeFilter } from '../types/message-types.js';
-import type { RecordsFilter, RecordsWriteDescriptor, RecordsWriteMessage } from '../types/records-types.js';
+import type { RangeCriterion, RecordsFilter, RecordsWriteDescriptor, RecordsWriteMessage } from '../types/records-types.js';
 
 import { Encoder } from './encoder.js';
 import { Encryption } from './encryption.js';
@@ -247,31 +247,45 @@ export class Records {
    * @returns {Filter} a generic Filter able to be used with MessageStore.
    */
   public static convertFilter(filter: RecordsFilter): Filter {
-    const filterCopy = { ...filter };
-    const { dateCreated } = filterCopy;
-
-    let rangeFilter: RangeFilter | undefined = undefined;
-    if (dateCreated !== undefined) {
-      if (dateCreated.to !== undefined && dateCreated.from !== undefined) {
-        rangeFilter = {
-          gte : dateCreated.from,
-          lt  : dateCreated.to,
-        };
-      } else if (dateCreated.to !== undefined) {
-        rangeFilter = {
-          lt: dateCreated.to,
-        };
-      } else if (dateCreated.from !== undefined) {
-        rangeFilter = {
-          gte: dateCreated.from,
-        };
-      }
+    const filterCopy = { ...filter } as Filter;
+
+    const { dateCreated, datePublished, dateUpdated } = filter;
+    const dateCreatedFilter = dateCreated ? this.convertRangeCriterion(dateCreated) : undefined;
+    if (dateCreatedFilter) {
+      filterCopy.dateCreated = dateCreatedFilter;
     }
 
-    if (rangeFilter) {
-      (filterCopy as Filter).dateCreated = rangeFilter;
+    const datePublishedFilter = datePublished ? this.convertRangeCriterion(datePublished): undefined;
+    if (datePublishedFilter) {
+      // only return published records when filtering with a datePublished range.
+      filterCopy.published = true;
+      filterCopy.datePublished = datePublishedFilter;
     }
 
+    const messageTimestampFilter = dateUpdated ? this.convertRangeCriterion(dateUpdated) : undefined;
+    if (messageTimestampFilter) {
+      filterCopy.messageTimestamp = messageTimestampFilter;
+      delete filterCopy.dateUpdated;
+    }
     return filterCopy as Filter;
   }
+
+  private static convertRangeCriterion(inputFilter: RangeCriterion): RangeFilter | undefined {
+    let rangeFilter: RangeFilter | undefined;
+    if (inputFilter.to !== undefined && inputFilter.from !== undefined) {
+      rangeFilter = {
+        gte : inputFilter.from,
+        lt  : inputFilter.to,
+      };
+    } else if (inputFilter.to !== undefined) {
+      rangeFilter = {
+        lt: inputFilter.to,
+      };
+    } else if (inputFilter.from !== undefined) {
+      rangeFilter = {
+        gte: inputFilter.from,
+      };
+    }
+    return rangeFilter;
+  }
 }