Allow sources to be annotated with traverse flag

packages/actor-context-preprocess-convert-shortcuts/lib/ActorContextPreprocessConvertShortcuts.ts

@@ -65,7 +65,8 @@ export interface IActorContextPreprocessConvertShortcutsArgs extends IActorConte
    *   "extensionFunctionCreator": "@comunica/actor-init-query:extensionFunctionCreator",
    *   "functionArgumentsCache": "@comunica/actor-init-query:functionArgumentsCache",
    *   "explain": "@comunica/actor-init-query:explain",
-   *   "unionDefaultGraph": "@comunica/bus-query-operation:unionDefaultGraph"
+   *   "unionDefaultGraph": "@comunica/bus-query-operation:unionDefaultGraph",
+   *   "traverse": "@comunica/bus-query-source-identify:traverse"
    * }}
    */
   contextKeyShortcuts: Record<string, string>;

packages/actor-optimize-query-operation-prune-empty-source-operations/lib/ActorOptimizeQueryOperationPruneEmptySourceOperations.ts

@@ -2,6 +2,7 @@ import type { IActionOptimizeQueryOperation,
   IActorOptimizeQueryOperationOutput, IActorOptimizeQueryOperationArgs } from '@comunica/bus-optimize-query-operation';
 import { ActorOptimizeQueryOperation } from '@comunica/bus-optimize-query-operation';
 import { ActorQueryOperation } from '@comunica/bus-query-operation';
+import { KeysQuerySourceIdentify } from '@comunica/context-entries';
 import type { IActorTest } from '@comunica/core';
 import type { IActionContext, IQuerySourceWrapper, MetadataBindings } from '@comunica/types';
 import { DataFactory } from 'rdf-data-factory';
@@ -128,6 +129,11 @@ export class ActorOptimizeQueryOperationPruneEmptySourceOperations extends Actor
     input: Algebra.Operation,
     context: IActionContext,
   ): Promise<boolean> {
+    // Traversal sources should never be considered empty at optimization time.
+    if (source.context?.get(KeysQuerySourceIdentify.traverse)) {
+      return true;
+    }
+
     // Send an ASK query
     if (this.useAskIfSupported) {
       const askOperation = AF.createAsk(input);

packages/actor-optimize-query-operation-prune-empty-source-operations/package.json

@@ -34,6 +34,7 @@
   "dependencies": {
     "@comunica/bus-optimize-query-operation": "^3.0.0",
     "@comunica/bus-query-operation": "^3.0.0",
+    "@comunica/context-entries": "^3.0.0",
     "@comunica/core": "^3.0.0",
     "@comunica/types": "^3.0.0",
     "rdf-data-factory": "^1.1.2",

packages/actor-optimize-query-operation-prune-empty-source-operations/test/ActorOptimizeQueryOperationPruneEmptySourceOperations-test.ts

@@ -1,4 +1,5 @@
 import { ActorQueryOperation } from '@comunica/bus-query-operation';
+import { KeysQuerySourceIdentify } from '@comunica/context-entries';
 import { ActionContext, Bus } from '@comunica/core';
 import type { IQuerySourceWrapper } from '@comunica/types';
 import { ArrayIterator } from 'asynciterator';
@@ -525,6 +526,16 @@ describe('ActorOptimizeQueryOperationPruneEmptySourceOperations', () => {
           await actor.hasSourceResults(source1, AF.createNop(), ctx);
           expect(source1.source.queryBindings).toBeCalledWith(AF.createNop(), ctx);
         });
+
+        it('should be true for cardinality = 0 on a traversal source', async() => {
+          source1.context = new ActionContext().set(KeysQuerySourceIdentify.traverse, true);
+          source1.source.queryBindings = () => {
+            const bindingsStream = new ArrayIterator([], { autoStart: false });
+            bindingsStream.setProperty('metadata', { cardinality: { value: 0 }});
+            return bindingsStream;
+          };
+          expect(await actor.hasSourceResults(source1, AF.createNop(), ctx)).toBeTruthy();
+        });
       });
 
       describe('for ask true', () => {

packages/actor-query-source-identify-hypermedia/README.md

@@ -45,7 +45,7 @@ After installing, this package can be added to your engine's configuration as fo
 
 * `cacheSize`: The maximum number of entries in the LRU cache, set to 0 to disable, defaults to 100.
 * `maxIterators`: The maximum number of links that can be followed in parallel, defaults to 64.
-* `aggregateStore`: If all discovered quads across all links from a seed source should be indexed in an aggregated store, to speed up later calls. This should only be used for sources without filter factor. Defaults to false.
+* `aggregateTraversalStore`: If all discovered quads across all links from a traversal source should be indexed in an aggregated store, to speed up later calls. This only applies to sources annotated with KeysQuerySourceIdentify.traverse. Defaults to true.
 * `httpInvalidator`: An optional actor that listens to HTTP invalidation events.
 * `mediatorDereferenceRdf`: A mediator over the [RDF Dereference bus](https://github.com/comunica/comunica/tree/master/packages/bus-dereference-rdf).
 * `mediatorMetadata`: A mediator over the [RDF Metadata bus](https://github.com/comunica/comunica/tree/master/packages/bus-rdf-metadata).

packages/actor-query-source-identify-hypermedia/lib/ActorQuerySourceIdentifyHypermedia.ts

@@ -10,6 +10,7 @@ import type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-a
 import type { MediatorRdfMetadataExtract } from '@comunica/bus-rdf-metadata-extract';
 import type { MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links';
 import type { MediatorRdfResolveHypermediaLinksQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue';
+import { KeysQuerySourceIdentify } from '@comunica/context-entries';
 import { ActionContext } from '@comunica/core';
 import type { IActorTest } from '@comunica/core';
 import { QuerySourceHypermedia } from './QuerySourceHypermedia';
@@ -28,7 +29,7 @@ export class ActorQuerySourceIdentifyHypermedia extends ActorQuerySourceIdentify
   public readonly mediatorMergeBindingsContext: MediatorMergeBindingsContext;
   public readonly cacheSize: number;
   public readonly maxIterators: number;
-  public readonly aggregateStore: boolean;
+  public readonly aggregateTraversalStore: boolean;
 
   public constructor(args: IActorQuerySourceIdentifyHypermediaArgs) {
     super(args);
@@ -49,7 +50,8 @@ export class ActorQuerySourceIdentifyHypermedia extends ActorQuerySourceIdentify
           <string> action.querySourceUnidentified.value,
           action.querySourceUnidentified.type,
           this.maxIterators,
-          this.aggregateStore,
+          this.aggregateTraversalStore &&
+          Boolean(action.querySourceUnidentified.context?.get(KeysQuerySourceIdentify.traverse)),
           {
             mediatorMetadata: this.mediatorMetadata,
             mediatorMetadataExtract: this.mediatorMetadataExtract,
@@ -81,12 +83,12 @@ export interface IActorQuerySourceIdentifyHypermediaArgs extends IActorQuerySour
    */
   maxIterators: number;
   /**
-   * If all discovered quads across all links from a seed source should be indexed in an aggregated store,
+   * If all discovered quads across all links from a traversal source should be indexed in an aggregated store,
    * to speed up later calls.
-   * This should only be used for sources without filter factor.
-   * @default {false}
+   * This only applies to sources annotated with KeysQuerySourceIdentify.traverse.
+   * @default {true}
    */
-  aggregateStore: boolean;
+  aggregateTraversalStore: boolean;
   /**
    * The RDF dereference mediator
    */

packages/actor-query-source-identify-hypermedia/test/ActorQuerySourceIdentifyHypermedia-test.ts

@@ -78,7 +78,7 @@ describe('ActorQuerySourceIdentifyHypermedia', () => {
           bus,
           cacheSize: 10,
           maxIterators: 64,
-          aggregateStore: false,
+          aggregateTraversalStore: false,
           mediatorMetadata,
           mediatorMetadataExtract,
           mediatorMetadataAccumulate,
@@ -199,7 +199,7 @@ describe('ActorQuerySourceIdentifyHypermedia', () => {
             bus,
             cacheSize: 10,
             maxIterators: 64,
-            aggregateStore: false,
+            aggregateTraversalStore: false,
             mediatorMetadata,
             mediatorMetadataExtract: mediatorMetadataExtractThis,
             mediatorMetadataAccumulate,
@@ -232,7 +232,7 @@ describe('ActorQuerySourceIdentifyHypermedia', () => {
             bus,
             cacheSize: 10,
             maxIterators: 64,
-            aggregateStore: false,
+            aggregateTraversalStore: false,
             mediatorMetadata,
             mediatorMetadataExtract: mediatorMetadataExtractThis,
             mediatorMetadataAccumulate,
@@ -263,7 +263,7 @@ describe('ActorQuerySourceIdentifyHypermedia', () => {
       });
     });
 
-    describe('An ActorQuerySourceIdentifyHypermedia instance with aggregateStore enabled', () => {
+    describe('An ActorQuerySourceIdentifyHypermedia instance with aggregateTraversalStore on a traverse source', () => {
       let actor: ActorQuerySourceIdentifyHypermedia;
       let context: IActionContext;
       let operation: Algebra.Operation;
@@ -274,7 +274,7 @@ describe('ActorQuerySourceIdentifyHypermedia', () => {
           bus,
           cacheSize: 10,
           maxIterators: 64,
-          aggregateStore: true,
+          aggregateTraversalStore: true,
           mediatorMetadata,
           mediatorMetadataExtract,
           mediatorMetadataAccumulate,
@@ -286,7 +286,10 @@ describe('ActorQuerySourceIdentifyHypermedia', () => {
           name: 'actor',
         });
         operation = <any> {};
-        querySourceUnidentified = { value: 'firstUrl' };
+        querySourceUnidentified = {
+          value: 'firstUrl',
+          context: new ActionContext().set(KeysQuerySourceIdentify.traverse, true),
+        };
       });
 
       describe('run without hypermediaSourcesAggregatedStores', () => {

packages/context-entries/lib/Keys.ts

@@ -260,6 +260,11 @@ export const KeysQuerySourceIdentify = {
   hypermediaSourcesAggregatedStores: new ActionContextKey<Map<string, IAggregatedStore>>(
     '@comunica/bus-query-source-identify:hypermediaSourcesAggregatedStores',
   ),
+  /**
+   * If links may be traversed from this source.
+   * This means that sources annotated with this flag are considered incomplete until all links have been traversed.
+   */
+  traverse: new ActionContextKey<boolean>('@comunica/bus-query-source-identify:traverse'),
 };
 
 export const KeysRdfUpdateQuads = {