From f9065487cd94ecff6b9406833a18d313d766ca6f Mon Sep 17 00:00:00 2001 From: Ruben Taelman Date: Mon, 22 May 2023 12:56:34 +0200 Subject: [PATCH] Enable incremental metadata updates This refactors internal metadata handling so that it can be incrementally updated. Metadata objects now have a `state` field, which represents their validation state, which can be invalidated, and be listened to. This allows actors to listen to metadata invalidations and act upon changes. This commit also adds a metadata accumulation bus that abstracts the logic for merging metadata from multiple sources that existed before in the federated actor. This will enable this logic to be used in other places, and for other actors to manage how metadata is merged. Closes #1156 Closes #1180 May be related to comunica/comunica-feature-link-traversal#102 --- .componentsjs-generator-config.json | 1 + .../config/config-default.json | 2 + .../config/config-rdfjs.json | 3 + .../rdf-metadata-accumulate/actors.json | 30 ++ .../rdf-metadata-accumulate/mediators.json | 10 + .../actors/federated.json | 3 +- .../actors/hypermedia.json | 1 + engines/query-sparql-file/package.json | 4 + engines/query-sparql-rdfjs/package.json | 4 + engines/query-sparql/package.json | 4 + lerna.json | 3 +- .../test/QueryEngineBase-test.ts | 5 + ...ActorQueryOperationDescribeSubject-test.ts | 15 +- .../lib/ActorQueryOperationNop.ts | 2 + .../actor-query-operation-nop/package.json | 1 + .../test/ActorQueryOperationPathAlt-test.ts | 3 + .../lib/ActorQueryOperationPathZeroOrOne.ts | 2 + .../package.json | 1 + .../ActorQueryOperationPathZeroOrOne-test.ts | 8 + .../lib/ActorQueryOperationService.ts | 8 +- .../package.json | 1 + .../test/ActorQueryOperationService-test.ts | 2 +- .../lib/ActorQueryOperationSparqlEndpoint.ts | 2 + .../package.json | 1 + .../ActorQueryOperationSparqlEndpoint-test.ts | 12 +- .../lib/ActorQueryOperationUnion.ts | 8 + .../actor-query-operation-union/package.json | 1 + .../test/ActorQueryOperationUnion-test.ts | 140 ++++++-- .../lib/ActorQueryOperationValues.ts | 2 + .../actor-query-operation-values/package.json | 1 + .../test/ActorQueryOperationValues-test.ts | 8 +- .../test/ActorRdfJoinHash-test.ts | 17 +- .../test/ActorRdfJoinMultiBind-test.ts | 89 ++++- .../lib/ActorRdfJoinMultiEmpty.ts | 2 + .../package.json | 1 + .../test/ActorRdfJoinMultiEmpty-test.ts | 2 +- .../test/ActorRdfJoinMultiSequential-test.ts | 10 + .../test/ActorRdfJoinMultiSmallest-test.ts | 10 + .../test/ActorRdfJoinNestedLoop-test.ts | 8 + .../lib/ActorRdfJoinNone.ts | 2 + .../actor-rdf-join-inner-none/package.json | 1 + .../test/ActorRdfJoinNone-test.ts | 2 +- .../test/ActorRdfJoinSymmetricHash-test.ts | 128 ++++++-- .../test/ActorRdfJoinOptionalBind-test.ts | 21 ++ .../ActorRdfJoinOptionalNestedLoop-test.ts | 4 + .../.npmignore | 0 .../README.md | 36 +++ ...orRdfMetadataAccumulateCanContainUndefs.ts | 31 ++ .../lib/index.ts | 1 + .../package.json | 43 +++ ...MetadataAccumulateCanContainUndefs-test.ts | 67 ++++ .../.npmignore | 0 .../README.md | 36 +++ .../ActorRdfMetadataAccumulateCardinality.ts | 66 ++++ .../lib/index.ts | 1 + .../package.json | 44 +++ ...orRdfMetadataAccumulateCardinality-test.ts | 121 +++++++ .../.npmignore | 0 .../README.md | 36 +++ .../lib/ActorRdfMetadataAccumulatePageSize.ts | 35 ++ .../lib/index.ts | 1 + .../package.json | 43 +++ ...ActorRdfMetadataAccumulatePageSize-test.ts | 64 ++++ .../.npmignore | 0 .../README.md | 36 +++ .../ActorRdfMetadataAccumulateRequestTime.ts | 35 ++ .../lib/index.ts | 1 + .../package.json | 43 +++ ...orRdfMetadataAccumulateRequestTime-test.ts | 64 ++++ .../lib/ActorRdfMetadataExtractHydraCount.ts | 12 +- .../ActorRdfMetadataExtractHydraCount-test.ts | 4 +- .../ActorRdfResolveHypermediaNone-test.ts | 7 +- .../lib/ActorRdfResolveHypermediaQpf.ts | 12 +- .../lib/RdfSourceQpf.ts | 5 +- .../test/ActorRdfResolveHypermediaQpf-test.ts | 2 +- .../test/RdfSourceQpf-test.ts | 9 +- .../ActorRdfResolveQuadPatternFederated.ts | 47 +++ .../lib/FederatedQuadSource.ts | 125 ++++--- .../package.json | 2 + ...ctorRdfResolveQuadPatternFederated-test.ts | 118 ++++++- .../test/FederatedQuadSource-test.ts | 304 +++++++++++++++--- .../ActorRdfResolveQuadPatternHypermedia.ts | 18 ++ .../lib/LinkedRdfSourcesAsyncRdfIterator.ts | 98 ++++-- ...ediatedLinkedRdfSourcesAsyncRdfIterator.ts | 71 +++- .../lib/MediatedQuadSource.ts | 7 +- .../lib/StreamingStoreMetadata.ts | 80 ++++- .../package.json | 4 +- ...torRdfResolveQuadPatternHypermedia-test.ts | 76 ++++- .../LinkedRdfSourcesAsyncRdfIterator-test.ts | 143 ++++---- ...edLinkedRdfSourcesAsyncRdfIterator-test.ts | 159 ++++++++- .../test/MediatedQuadSource-test.ts | 212 +++++++++++- .../test/MediatorDereferenceRdf-util.ts | 27 ++ .../test/StreamingStoreMetadata-test.ts | 14 + .../lib/RdfJsQuadSource.ts | 7 +- .../package.json | 1 + ...orRdfResolveQuadPatternRdfJsSource-test.ts | 13 +- .../lib/ActorQueryOperation.ts | 17 +- .../test/ActorQueryOperation-test.ts | 25 +- packages/bus-rdf-join/lib/ActorRdfJoin.ts | 9 + packages/bus-rdf-join/package.json | 1 + .../bus-rdf-join/test/ActorRdfJoin-test.ts | 178 ++++++++-- .../bus-rdf-metadata-accumulate/.npmignore | 0 .../bus-rdf-metadata-accumulate/README.md | 25 ++ .../lib/ActorRdfMetadataAccumulate.ts | 66 ++++ .../bus-rdf-metadata-accumulate/lib/index.ts | 1 + .../bus-rdf-metadata-accumulate/package.json | 42 +++ packages/bus-rdf-metadata-extract/README.md | 2 +- .../lib/ActorRdfResolveQuadPatternSource.ts | 4 +- packages/metadata/.npmignore | 2 + packages/metadata/README.md | 16 + .../metadata/lib/MetadataValidationState.ts | 22 ++ packages/metadata/lib/index.ts | 1 + packages/metadata/package.json | 43 +++ .../test/MetadataValidationState-test.ts | 51 +++ packages/types/lib/IAggregatedStore.ts | 8 + packages/types/lib/IMetadata.ts | 42 ++- packages/types/lib/IQueryOperationResult.ts | 9 +- yarn.lock | 8 +- 118 files changed, 3204 insertions(+), 334 deletions(-) create mode 100644 engines/config-query-sparql/config/rdf-metadata-accumulate/actors.json create mode 100644 engines/config-query-sparql/config/rdf-metadata-accumulate/mediators.json create mode 100644 packages/actor-rdf-metadata-accumulate-cancontainundefs/.npmignore create mode 100644 packages/actor-rdf-metadata-accumulate-cancontainundefs/README.md create mode 100644 packages/actor-rdf-metadata-accumulate-cancontainundefs/lib/ActorRdfMetadataAccumulateCanContainUndefs.ts create mode 100644 packages/actor-rdf-metadata-accumulate-cancontainundefs/lib/index.ts create mode 100644 packages/actor-rdf-metadata-accumulate-cancontainundefs/package.json create mode 100644 packages/actor-rdf-metadata-accumulate-cancontainundefs/test/ActorRdfMetadataAccumulateCanContainUndefs-test.ts create mode 100644 packages/actor-rdf-metadata-accumulate-cardinality/.npmignore create mode 100644 packages/actor-rdf-metadata-accumulate-cardinality/README.md create mode 100644 packages/actor-rdf-metadata-accumulate-cardinality/lib/ActorRdfMetadataAccumulateCardinality.ts create mode 100644 packages/actor-rdf-metadata-accumulate-cardinality/lib/index.ts create mode 100644 packages/actor-rdf-metadata-accumulate-cardinality/package.json create mode 100644 packages/actor-rdf-metadata-accumulate-cardinality/test/ActorRdfMetadataAccumulateCardinality-test.ts create mode 100644 packages/actor-rdf-metadata-accumulate-pagesize/.npmignore create mode 100644 packages/actor-rdf-metadata-accumulate-pagesize/README.md create mode 100644 packages/actor-rdf-metadata-accumulate-pagesize/lib/ActorRdfMetadataAccumulatePageSize.ts create mode 100644 packages/actor-rdf-metadata-accumulate-pagesize/lib/index.ts create mode 100644 packages/actor-rdf-metadata-accumulate-pagesize/package.json create mode 100644 packages/actor-rdf-metadata-accumulate-pagesize/test/ActorRdfMetadataAccumulatePageSize-test.ts create mode 100644 packages/actor-rdf-metadata-accumulate-requesttime/.npmignore create mode 100644 packages/actor-rdf-metadata-accumulate-requesttime/README.md create mode 100644 packages/actor-rdf-metadata-accumulate-requesttime/lib/ActorRdfMetadataAccumulateRequestTime.ts create mode 100644 packages/actor-rdf-metadata-accumulate-requesttime/lib/index.ts create mode 100644 packages/actor-rdf-metadata-accumulate-requesttime/package.json create mode 100644 packages/actor-rdf-metadata-accumulate-requesttime/test/ActorRdfMetadataAccumulateRequestTime-test.ts create mode 100644 packages/actor-rdf-resolve-quad-pattern-hypermedia/test/StreamingStoreMetadata-test.ts create mode 100644 packages/bus-rdf-metadata-accumulate/.npmignore create mode 100644 packages/bus-rdf-metadata-accumulate/README.md create mode 100644 packages/bus-rdf-metadata-accumulate/lib/ActorRdfMetadataAccumulate.ts create mode 100644 packages/bus-rdf-metadata-accumulate/lib/index.ts create mode 100644 packages/bus-rdf-metadata-accumulate/package.json create mode 100644 packages/metadata/.npmignore create mode 100644 packages/metadata/README.md create mode 100644 packages/metadata/lib/MetadataValidationState.ts create mode 100644 packages/metadata/lib/index.ts create mode 100644 packages/metadata/package.json create mode 100644 packages/metadata/test/MetadataValidationState-test.ts diff --git a/.componentsjs-generator-config.json b/.componentsjs-generator-config.json index 9f4d01b56d..06e05d4132 100644 --- a/.componentsjs-generator-config.json +++ b/.componentsjs-generator-config.json @@ -53,6 +53,7 @@ "IQueryableResultVoid", "PathVariableObjectIterator", "ClosableTransformIterator", + "MetadataValidationState", "RDF.Stream", "Omit", diff --git a/engines/config-query-sparql/config/config-default.json b/engines/config-query-sparql/config/config-default.json index a2d6df6a03..57a54146ca 100644 --- a/engines/config-query-sparql/config/config-default.json +++ b/engines/config-query-sparql/config/config-default.json @@ -32,6 +32,8 @@ "ccqs:config/rdf-join-selectivity/mediators.json", "ccqs:config/rdf-metadata/actors.json", "ccqs:config/rdf-metadata/mediators.json", + "ccqs:config/rdf-metadata-accumulate/actors.json", + "ccqs:config/rdf-metadata-accumulate/mediators.json", "ccqs:config/rdf-metadata-extract/actors.json", "ccqs:config/rdf-metadata-extract/mediators.json", "ccqs:config/rdf-parse/actors.json", diff --git a/engines/config-query-sparql/config/config-rdfjs.json b/engines/config-query-sparql/config/config-rdfjs.json index fe1a95d40d..2b26b184fc 100644 --- a/engines/config-query-sparql/config/config-rdfjs.json +++ b/engines/config-query-sparql/config/config-rdfjs.json @@ -28,6 +28,9 @@ "ccqs:config/rdf-join-selectivity/actors.json", "ccqs:config/rdf-join-selectivity/mediators.json", + "ccqs:config/rdf-metadata-accumulate/actors.json", + "ccqs:config/rdf-metadata-accumulate/mediators.json", + "ccqs:config/rdf-resolve-quad-pattern/actors-rdfjs.json", "ccqs:config/rdf-resolve-quad-pattern/mediators.json", "ccqs:config/rdf-serialize/actors.json", diff --git a/engines/config-query-sparql/config/rdf-metadata-accumulate/actors.json b/engines/config-query-sparql/config/rdf-metadata-accumulate/actors.json new file mode 100644 index 0000000000..49250745ab --- /dev/null +++ b/engines/config-query-sparql/config/rdf-metadata-accumulate/actors.json @@ -0,0 +1,30 @@ +{ + "@context": [ + "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/runner/^2.0.0/components/context.jsonld", + + "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/actor-rdf-metadata-accumulate-cancontainundefs/^2.0.0/components/context.jsonld", + "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/actor-rdf-metadata-accumulate-cardinality/^2.0.0/components/context.jsonld", + "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/actor-rdf-metadata-accumulate-pagesize/^2.0.0/components/context.jsonld", + "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/actor-rdf-metadata-accumulate-requesttime/^2.0.0/components/context.jsonld" + ], + "@id": "urn:comunica:default:Runner", + "@type": "Runner", + "actors": [ + { + "@id": "urn:comunica:default:rdf-metadata-accumulate/actors#cancontainundefs", + "@type": "ActorRdfMetadataAccumulateCanContainUndefs" + }, + { + "@id": "urn:comunica:default:rdf-metadata-accumulate/actors#cardinality", + "@type": "ActorRdfMetadataAccumulateCardinality" + }, + { + "@id": "urn:comunica:default:rdf-metadata-accumulate/actors#pagesize", + "@type": "ActorRdfMetadataAccumulatePageSize" + }, + { + "@id": "urn:comunica:default:rdf-metadata-accumulate/actors#requesttime", + "@type": "ActorRdfMetadataAccumulateRequestTime" + } + ] +} diff --git a/engines/config-query-sparql/config/rdf-metadata-accumulate/mediators.json b/engines/config-query-sparql/config/rdf-metadata-accumulate/mediators.json new file mode 100644 index 0000000000..117e20d22a --- /dev/null +++ b/engines/config-query-sparql/config/rdf-metadata-accumulate/mediators.json @@ -0,0 +1,10 @@ +{ + "@context": [ + "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/bus-rdf-metadata-accumulate/^2.0.0/components/context.jsonld", + "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/mediator-combine-union/^2.0.0/components/context.jsonld" + ], + "@id": "urn:comunica:default:rdf-metadata-accumulate/mediators#main", + "@type": "MediatorCombineUnion", + "bus": { "@id": "ActorRdfMetadataAccumulate:_default_bus" }, + "field": "metadata" +} diff --git a/engines/config-query-sparql/config/rdf-resolve-quad-pattern/actors/federated.json b/engines/config-query-sparql/config/rdf-resolve-quad-pattern/actors/federated.json index 722996a89c..ecc60226ca 100644 --- a/engines/config-query-sparql/config/rdf-resolve-quad-pattern/actors/federated.json +++ b/engines/config-query-sparql/config/rdf-resolve-quad-pattern/actors/federated.json @@ -10,7 +10,8 @@ { "@id": "urn:comunica:default:rdf-resolve-quad-pattern/actors#federated", "@type": "ActorRdfResolveQuadPatternFederated", - "mediatorResolveQuadPattern": { "@id": "urn:comunica:default:rdf-resolve-quad-pattern/mediators#main" } + "mediatorResolveQuadPattern": { "@id": "urn:comunica:default:rdf-resolve-quad-pattern/mediators#main" }, + "mediatorRdfMetadataAccumulate": { "@id": "urn:comunica:default:rdf-metadata-accumulate/mediators#main" } } ] } diff --git a/engines/config-query-sparql/config/rdf-resolve-quad-pattern/actors/hypermedia.json b/engines/config-query-sparql/config/rdf-resolve-quad-pattern/actors/hypermedia.json index 37e1351818..b0cfac1a0d 100644 --- a/engines/config-query-sparql/config/rdf-resolve-quad-pattern/actors/hypermedia.json +++ b/engines/config-query-sparql/config/rdf-resolve-quad-pattern/actors/hypermedia.json @@ -13,6 +13,7 @@ "mediatorDereferenceRdf": { "@id": "urn:comunica:default:dereference-rdf/mediators#main" }, "mediatorMetadata": { "@id": "urn:comunica:default:rdf-metadata/mediators#main" }, "mediatorMetadataExtract": { "@id": "urn:comunica:default:rdf-metadata-extract/mediators#main" }, + "mediatorMetadataAccumulate": { "@id": "urn:comunica:default:rdf-metadata-accumulate/mediators#main" }, "mediatorRdfResolveHypermedia": { "@id": "urn:comunica:default:rdf-resolve-hypermedia/mediators#main" }, "mediatorRdfResolveHypermediaLinks": { "@id": "urn:comunica:default:rdf-resolve-hypermedia-links/mediators#main" }, "mediatorRdfResolveHypermediaLinksQueue": { "@id": "urn:comunica:default:rdf-resolve-hypermedia-links-queue/mediators#main" } diff --git a/engines/query-sparql-file/package.json b/engines/query-sparql-file/package.json index 962e1c3eeb..75f3f4c7d4 100644 --- a/engines/query-sparql-file/package.json +++ b/engines/query-sparql-file/package.json @@ -125,6 +125,10 @@ "@comunica/actor-rdf-join-optional-bind": "^2.6.8", "@comunica/actor-rdf-join-optional-nestedloop": "^2.6.8", "@comunica/actor-rdf-join-selectivity-variable-counting": "^2.6.8", + "@comunica/actor-rdf-metadata-accumulate-cancontainundefs": "^2.5.0", + "@comunica/actor-rdf-metadata-accumulate-cardinality": "^2.5.0", + "@comunica/actor-rdf-metadata-accumulate-pagesize": "^2.5.0", + "@comunica/actor-rdf-metadata-accumulate-requesttime": "^2.5.0", "@comunica/actor-rdf-metadata-all": "^2.6.8", "@comunica/actor-rdf-metadata-extract-allow-http-methods": "^2.6.8", "@comunica/actor-rdf-metadata-extract-hydra-controls": "^2.6.8", diff --git a/engines/query-sparql-rdfjs/package.json b/engines/query-sparql-rdfjs/package.json index daddd0fa76..116aaf18bf 100644 --- a/engines/query-sparql-rdfjs/package.json +++ b/engines/query-sparql-rdfjs/package.json @@ -110,6 +110,10 @@ "@comunica/actor-rdf-join-optional-bind": "^2.6.8", "@comunica/actor-rdf-join-optional-nestedloop": "^2.6.8", "@comunica/actor-rdf-join-selectivity-variable-counting": "^2.6.8", + "@comunica/actor-rdf-metadata-accumulate-cancontainundefs": "^2.5.0", + "@comunica/actor-rdf-metadata-accumulate-cardinality": "^2.5.0", + "@comunica/actor-rdf-metadata-accumulate-pagesize": "^2.5.0", + "@comunica/actor-rdf-metadata-accumulate-requesttime": "^2.5.0", "@comunica/actor-rdf-resolve-quad-pattern-federated": "^2.6.8", "@comunica/actor-rdf-resolve-quad-pattern-rdfjs-source": "^2.6.8", "@comunica/actor-rdf-serialize-jsonld": "^2.6.8", diff --git a/engines/query-sparql/package.json b/engines/query-sparql/package.json index a52fa69394..5b498781cd 100644 --- a/engines/query-sparql/package.json +++ b/engines/query-sparql/package.json @@ -123,6 +123,10 @@ "@comunica/actor-rdf-join-optional-bind": "^2.6.8", "@comunica/actor-rdf-join-optional-nestedloop": "^2.6.8", "@comunica/actor-rdf-join-selectivity-variable-counting": "^2.6.8", + "@comunica/actor-rdf-metadata-accumulate-cancontainundefs": "^2.5.0", + "@comunica/actor-rdf-metadata-accumulate-cardinality": "^2.5.0", + "@comunica/actor-rdf-metadata-accumulate-pagesize": "^2.5.0", + "@comunica/actor-rdf-metadata-accumulate-requesttime": "^2.5.0", "@comunica/actor-rdf-metadata-all": "^2.6.8", "@comunica/actor-rdf-metadata-extract-allow-http-methods": "^2.6.8", "@comunica/actor-rdf-metadata-extract-hydra-controls": "^2.6.8", diff --git a/lerna.json b/lerna.json index 4ebe1c7602..b47b21eada 100644 --- a/lerna.json +++ b/lerna.json @@ -8,7 +8,8 @@ "message": "Bump to release version %s", "allowBranch": [ "master", - "next/major" + "next/major", + "feature/adaptive-join" ], "npmClient": "npm" } diff --git a/packages/actor-init-query/test/QueryEngineBase-test.ts b/packages/actor-init-query/test/QueryEngineBase-test.ts index 5ebb24a4f1..8919c7027f 100644 --- a/packages/actor-init-query/test/QueryEngineBase-test.ts +++ b/packages/actor-init-query/test/QueryEngineBase-test.ts @@ -2,6 +2,7 @@ import { Readable, Transform } from 'stream'; import { BindingsFactory } from '@comunica/bindings-factory'; import { KeysInitQuery } from '@comunica/context-entries'; import { Bus, ActionContext, ActionContextKey } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IPhysicalQueryPlanLogger, IActionContext, QueryStringContext, IQueryBindingsEnhanced, IQueryQuadsEnhanced, @@ -551,6 +552,7 @@ describe('QueryEngineBase', () => { ]), ]), metadata: async() => ({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -565,6 +567,7 @@ describe('QueryEngineBase', () => { ]), ]); expect(await final.metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -579,6 +582,7 @@ describe('QueryEngineBase', () => { DF.quad(DF.namedNode('ex:a'), DF.namedNode('ex:a'), DF.namedNode('ex:a')), ]), metadata: async() => ({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -591,6 +595,7 @@ describe('QueryEngineBase', () => { DF.quad(DF.namedNode('ex:a'), DF.namedNode('ex:a'), DF.namedNode('ex:a')), ]); expect(await final.metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('a') ], diff --git a/packages/actor-query-operation-describe-subject/test/ActorQueryOperationDescribeSubject-test.ts b/packages/actor-query-operation-describe-subject/test/ActorQueryOperationDescribeSubject-test.ts index 8221d26a9f..e198ef31c2 100644 --- a/packages/actor-query-operation-describe-subject/test/ActorQueryOperationDescribeSubject-test.ts +++ b/packages/actor-query-operation-describe-subject/test/ActorQueryOperationDescribeSubject-test.ts @@ -1,5 +1,6 @@ import { ActorQueryOperation } from '@comunica/bus-query-operation'; import { ActionContext, Bus } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IQueryOperationResultQuads } from '@comunica/types'; import type * as RDF from '@rdfjs/types'; import arrayifyStream from 'arrayify-stream'; @@ -21,6 +22,7 @@ describe('ActorQueryOperationDescribeSubject', () => { const patterns = [ ...arg.operation.input.input[0].patterns, ...arg.operation.input.input[1].patterns ]; return { metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: arg.operation.input.input[0].patterns.length + arg.operation.input.input[1].patterns.length, @@ -36,6 +38,7 @@ describe('ActorQueryOperationDescribeSubject', () => { } return { metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: arg.operation.input.patterns.length }, }), quadStream: new ArrayIterator(arg.operation.input.patterns.map( @@ -94,7 +97,9 @@ describe('ActorQueryOperationDescribeSubject', () => { }; return actor.run(op).then(async(output: IQueryOperationResultQuads) => { expect(await output.metadata()) - .toEqual({ cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false }); + .toEqual({ state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: 2 }, + canContainUndefs: false }); expect(output.type).toEqual('quads'); expect(await arrayifyStream(output.quadStream)).toEqual([ DF.quad(DF.namedNode('a'), DF.namedNode('__predicate'), DF.namedNode('__object')), @@ -114,7 +119,9 @@ describe('ActorQueryOperationDescribeSubject', () => { }; return actor.run(op).then(async(output: IQueryOperationResultQuads) => { expect(await output.metadata()) - .toEqual({ cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false }); + .toEqual({ state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: 3 }, + canContainUndefs: false }); expect(output.type).toEqual('quads'); expect(await arrayifyStream(output.quadStream)).toEqual([ DF.quad(DF.namedNode('a'), DF.namedNode('b'), DF.namedNode('dummy')), @@ -135,7 +142,9 @@ describe('ActorQueryOperationDescribeSubject', () => { }; return actor.run(op).then(async(output: IQueryOperationResultQuads) => { expect(await output.metadata()) - .toEqual({ cardinality: { type: 'estimate', value: 4 }, canContainUndefs: false }); + .toEqual({ state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: 4 }, + canContainUndefs: false }); expect(output.type).toEqual('quads'); expect(await arrayifyStream(output.quadStream)).toEqual([ DF.quad(DF.namedNode('c'), DF.namedNode('__predicate'), DF.namedNode('__object')), diff --git a/packages/actor-query-operation-nop/lib/ActorQueryOperationNop.ts b/packages/actor-query-operation-nop/lib/ActorQueryOperationNop.ts index e97fa40339..e31634ccf7 100644 --- a/packages/actor-query-operation-nop/lib/ActorQueryOperationNop.ts +++ b/packages/actor-query-operation-nop/lib/ActorQueryOperationNop.ts @@ -2,6 +2,7 @@ import { BindingsFactory } from '@comunica/bindings-factory'; import type { IActorQueryOperationTypedMediatedArgs } from '@comunica/bus-query-operation'; import { ActorQueryOperationTypedMediated } from '@comunica/bus-query-operation'; import type { IActorTest } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IActionContext, IQueryOperationResult } from '@comunica/types'; import { SingletonIterator } from 'asynciterator'; import type { Algebra } from 'sparqlalgebrajs'; @@ -24,6 +25,7 @@ export class ActorQueryOperationNop extends ActorQueryOperationTypedMediated Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'exact', value: 1 }, canContainUndefs: false, variables: [], diff --git a/packages/actor-query-operation-nop/package.json b/packages/actor-query-operation-nop/package.json index a9197b100e..f0eca14b53 100644 --- a/packages/actor-query-operation-nop/package.json +++ b/packages/actor-query-operation-nop/package.json @@ -35,6 +35,7 @@ "@comunica/bindings-factory": "^2.5.1", "@comunica/bus-query-operation": "^2.6.8", "@comunica/core": "^2.6.8", + "@comunica/metadata": "^2.5.0", "@comunica/types": "^2.6.8", "asynciterator": "^3.8.0", "sparqlalgebrajs": "^4.0.5" diff --git a/packages/actor-query-operation-path-alt/test/ActorQueryOperationPathAlt-test.ts b/packages/actor-query-operation-path-alt/test/ActorQueryOperationPathAlt-test.ts index dd073eba0e..fc427e78df 100644 --- a/packages/actor-query-operation-path-alt/test/ActorQueryOperationPathAlt-test.ts +++ b/packages/actor-query-operation-path-alt/test/ActorQueryOperationPathAlt-test.ts @@ -1,6 +1,7 @@ import { BindingsFactory } from '@comunica/bindings-factory'; import { ActorQueryOperation } from '@comunica/bus-query-operation'; import { ActionContext, Bus } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import { ArrayIterator } from 'asynciterator'; import { DataFactory } from 'rdf-data-factory'; import { Algebra, Factory } from 'sparqlalgebrajs'; @@ -25,6 +26,7 @@ describe('ActorQueryOperationPathAlt', () => { BF.bindings([[ DF.variable('x'), DF.literal('3') ]]), ]), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -88,6 +90,7 @@ describe('ActorQueryOperationPathAlt', () => { context: new ActionContext() }; const output = ActorQueryOperation.getSafeBindings(await actor.run(op)); expect(await output.metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 6 }, canContainUndefs: false, variables: [ DF.variable('a') ], diff --git a/packages/actor-query-operation-path-zero-or-one/lib/ActorQueryOperationPathZeroOrOne.ts b/packages/actor-query-operation-path-zero-or-one/lib/ActorQueryOperationPathZeroOrOne.ts index bab7664d55..3803168a2c 100644 --- a/packages/actor-query-operation-path-zero-or-one/lib/ActorQueryOperationPathZeroOrOne.ts +++ b/packages/actor-query-operation-path-zero-or-one/lib/ActorQueryOperationPathZeroOrOne.ts @@ -2,6 +2,7 @@ import { ActorAbstractPath } from '@comunica/actor-abstract-path'; import { BindingsFactory } from '@comunica/bindings-factory'; import type { IActorQueryOperationTypedMediatedArgs } from '@comunica/bus-query-operation'; import { ActorQueryOperation } from '@comunica/bus-query-operation'; +import { MetadataValidationState } from '@comunica/metadata'; import type { Bindings, IQueryOperationResult, IActionContext } from '@comunica/types'; import { SingletonIterator } from 'asynciterator'; import { Algebra } from 'sparqlalgebrajs'; @@ -33,6 +34,7 @@ export class ActorQueryOperationPathZeroOrOne extends ActorAbstractPath { type: 'bindings', bindingsStream: new SingletonIterator(BF.bindings()), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'exact', value: 1 }, canContainUndefs: false, variables: [], diff --git a/packages/actor-query-operation-path-zero-or-one/package.json b/packages/actor-query-operation-path-zero-or-one/package.json index bf6f5f9af8..fe1ebbcbec 100644 --- a/packages/actor-query-operation-path-zero-or-one/package.json +++ b/packages/actor-query-operation-path-zero-or-one/package.json @@ -35,6 +35,7 @@ "@comunica/actor-abstract-path": "^2.6.8", "@comunica/bindings-factory": "^2.5.1", "@comunica/bus-query-operation": "^2.6.8", + "@comunica/metadata": "^2.5.0", "@comunica/types": "^2.6.8", "asynciterator": "^3.8.0", "sparqlalgebrajs": "^4.0.5" diff --git a/packages/actor-query-operation-path-zero-or-one/test/ActorQueryOperationPathZeroOrOne-test.ts b/packages/actor-query-operation-path-zero-or-one/test/ActorQueryOperationPathZeroOrOne-test.ts index 022bb6ec28..c1efc2b9f1 100644 --- a/packages/actor-query-operation-path-zero-or-one/test/ActorQueryOperationPathZeroOrOne-test.ts +++ b/packages/actor-query-operation-path-zero-or-one/test/ActorQueryOperationPathZeroOrOne-test.ts @@ -2,6 +2,7 @@ import { BindingsFactory } from '@comunica/bindings-factory'; import { ActorQueryOperation } from '@comunica/bus-query-operation'; import { KeysQueryOperation } from '@comunica/context-entries'; import { Bus, ActionContext } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type * as RDF from '@rdfjs/types'; import { ArrayIterator } from 'asynciterator'; import { DataFactory } from 'rdf-data-factory'; @@ -51,6 +52,7 @@ describe('ActorQueryOperationPathZeroOrOne', () => { return Promise.resolve({ bindingsStream: new ArrayIterator(distinct ? [ bindings[0] ] : bindings), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: distinct ? 1 : 3 }, canContainUndefs: false, variables: vars, @@ -107,6 +109,7 @@ describe('ActorQueryOperationPathZeroOrOne', () => { context: new ActionContext() }; const output = ActorQueryOperation.getSafeBindings(await actor.run(op)); expect(await output.metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('x') ], @@ -125,6 +128,7 @@ describe('ActorQueryOperationPathZeroOrOne', () => { context: new ActionContext({ [KeysQueryOperation.isPathArbitraryLengthDistinctKey.name]: false }) }; const output = ActorQueryOperation.getSafeBindings(await actor.run(op)); expect(await output.metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('x') ], @@ -143,6 +147,7 @@ describe('ActorQueryOperationPathZeroOrOne', () => { context: new ActionContext({ [KeysQueryOperation.isPathArbitraryLengthDistinctKey.name]: true }) }; const output = ActorQueryOperation.getSafeBindings(await actor.run(op)); expect(await output.metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('x') ], @@ -164,6 +169,7 @@ describe('ActorQueryOperationPathZeroOrOne', () => { context: new ActionContext({ [KeysQueryOperation.isPathArbitraryLengthDistinctKey.name]: true }) }; const output = ActorQueryOperation.getSafeBindings(await actor.run(op)); expect(await output.metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('x') ], @@ -185,6 +191,7 @@ describe('ActorQueryOperationPathZeroOrOne', () => { context: new ActionContext({ [KeysQueryOperation.isPathArbitraryLengthDistinctKey.name]: true }) }; const output = ActorQueryOperation.getSafeBindings(await actor.run(op)); expect(await output.metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [], @@ -203,6 +210,7 @@ describe('ActorQueryOperationPathZeroOrOne', () => { context: new ActionContext({ [KeysQueryOperation.isPathArbitraryLengthDistinctKey.name]: true }) }; const output = ActorQueryOperation.getSafeBindings(await actor.run(op)); expect(await output.metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'exact', value: 1 }, canContainUndefs: false, variables: [], diff --git a/packages/actor-query-operation-service/lib/ActorQueryOperationService.ts b/packages/actor-query-operation-service/lib/ActorQueryOperationService.ts index d74fa203fa..9c118fc62f 100644 --- a/packages/actor-query-operation-service/lib/ActorQueryOperationService.ts +++ b/packages/actor-query-operation-service/lib/ActorQueryOperationService.ts @@ -3,6 +3,7 @@ import type { IActorQueryOperationTypedMediatedArgs } from '@comunica/bus-query- import { ActorQueryOperation, ActorQueryOperationTypedMediated } from '@comunica/bus-query-operation'; import { KeysInitQuery, KeysRdfResolveQuadPattern } from '@comunica/context-entries'; import type { IActorTest } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IActionContext, IQueryOperationResult, IQueryOperationResultBindings } from '@comunica/types'; import { SingletonIterator } from 'asynciterator'; import type { Algebra } from 'sparqlalgebrajs'; @@ -50,7 +51,12 @@ export class ActorQueryOperationService extends ActorQueryOperationTypedMediated output = { bindingsStream: new SingletonIterator(BF.bindings()), type: 'bindings', - metadata: async() => ({ cardinality: { type: 'exact', value: 1 }, canContainUndefs: false, variables: []}), + metadata: async() => ({ + state: new MetadataValidationState(), + cardinality: { type: 'exact', value: 1 }, + canContainUndefs: false, + variables: [], + }), }; } else { throw error; diff --git a/packages/actor-query-operation-service/package.json b/packages/actor-query-operation-service/package.json index 0fe4eb495c..cb3ebdb948 100644 --- a/packages/actor-query-operation-service/package.json +++ b/packages/actor-query-operation-service/package.json @@ -36,6 +36,7 @@ "@comunica/bus-query-operation": "^2.6.8", "@comunica/context-entries": "^2.6.8", "@comunica/core": "^2.6.8", + "@comunica/metadata": "^2.5.0", "@comunica/types": "^2.6.8", "asynciterator": "^3.8.0", "sparqlalgebrajs": "^4.0.5" diff --git a/packages/actor-query-operation-service/test/ActorQueryOperationService-test.ts b/packages/actor-query-operation-service/test/ActorQueryOperationService-test.ts index 69595bd614..82853f6790 100644 --- a/packages/actor-query-operation-service/test/ActorQueryOperationService-test.ts +++ b/packages/actor-query-operation-service/test/ActorQueryOperationService-test.ts @@ -116,7 +116,7 @@ describe('ActorQueryOperationService', () => { context: new ActionContext() }; return actor.run(op).then(async(output: IQueryOperationResultBindings) => { expect(await output.metadata()) - .toEqual({ cardinality: { type: 'exact', value: 1 }, canContainUndefs: false, variables: []}); + .toMatchObject({ cardinality: { type: 'exact', value: 1 }, canContainUndefs: false, variables: []}); await expect(output.bindingsStream).toEqualBindingsStream([ BF.bindings(), ]); diff --git a/packages/actor-query-operation-sparql-endpoint/lib/ActorQueryOperationSparqlEndpoint.ts b/packages/actor-query-operation-sparql-endpoint/lib/ActorQueryOperationSparqlEndpoint.ts index 9be3e0c83a..dd2d199ce6 100644 --- a/packages/actor-query-operation-sparql-endpoint/lib/ActorQueryOperationSparqlEndpoint.ts +++ b/packages/actor-query-operation-sparql-endpoint/lib/ActorQueryOperationSparqlEndpoint.ts @@ -11,6 +11,7 @@ import { import { KeysInitQuery } from '@comunica/context-entries'; import type { IActorArgs, IActorTest } from '@comunica/core'; import type { IMediatorTypeHttpRequests } from '@comunica/mediatortype-httprequests'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IQueryOperationResult, IQueryOperationResultBindings, IQueryOperationResultBoolean, @@ -154,6 +155,7 @@ export class ActorQueryOperationSparqlEndpoint extends ActorQueryOperation { const metadata: () => Promise> = ActorQueryOperationSparqlEndpoint.cachifyMetadata( async() => ({ + state: new MetadataValidationState(), cardinality: { type: 'exact', value: await resultStream.getCardinality() }, canContainUndefs: true, variables, diff --git a/packages/actor-query-operation-sparql-endpoint/package.json b/packages/actor-query-operation-sparql-endpoint/package.json index cf55078572..69a82b1bbf 100644 --- a/packages/actor-query-operation-sparql-endpoint/package.json +++ b/packages/actor-query-operation-sparql-endpoint/package.json @@ -39,6 +39,7 @@ "@comunica/bus-rdf-update-quads": "^2.6.9", "@comunica/context-entries": "^2.6.8", "@comunica/core": "^2.6.8", + "@comunica/metadata": "^2.5.0", "@comunica/mediatortype-httprequests": "^2.6.8", "@comunica/types": "^2.6.8", "@rdfjs/types": "*", diff --git a/packages/actor-query-operation-sparql-endpoint/test/ActorQueryOperationSparqlEndpoint-test.ts b/packages/actor-query-operation-sparql-endpoint/test/ActorQueryOperationSparqlEndpoint-test.ts index 626048764d..2b69b98920 100644 --- a/packages/actor-query-operation-sparql-endpoint/test/ActorQueryOperationSparqlEndpoint-test.ts +++ b/packages/actor-query-operation-sparql-endpoint/test/ActorQueryOperationSparqlEndpoint-test.ts @@ -200,7 +200,7 @@ describe('ActorQueryOperationSparqlEndpoint', () => { const op: any = { context, operation: factory.createPattern(DF.namedNode('http://s'), DF.variable('p'), DF.namedNode('http://o')) }; const output: IQueryOperationResultBindings = await actor.run(op); - expect(await output.metadata()).toEqual({ + expect(await output.metadata()).toMatchObject({ cardinality: { type: 'exact', value: 3 }, canContainUndefs: true, variables: [ DF.variable('p') ], @@ -229,7 +229,7 @@ describe('ActorQueryOperationSparqlEndpoint', () => { [ DF.variable('myP') ], ) }; const output: IQueryOperationResultBindings = await actor.run(op); - expect(await output.metadata()).toEqual({ + expect(await output.metadata()).toMatchObject({ cardinality: { type: 'exact', value: 3 }, canContainUndefs: true, variables: [ DF.variable('myP') ], @@ -263,7 +263,7 @@ describe('ActorQueryOperationSparqlEndpoint', () => { const op: any = { context, operation: factory.createPattern(DF.namedNode('http://s'), DF.variable('p'), DF.namedNode('http://o')) }; const output: IQueryOperationResultBindings = await actor.run(op); - expect(await output.metadata()).toEqual({ + expect(await output.metadata()).toMatchObject({ cardinality: { type: 'exact', value: 3 }, canContainUndefs: true, variables: [ DF.variable('p') ], @@ -300,7 +300,7 @@ describe('ActorQueryOperationSparqlEndpoint', () => { [ DF.variable('myP') ], ) }; const output: IQueryOperationResultBindings = await actor.run(op); - expect(await output.metadata()).toEqual({ + expect(await output.metadata()).toMatchObject({ cardinality: { type: 'exact', value: 3 }, canContainUndefs: true, variables: [ DF.variable('myP') ], @@ -346,7 +346,7 @@ describe('ActorQueryOperationSparqlEndpoint', () => { ) }; const output: IQueryOperationResultQuads = await actor.run(op); - expect(await output.metadata()).toEqual({ + expect(await output.metadata()).toMatchObject({ cardinality: { type: 'exact', value: 2 }, canContainUndefs: true, }); @@ -432,7 +432,7 @@ this is a body`)); const op: any = { context, operation: factory.createPattern(DF.namedNode('http://s'), DF.variable('p'), DF.namedNode('http://o')) }; const output: IQueryOperationResultBindings = await actor.run(op); - expect(await ( output).metadata()).toEqual({ + expect(await ( output).metadata()).toMatchObject({ cardinality: { type: 'exact', value: 3 }, canContainUndefs: true, variables: [ DF.variable('p') ], diff --git a/packages/actor-query-operation-union/lib/ActorQueryOperationUnion.ts b/packages/actor-query-operation-union/lib/ActorQueryOperationUnion.ts index f568133b73..0c37790b03 100644 --- a/packages/actor-query-operation-union/lib/ActorQueryOperationUnion.ts +++ b/packages/actor-query-operation-union/lib/ActorQueryOperationUnion.ts @@ -4,6 +4,7 @@ import { ActorQueryOperationTypedMediated, } from '@comunica/bus-query-operation'; import type { IActorTest } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { BindingsStream, IQueryOperationResultBindings, IActionContext, @@ -60,10 +61,17 @@ export class ActorQueryOperationUnion extends ActorQueryOperationTypedMediated metadata.canContainUndefs), }; + // Propagate metadata invalidations + const invalidateListener = (): void => metadataBase.state.invalidate(); + for (const metadata of metadatas) { + metadata.state.addInvalidateListener(invalidateListener); + } + // Union variables if (bindings) { metadataBase.variables = ActorQueryOperationUnion.unionVariables(metadatas.map(metadata => metadata.variables)); diff --git a/packages/actor-query-operation-union/package.json b/packages/actor-query-operation-union/package.json index 450b5d8991..9a4ef3d644 100644 --- a/packages/actor-query-operation-union/package.json +++ b/packages/actor-query-operation-union/package.json @@ -34,6 +34,7 @@ "dependencies": { "@comunica/bus-query-operation": "^2.6.8", "@comunica/core": "^2.6.8", + "@comunica/metadata": "^2.5.0", "@comunica/types": "^2.6.8", "@rdfjs/types": "*", "asynciterator": "^3.8.0", diff --git a/packages/actor-query-operation-union/test/ActorQueryOperationUnion-test.ts b/packages/actor-query-operation-union/test/ActorQueryOperationUnion-test.ts index fc9de0e5da..b2c31d1051 100644 --- a/packages/actor-query-operation-union/test/ActorQueryOperationUnion-test.ts +++ b/packages/actor-query-operation-union/test/ActorQueryOperationUnion-test.ts @@ -1,6 +1,7 @@ import { BindingsFactory } from '@comunica/bindings-factory'; import { ActorQueryOperation } from '@comunica/bus-query-operation'; import { ActionContext, Bus } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IQueryOperationResultBindings } from '@comunica/types'; import { ArrayIterator } from 'asynciterator'; import { DataFactory } from 'rdf-data-factory'; @@ -30,6 +31,7 @@ describe('ActorQueryOperationUnion', () => { }; op3 = () => ({ metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -43,6 +45,7 @@ describe('ActorQueryOperationUnion', () => { }); op2 = () => ({ metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false, variables: [ DF.variable('b') ], @@ -55,6 +58,7 @@ describe('ActorQueryOperationUnion', () => { }); op2Undef = () => ({ metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: true, variables: [ DF.variable('b') ], @@ -140,91 +144,125 @@ describe('ActorQueryOperationUnion', () => { describe('ActorQueryOperationUnion#unionMetadata', () => { it('should return 0 items for an empty input', () => { return expect(ActorQueryOperationUnion.unionMetadata([], false)) - .toEqual({ cardinality: { type: 'exact', value: 0 }, canContainUndefs: false }); + .toMatchObject({ cardinality: { type: 'exact', value: 0 }, canContainUndefs: false }); }); it('should return 1 items for a single input with 1', () => { return expect(ActorQueryOperationUnion.unionMetadata([ - { cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false }, + { state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false }, ], false)) - .toEqual({ cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false }); + .toMatchObject({ cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false }); }); it('should return 0 items for a single input with 0', () => { return expect(ActorQueryOperationUnion.unionMetadata([ - { cardinality: { type: 'estimate', value: 0 }, canContainUndefs: false }, + { state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 0 }, canContainUndefs: false }, ], false)) - .toEqual({ cardinality: { type: 'estimate', value: 0 }, canContainUndefs: false }); + .toMatchObject({ cardinality: { type: 'estimate', value: 0 }, canContainUndefs: false }); }); it('should return infinite items for a single input with Infinity', () => { return expect(ActorQueryOperationUnion.unionMetadata([ - { cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false }, + { + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, + canContainUndefs: false, + }, ], false)) - .toEqual({ cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false }); + .toMatchObject({ cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false }); }); it('should return 3 items for inputs with 1 and 2 as estimate', () => { return expect(ActorQueryOperationUnion.unionMetadata([ - { cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false }, - { cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false }, + { state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false }, + { state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false }, ], false)) - .toEqual({ cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false }); + .toMatchObject({ cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false }); }); it('should return 3 items for inputs with 1 and 2 as exact', () => { return expect(ActorQueryOperationUnion.unionMetadata([ - { cardinality: { type: 'exact', value: 1 }, canContainUndefs: false }, - { cardinality: { type: 'exact', value: 2 }, canContainUndefs: false }, + { state: new MetadataValidationState(), cardinality: { type: 'exact', value: 1 }, canContainUndefs: false }, + { state: new MetadataValidationState(), cardinality: { type: 'exact', value: 2 }, canContainUndefs: false }, ], false)) - .toEqual({ cardinality: { type: 'exact', value: 3 }, canContainUndefs: false }); + .toMatchObject({ cardinality: { type: 'exact', value: 3 }, canContainUndefs: false }); }); it('should return 3 items for inputs with 1 and 2 as exact and estimate', () => { return expect(ActorQueryOperationUnion.unionMetadata([ - { cardinality: { type: 'exact', value: 1 }, canContainUndefs: false }, - { cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false }, + { state: new MetadataValidationState(), cardinality: { type: 'exact', value: 1 }, canContainUndefs: false }, + { state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false }, ], false)) - .toEqual({ cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false }); + .toMatchObject({ cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false }); }); it('should return infinite items for inputs with Infinity and 2', () => { return expect(ActorQueryOperationUnion .unionMetadata([ - { cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false }, - { cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false }, + { + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, + canContainUndefs: false, + }, + { + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 2 }, + canContainUndefs: false, + }, ], false)) - .toEqual({ cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false }); + .toMatchObject({ cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false }); }); it('should return infinite items for inputs with 1 and Infinity', () => { return expect(ActorQueryOperationUnion .unionMetadata([ - { cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false }, - { cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false }, + { + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 1 }, + canContainUndefs: false, + }, + { + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, + canContainUndefs: false, + }, ], false)) - .toEqual({ cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false }); + .toMatchObject({ cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false }); }); it('should return infinite items for inputs with Infinity and Infinity', () => { return expect(ActorQueryOperationUnion .unionMetadata([ - { cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false }, - { cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false }, + { + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, + canContainUndefs: false, + }, + { + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, + canContainUndefs: false, + }, ], false)) - .toEqual({ cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false }); + .toMatchObject({ cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false }); }); it('should union variables if bindings is true', () => { return expect(ActorQueryOperationUnion.unionMetadata([ - { cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('a') ]}, { + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 1 }, + canContainUndefs: false, + variables: [ DF.variable('a') ], + }, + { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], }, ], true)) - .toEqual({ + .toMatchObject({ cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -259,7 +297,7 @@ describe('ActorQueryOperationUnion', () => { it('should run on two streams', () => { const op: any = { operation: { type: 'union', input: [ op3(), op2() ]}, context: new ActionContext() }; return actor.run(op).then(async(output: IQueryOperationResultBindings) => { - expect(await output.metadata()).toEqual({ + expect(await output.metadata()).toMatchObject({ cardinality: { type: 'estimate', value: 5 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -281,6 +319,7 @@ describe('ActorQueryOperationUnion', () => { context: new ActionContext(), }).then(async(output: IQueryOperationResultBindings) => { expect(await output.metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 7 }, canContainUndefs: true, variables: [ DF.variable('a'), DF.variable('b') ], @@ -301,7 +340,7 @@ describe('ActorQueryOperationUnion', () => { it('should run with a right stream with undefs', () => { const op: any = { operation: { type: 'union', input: [ op3(), op2Undef() ]}, context: new ActionContext() }; return actor.run(op).then(async(output: IQueryOperationResultBindings) => { - expect(await output.metadata()).toEqual({ + expect(await output.metadata()).toMatchObject({ cardinality: { type: 'estimate', value: 5 }, canContainUndefs: true, variables: [ DF.variable('a'), DF.variable('b') ], @@ -316,5 +355,48 @@ describe('ActorQueryOperationUnion', () => { ]); }); }); + + it('should run on two streams with metadata invalidation', async() => { + // An operation in which we can access the metadata state + const state = new MetadataValidationState(); + const opCustom = { + metadata: () => Promise.resolve({ + state, + cardinality: { type: 'estimate', value: 3 }, + canContainUndefs: false, + variables: [ DF.variable('a') ], + }), + stream: new ArrayIterator([ + BF.bindings([[ DF.variable('a'), DF.literal('1') ]]), + BF.bindings([[ DF.variable('a'), DF.literal('2') ]]), + BF.bindings([[ DF.variable('a'), DF.literal('3') ]]), + ], { autoStart: false }), + type: 'bindings', + }; + + // Execute the operation, and expect a valid metadata + const op: any = { operation: { type: 'union', input: [ op3(), opCustom ]}, context: new ActionContext() }; + const output: IQueryOperationResultBindings = await actor.run(op); + const outputMetadata = await output.metadata(); + expect(outputMetadata).toMatchObject({ + state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: 6 }, + canContainUndefs: false, + variables: [ DF.variable('a') ], + }); + + // After invoking this, we expect the returned metadata to also be invalidated + state.invalidate(); + expect(outputMetadata.state.valid).toBeFalsy(); + + // We can request a new metadata object, which will be valid again. + const outputMetadata2 = await output.metadata(); + expect(outputMetadata2).toMatchObject({ + state: { valid: true }, + cardinality: { type: 'estimate', value: 6 }, + canContainUndefs: false, + variables: [ DF.variable('a') ], + }); + }); }); }); diff --git a/packages/actor-query-operation-values/lib/ActorQueryOperationValues.ts b/packages/actor-query-operation-values/lib/ActorQueryOperationValues.ts index 31539d0ee6..f247dcbbbb 100644 --- a/packages/actor-query-operation-values/lib/ActorQueryOperationValues.ts +++ b/packages/actor-query-operation-values/lib/ActorQueryOperationValues.ts @@ -2,6 +2,7 @@ import { BindingsFactory } from '@comunica/bindings-factory'; import type { IActionQueryOperation } from '@comunica/bus-query-operation'; import { ActorQueryOperationTyped } from '@comunica/bus-query-operation'; import type { IActorArgs, IActorTest } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IQueryOperationResult, BindingsStream, Bindings, @@ -33,6 +34,7 @@ export class ActorQueryOperationValues extends ActorQueryOperationTyped [ DF.variable(key.slice(1)), value ])))); const variables = operation.variables; const metadata = (): Promise => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'exact', value: operation.bindings.length }, canContainUndefs: operation.bindings.some(bindings => variables.some(variable => !(`?${variable.value}` in bindings))), variables, diff --git a/packages/actor-query-operation-values/package.json b/packages/actor-query-operation-values/package.json index ee98f6266a..25525e66bf 100644 --- a/packages/actor-query-operation-values/package.json +++ b/packages/actor-query-operation-values/package.json @@ -35,6 +35,7 @@ "@comunica/bindings-factory": "^2.5.1", "@comunica/bus-query-operation": "^2.6.8", "@comunica/core": "^2.6.8", + "@comunica/metadata": "^2.5.0", "@comunica/types": "^2.6.8", "asynciterator": "^3.8.0", "rdf-data-factory": "^1.1.1", diff --git a/packages/actor-query-operation-values/test/ActorQueryOperationValues-test.ts b/packages/actor-query-operation-values/test/ActorQueryOperationValues-test.ts index 8ca4547d8e..e004a7df85 100644 --- a/packages/actor-query-operation-values/test/ActorQueryOperationValues-test.ts +++ b/packages/actor-query-operation-values/test/ActorQueryOperationValues-test.ts @@ -55,7 +55,7 @@ describe('ActorQueryOperationValues', () => { const bindings = [{ '?v': DF.namedNode('v1') }]; const op: any = { operation: { type: 'values', variables, bindings }, context: new ActionContext() }; return actor.run(op).then(async(output: IQueryOperationResultBindings) => { - expect(await output.metadata()).toEqual({ + expect(await output.metadata()).toMatchObject({ cardinality: { type: 'exact', value: 1 }, canContainUndefs: false, variables: [ DF.variable('v') ], @@ -74,7 +74,7 @@ describe('ActorQueryOperationValues', () => { const bindings = [{ '?v': DF.namedNode('v1') }, { '?v': DF.namedNode('v2') }]; const op: any = { operation: { type: 'values', variables, bindings }, context: new ActionContext() }; return actor.run(op).then(async(output: IQueryOperationResultBindings) => { - expect(await output.metadata()).toEqual({ + expect(await output.metadata()).toMatchObject({ cardinality: { type: 'exact', value: 2 }, canContainUndefs: false, variables: [ DF.variable('v') ], @@ -99,7 +99,7 @@ describe('ActorQueryOperationValues', () => { ]; const op: any = { operation: { type: 'values', variables, bindings }, context: new ActionContext() }; return actor.run(op).then(async(output: IQueryOperationResultBindings) => { - expect(await output.metadata()).toEqual({ + expect(await output.metadata()).toMatchObject({ cardinality: { type: 'exact', value: 2 }, canContainUndefs: false, variables: [ DF.variable('v'), DF.variable('w') ], @@ -126,7 +126,7 @@ describe('ActorQueryOperationValues', () => { ]; const op: any = { operation: { type: 'values', variables, bindings }, context: new ActionContext() }; return actor.run(op).then(async(output: IQueryOperationResultBindings) => { - expect(await output.metadata()).toEqual({ + expect(await output.metadata()).toMatchObject({ cardinality: { type: 'exact', value: 2 }, canContainUndefs: true, variables: [ DF.variable('v'), DF.variable('w') ], diff --git a/packages/actor-rdf-join-inner-hash/test/ActorRdfJoinHash-test.ts b/packages/actor-rdf-join-inner-hash/test/ActorRdfJoinHash-test.ts index 16d48305dc..9cba3d34e1 100644 --- a/packages/actor-rdf-join-inner-hash/test/ActorRdfJoinHash-test.ts +++ b/packages/actor-rdf-join-inner-hash/test/ActorRdfJoinHash-test.ts @@ -4,6 +4,7 @@ import { ActorRdfJoin } from '@comunica/bus-rdf-join'; import type { IActionRdfJoinSelectivity, IActorRdfJoinSelectivityOutput } from '@comunica/bus-rdf-join-selectivity'; import type { Actor, IActorTest, Mediator } from '@comunica/core'; import { ActionContext, Bus } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IQueryOperationResultBindings, Bindings, IActionContext } from '@comunica/types'; import type * as RDF from '@rdfjs/types'; import arrayifyStream from 'arrayify-stream'; @@ -74,6 +75,7 @@ describe('ActorRdfJoinHash', () => { bindingsStream: iterators[0], metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 4 }, pageSize: 100, requestTime: 10, @@ -90,6 +92,7 @@ describe('ActorRdfJoinHash', () => { bindingsStream: iterators[1], metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, pageSize: 100, requestTime: 20, @@ -121,6 +124,7 @@ describe('ActorRdfJoinHash', () => { bindingsStream: iterators[0], metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 4 }, pageSize: 100, requestTime: 10, @@ -137,6 +141,7 @@ describe('ActorRdfJoinHash', () => { bindingsStream: iterators[1], metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, pageSize: 100, requestTime: 20, @@ -166,6 +171,7 @@ describe('ActorRdfJoinHash', () => { bindingsStream: iterators[0], metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 4 }, pageSize: 100, requestTime: 10, @@ -181,6 +187,7 @@ describe('ActorRdfJoinHash', () => { output: { bindingsStream: iterators[1], metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, pageSize: 100, requestTime: 20, @@ -208,6 +215,7 @@ describe('ActorRdfJoinHash', () => { output: { bindingsStream: iterators[0], metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 4 }, pageSize: 100, requestTime: 10, @@ -222,6 +230,7 @@ describe('ActorRdfJoinHash', () => { output: { bindingsStream: iterators[1], metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, pageSize: 100, requestTime: 20, @@ -269,7 +278,10 @@ describe('ActorRdfJoinHash', () => { it('should return an empty stream for empty input', () => { return actor.run(action).then(async(output: IQueryOperationResultBindings) => { expect(await output.metadata()) - .toEqual({ cardinality: { type: 'estimate', value: 20 }, canContainUndefs: false, variables: []}); + .toEqual({ state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: 20 }, + canContainUndefs: false, + variables: []}); await expect(output.bindingsStream).toEqualBindingsStream([]); }); }); @@ -295,6 +307,7 @@ describe('ActorRdfJoinHash', () => { return actor.run(action).then(async(output: IQueryOperationResultBindings) => { expect(await output.metadata()) .toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 20 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b'), DF.variable('c') ], @@ -329,6 +342,7 @@ describe('ActorRdfJoinHash', () => { variables1 = [ DF.variable('a'), DF.variable('c') ]; return actor.run(action).then(async(output: IQueryOperationResultBindings) => { expect(await output.metadata()).toEqual({ + state: expect.any(MetadataValidationState), canContainUndefs: false, cardinality: { type: 'estimate', value: 20 }, variables: [ DF.variable('a'), DF.variable('b'), DF.variable('c') ], @@ -439,6 +453,7 @@ describe('ActorRdfJoinHash', () => { ]), ]; expect(await output.metadata()).toEqual({ + state: expect.any(MetadataValidationState), canContainUndefs: false, cardinality: { type: 'estimate', value: 20 }, variables: [ DF.variable('a'), DF.variable('b'), DF.variable('c') ], diff --git a/packages/actor-rdf-join-inner-multi-bind/test/ActorRdfJoinMultiBind-test.ts b/packages/actor-rdf-join-inner-multi-bind/test/ActorRdfJoinMultiBind-test.ts index 6be521f71f..7289ab6771 100644 --- a/packages/actor-rdf-join-inner-multi-bind/test/ActorRdfJoinMultiBind-test.ts +++ b/packages/actor-rdf-join-inner-multi-bind/test/ActorRdfJoinMultiBind-test.ts @@ -6,6 +6,7 @@ import type { IActionRdfJoinSelectivity, IActorRdfJoinSelectivityOutput } from ' import { KeysQueryOperation } from '@comunica/context-entries'; import type { Actor, IActorTest, Mediator } from '@comunica/core'; import { ActionContext, Bus } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IActionContext, IQueryOperationResultBindings } from '@comunica/types'; import { ArrayIterator } from 'asynciterator'; import { DataFactory } from 'rdf-data-factory'; @@ -63,6 +64,7 @@ describe('ActorRdfJoinMultiBind', () => { ]), ], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('bound') ], @@ -106,6 +108,7 @@ describe('ActorRdfJoinMultiBind', () => { }, [ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, pageSize: 100, requestTime: 10, @@ -113,6 +116,7 @@ describe('ActorRdfJoinMultiBind', () => { variables: [ DF.variable('a') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, pageSize: 100, requestTime: 20, @@ -120,6 +124,7 @@ describe('ActorRdfJoinMultiBind', () => { variables: [ DF.variable('a') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, pageSize: 100, requestTime: 30, @@ -157,6 +162,7 @@ describe('ActorRdfJoinMultiBind', () => { }, [ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, pageSize: 100, requestTime: 10, @@ -164,6 +170,7 @@ describe('ActorRdfJoinMultiBind', () => { variables: [ DF.variable('a'), DF.variable('b') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, pageSize: 100, requestTime: 20, @@ -171,6 +178,7 @@ describe('ActorRdfJoinMultiBind', () => { variables: [ DF.variable('a'), DF.variable('b') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, pageSize: 100, requestTime: 30, @@ -215,6 +223,7 @@ describe('ActorRdfJoinMultiBind', () => { }, [ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, pageSize: 100, requestTime: 10, @@ -222,6 +231,7 @@ describe('ActorRdfJoinMultiBind', () => { variables: [ DF.variable('a') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, pageSize: 100, requestTime: 20, @@ -250,17 +260,21 @@ describe('ActorRdfJoinMultiBind', () => { }, [ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, pageSize: 100, requestTime: 10, canContainUndefs: false, variables: [ DF.variable('a') ], }, - { cardinality: { type: 'estimate', value: 2 }, + { + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 2 }, pageSize: 100, requestTime: 20, canContainUndefs: false, - variables: [ DF.variable('a') ]}, + variables: [ DF.variable('a') ], + }, ], )).rejects.toThrowError('Actor actor can not bind on Extend and Group operations'); }); @@ -283,6 +297,7 @@ describe('ActorRdfJoinMultiBind', () => { }, [ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, pageSize: 100, requestTime: 10, @@ -290,6 +305,7 @@ describe('ActorRdfJoinMultiBind', () => { variables: [ DF.variable('a') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, pageSize: 100, requestTime: 20, @@ -321,6 +337,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -330,6 +347,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -342,6 +360,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -351,6 +370,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -365,6 +385,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -374,6 +395,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -383,6 +405,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -394,6 +417,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -403,6 +427,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -412,6 +437,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -427,6 +453,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -436,6 +463,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -445,6 +473,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -457,6 +486,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -466,6 +496,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -475,6 +506,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -490,6 +522,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -499,6 +532,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -508,6 +542,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: true, variables: [ DF.variable('a') ], @@ -520,6 +555,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -529,6 +565,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -538,6 +575,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: true, variables: [ DF.variable('a') ], @@ -553,6 +591,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a1'), DF.variable('b1') ], @@ -562,6 +601,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false, variables: [ DF.variable('a2'), DF.variable('b2') ], @@ -579,6 +619,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('b') ], @@ -588,6 +629,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -597,6 +639,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -609,6 +652,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -618,6 +662,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -627,6 +672,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('b') ], @@ -642,6 +688,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -651,6 +698,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('b') ], @@ -660,6 +708,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 20 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -669,6 +718,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 20 }, canContainUndefs: false, variables: [ DF.variable('c') ], @@ -678,6 +728,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -687,6 +738,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('d') ], @@ -696,6 +748,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -708,6 +761,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -717,6 +771,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -726,6 +781,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -735,6 +791,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 20 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -744,6 +801,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('b') ], @@ -753,6 +811,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('d') ], @@ -762,6 +821,7 @@ describe('ActorRdfJoinMultiBind', () => { output: {}, operation: {}, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 20 }, canContainUndefs: false, variables: [ DF.variable('c') ], @@ -791,6 +851,7 @@ describe('ActorRdfJoinMultiBind', () => { ]), ], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -810,6 +871,7 @@ describe('ActorRdfJoinMultiBind', () => { ]), ], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -851,6 +913,7 @@ describe('ActorRdfJoinMultiBind', () => { ]), ]); expect(await result.metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 2.400_000_000_000_000_4 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -867,6 +930,7 @@ describe('ActorRdfJoinMultiBind', () => { expect(logSpy.mock.calls[0][2]()).toEqual({ entry: action.entries[1].operation, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -878,11 +942,13 @@ describe('ActorRdfJoinMultiBind', () => { context: new ActionContext({ a: 'b', [KeysQueryOperation.joinLeftMetadata.name]: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('a') ], }, [KeysQueryOperation.joinRightMetadatas.name]: [{ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -897,11 +963,13 @@ describe('ActorRdfJoinMultiBind', () => { context: new ActionContext({ a: 'b', [KeysQueryOperation.joinLeftMetadata.name]: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('a') ], }, [KeysQueryOperation.joinRightMetadatas.name]: [{ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -942,6 +1010,7 @@ describe('ActorRdfJoinMultiBind', () => { ]), ], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -961,6 +1030,7 @@ describe('ActorRdfJoinMultiBind', () => { ]), ], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -1002,6 +1072,7 @@ describe('ActorRdfJoinMultiBind', () => { ]), ]); expect(await result.metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 2.400_000_000_000_000_4 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -1026,6 +1097,7 @@ describe('ActorRdfJoinMultiBind', () => { ]), ], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -1045,6 +1117,7 @@ describe('ActorRdfJoinMultiBind', () => { ]), ], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -1087,6 +1160,7 @@ describe('ActorRdfJoinMultiBind', () => { ]), ]); expect(await result.metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 2.400_000_000_000_000_4 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -1112,6 +1186,7 @@ describe('ActorRdfJoinMultiBind', () => { ]), ], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -1134,6 +1209,7 @@ describe('ActorRdfJoinMultiBind', () => { ]), ], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 4 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('c') ], @@ -1153,6 +1229,7 @@ describe('ActorRdfJoinMultiBind', () => { ]), ], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -1194,6 +1271,7 @@ describe('ActorRdfJoinMultiBind', () => { ]), ]); expect(await result.metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 9.600_000_000_000_001 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b'), DF.variable('c') ], @@ -1204,6 +1282,7 @@ describe('ActorRdfJoinMultiBind', () => { expect(logSpy.mock.calls[0][2]()).toEqual({ entry: action.entries[2].operation, metadata: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -1218,17 +1297,20 @@ describe('ActorRdfJoinMultiBind', () => { context: new ActionContext({ a: 'b', [KeysQueryOperation.joinLeftMetadata.name]: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('a') ], }, [KeysQueryOperation.joinRightMetadatas.name]: [ { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], }, { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 4 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('c') ], @@ -1247,17 +1329,20 @@ describe('ActorRdfJoinMultiBind', () => { context: new ActionContext({ a: 'b', [KeysQueryOperation.joinLeftMetadata.name]: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 1 }, canContainUndefs: false, variables: [ DF.variable('a') ], }, [KeysQueryOperation.joinRightMetadatas.name]: [ { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], }, { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 4 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('c') ], diff --git a/packages/actor-rdf-join-inner-multi-empty/lib/ActorRdfJoinMultiEmpty.ts b/packages/actor-rdf-join-inner-multi-empty/lib/ActorRdfJoinMultiEmpty.ts index be6a2711e9..8f9c1b3a55 100644 --- a/packages/actor-rdf-join-inner-multi-empty/lib/ActorRdfJoinMultiEmpty.ts +++ b/packages/actor-rdf-join-inner-multi-empty/lib/ActorRdfJoinMultiEmpty.ts @@ -1,6 +1,7 @@ import type { IActionRdfJoin, IActorRdfJoinOutputInner, IActorRdfJoinArgs } from '@comunica/bus-rdf-join'; import { ActorRdfJoin } from '@comunica/bus-rdf-join'; import type { IMediatorTypeJoinCoefficients } from '@comunica/mediatortype-join-coefficients'; +import { MetadataValidationState } from '@comunica/metadata'; import type { MetadataBindings } from '@comunica/types'; import { ArrayIterator } from 'asynciterator'; @@ -33,6 +34,7 @@ export class ActorRdfJoinMultiEmpty extends ActorRdfJoin { result: { bindingsStream: new ArrayIterator([], { autoStart: false }), metadata: async() => ({ + state: new MetadataValidationState(), cardinality: { type: 'exact', value: 0 }, canContainUndefs: false, variables: ActorRdfJoin.joinVariables(await ActorRdfJoin.getMetadatas(action.entries)), diff --git a/packages/actor-rdf-join-inner-multi-empty/package.json b/packages/actor-rdf-join-inner-multi-empty/package.json index 53920311e2..b3c97453f6 100644 --- a/packages/actor-rdf-join-inner-multi-empty/package.json +++ b/packages/actor-rdf-join-inner-multi-empty/package.json @@ -34,6 +34,7 @@ "dependencies": { "@comunica/bus-rdf-join": "^2.6.8", "@comunica/mediatortype-join-coefficients": "^2.6.8", + "@comunica/metadata": "^2.5.0", "@comunica/types": "^2.6.8", "asynciterator": "^3.8.0" }, diff --git a/packages/actor-rdf-join-inner-multi-empty/test/ActorRdfJoinMultiEmpty-test.ts b/packages/actor-rdf-join-inner-multi-empty/test/ActorRdfJoinMultiEmpty-test.ts index 0eadff1412..1aa119162a 100644 --- a/packages/actor-rdf-join-inner-multi-empty/test/ActorRdfJoinMultiEmpty-test.ts +++ b/packages/actor-rdf-join-inner-multi-empty/test/ActorRdfJoinMultiEmpty-test.ts @@ -106,7 +106,7 @@ describe('ActorRdfJoinMultiEmpty', () => { }); await expect(output.bindingsStream).toEqualBindingsStream([]); expect(await output.metadata()) - .toEqual({ cardinality: { type: 'exact', value: 0 }, canContainUndefs: false, variables: []}); + .toMatchObject({ cardinality: { type: 'exact', value: 0 }, canContainUndefs: false, variables: []}); }); }); }); diff --git a/packages/actor-rdf-join-inner-multi-sequential/test/ActorRdfJoinMultiSequential-test.ts b/packages/actor-rdf-join-inner-multi-sequential/test/ActorRdfJoinMultiSequential-test.ts index 7344ccef7b..02cd94c177 100644 --- a/packages/actor-rdf-join-inner-multi-sequential/test/ActorRdfJoinMultiSequential-test.ts +++ b/packages/actor-rdf-join-inner-multi-sequential/test/ActorRdfJoinMultiSequential-test.ts @@ -5,6 +5,7 @@ import { ActorRdfJoin } from '@comunica/bus-rdf-join'; import type { IActionRdfJoinSelectivity, IActorRdfJoinSelectivityOutput } from '@comunica/bus-rdf-join-selectivity'; import type { Actor, IActorTest, Mediator } from '@comunica/core'; import { ActionContext, Bus } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IActionContext } from '@comunica/types'; import { ArrayIterator } from 'asynciterator'; import { DataFactory } from 'rdf-data-factory'; @@ -84,6 +85,7 @@ describe('ActorRdfJoinMultiSequential', () => { ]), metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 4 }, pageSize: 100, requestTime: 10, @@ -109,6 +111,7 @@ describe('ActorRdfJoinMultiSequential', () => { ]), metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, pageSize: 100, requestTime: 20, @@ -134,6 +137,7 @@ describe('ActorRdfJoinMultiSequential', () => { ]), metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, pageSize: 100, requestTime: 30, @@ -165,6 +169,7 @@ describe('ActorRdfJoinMultiSequential', () => { ]), metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 4 }, pageSize: 100, requestTime: 10, @@ -190,6 +195,7 @@ describe('ActorRdfJoinMultiSequential', () => { ]), metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, pageSize: 100, requestTime: 20, @@ -215,6 +221,7 @@ describe('ActorRdfJoinMultiSequential', () => { ]), metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, pageSize: 100, requestTime: 30, @@ -240,6 +247,7 @@ describe('ActorRdfJoinMultiSequential', () => { ]), metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, pageSize: 100, requestTime: 40, @@ -293,6 +301,7 @@ describe('ActorRdfJoinMultiSequential', () => { const output = await actor.run(action3); expect(output.type).toEqual('bindings'); expect(await ( output).metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 40 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b'), DF.variable('c') ], @@ -320,6 +329,7 @@ describe('ActorRdfJoinMultiSequential', () => { const output = await actor.run(action4); expect(output.type).toEqual('bindings'); expect(await ( output).metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 80 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b'), DF.variable('c'), DF.variable('d') ], diff --git a/packages/actor-rdf-join-inner-multi-smallest/test/ActorRdfJoinMultiSmallest-test.ts b/packages/actor-rdf-join-inner-multi-smallest/test/ActorRdfJoinMultiSmallest-test.ts index ea5a214c6f..f8d43225de 100644 --- a/packages/actor-rdf-join-inner-multi-smallest/test/ActorRdfJoinMultiSmallest-test.ts +++ b/packages/actor-rdf-join-inner-multi-smallest/test/ActorRdfJoinMultiSmallest-test.ts @@ -6,6 +6,7 @@ import type { IActionRdfJoinEntriesSort, MediatorRdfJoinEntriesSort } from '@com import type { IActionRdfJoinSelectivity, IActorRdfJoinSelectivityOutput } from '@comunica/bus-rdf-join-selectivity'; import type { Actor, IActorTest, Mediator } from '@comunica/core'; import { ActionContext, Bus } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IActionContext } from '@comunica/types'; import { ArrayIterator } from 'asynciterator'; import { DataFactory } from 'rdf-data-factory'; @@ -96,6 +97,7 @@ describe('ActorRdfJoinMultiSmallest', () => { ]), metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 4 }, pageSize: 100, requestTime: 10, @@ -121,6 +123,7 @@ describe('ActorRdfJoinMultiSmallest', () => { ]), metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, pageSize: 100, requestTime: 20, @@ -146,6 +149,7 @@ describe('ActorRdfJoinMultiSmallest', () => { ]), metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, pageSize: 100, requestTime: 30, @@ -177,6 +181,7 @@ describe('ActorRdfJoinMultiSmallest', () => { ]), metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 4 }, pageSize: 100, requestTime: 10, @@ -202,6 +207,7 @@ describe('ActorRdfJoinMultiSmallest', () => { ]), metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, pageSize: 100, requestTime: 20, @@ -227,6 +233,7 @@ describe('ActorRdfJoinMultiSmallest', () => { ]), metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, pageSize: 100, requestTime: 30, @@ -252,6 +259,7 @@ describe('ActorRdfJoinMultiSmallest', () => { ]), metadata: () => Promise.resolve( { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, pageSize: 100, requestTime: 40, @@ -310,6 +318,7 @@ describe('ActorRdfJoinMultiSmallest', () => { const output = await actor.run(action); expect(output.type).toEqual('bindings'); expect(await ( output).metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 40 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('c'), DF.variable('b') ], @@ -338,6 +347,7 @@ describe('ActorRdfJoinMultiSmallest', () => { const output = await actor.run(action); expect(output.type).toEqual('bindings'); expect(await ( output).metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 80 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('c'), DF.variable('b'), DF.variable('d') ], diff --git a/packages/actor-rdf-join-inner-nestedloop/test/ActorRdfJoinNestedLoop-test.ts b/packages/actor-rdf-join-inner-nestedloop/test/ActorRdfJoinNestedLoop-test.ts index 2067511ae1..3c6c50516d 100644 --- a/packages/actor-rdf-join-inner-nestedloop/test/ActorRdfJoinNestedLoop-test.ts +++ b/packages/actor-rdf-join-inner-nestedloop/test/ActorRdfJoinNestedLoop-test.ts @@ -4,6 +4,7 @@ import { ActorRdfJoin } from '@comunica/bus-rdf-join'; import type { IActionRdfJoinSelectivity, IActorRdfJoinSelectivityOutput } from '@comunica/bus-rdf-join-selectivity'; import type { Actor, IActorTest, Mediator } from '@comunica/core'; import { ActionContext, Bus } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IQueryOperationResultBindings, Bindings, IActionContext } from '@comunica/types'; import type * as RDF from '@rdfjs/types'; import arrayifyStream from 'arrayify-stream'; @@ -68,6 +69,7 @@ describe('ActorRdfJoinNestedLoop', () => { output: { bindingsStream: new ArrayIterator([], { autoStart: false }), metadata: async() => ({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 4 }, pageSize: 100, requestTime: 10, @@ -82,6 +84,7 @@ describe('ActorRdfJoinNestedLoop', () => { output: { bindingsStream: new ArrayIterator([], { autoStart: false }), metadata: async() => ({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, pageSize: 100, requestTime: 20, @@ -109,6 +112,7 @@ describe('ActorRdfJoinNestedLoop', () => { it('should handle undefs in left stream', () => { action.entries[0].output.metadata = async() => ({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 4 }, pageSize: 100, requestTime: 10, @@ -126,6 +130,7 @@ describe('ActorRdfJoinNestedLoop', () => { it('should handle undefs in right stream', () => { action.entries[1].output.metadata = async() => ({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, pageSize: 100, requestTime: 20, @@ -143,6 +148,7 @@ describe('ActorRdfJoinNestedLoop', () => { it('should handle undefs in left and right stream', () => { action.entries[0].output.metadata = async() => ({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 4 }, pageSize: 100, requestTime: 10, @@ -150,6 +156,7 @@ describe('ActorRdfJoinNestedLoop', () => { variables: [], }); action.entries[1].output.metadata = async() => ({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, pageSize: 100, requestTime: 20, @@ -379,6 +386,7 @@ describe('ActorRdfJoinNestedLoop', () => { ]), ]); action.entries[1].output.metadata = async() => ({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, pageSize: 100, requestTime: 20, diff --git a/packages/actor-rdf-join-inner-none/lib/ActorRdfJoinNone.ts b/packages/actor-rdf-join-inner-none/lib/ActorRdfJoinNone.ts index 575f4ae393..463f7f90d7 100644 --- a/packages/actor-rdf-join-inner-none/lib/ActorRdfJoinNone.ts +++ b/packages/actor-rdf-join-inner-none/lib/ActorRdfJoinNone.ts @@ -2,6 +2,7 @@ import { BindingsFactory } from '@comunica/bindings-factory'; import type { IActionRdfJoin, IActorRdfJoinOutputInner, IActorRdfJoinArgs } from '@comunica/bus-rdf-join'; import { ActorRdfJoin } from '@comunica/bus-rdf-join'; import type { IMediatorTypeJoinCoefficients } from '@comunica/mediatortype-join-coefficients'; +import { MetadataValidationState } from '@comunica/metadata'; import { ArrayIterator } from 'asynciterator'; const BF = new BindingsFactory(); @@ -31,6 +32,7 @@ export class ActorRdfJoinNone extends ActorRdfJoin { result: { bindingsStream: new ArrayIterator([ BF.bindings() ], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'exact', value: 1 }, canContainUndefs: false, variables: [], diff --git a/packages/actor-rdf-join-inner-none/package.json b/packages/actor-rdf-join-inner-none/package.json index 53c58348c7..abeec5bcae 100644 --- a/packages/actor-rdf-join-inner-none/package.json +++ b/packages/actor-rdf-join-inner-none/package.json @@ -35,6 +35,7 @@ "@comunica/bindings-factory": "^2.5.1", "@comunica/bus-rdf-join": "^2.6.8", "@comunica/mediatortype-join-coefficients": "^2.6.8", + "@comunica/metadata": "^2.5.0", "asynciterator": "^3.8.0" }, "scripts": { diff --git a/packages/actor-rdf-join-inner-none/test/ActorRdfJoinNone-test.ts b/packages/actor-rdf-join-inner-none/test/ActorRdfJoinNone-test.ts index 4c9f5451d8..1b73e50107 100644 --- a/packages/actor-rdf-join-inner-none/test/ActorRdfJoinNone-test.ts +++ b/packages/actor-rdf-join-inner-none/test/ActorRdfJoinNone-test.ts @@ -69,7 +69,7 @@ describe('ActorRdfJoinNone', () => { }); await expect(output.bindingsStream).toEqualBindingsStream([ BF.bindings() ]); expect(await output.metadata()) - .toEqual({ cardinality: { type: 'exact', value: 1 }, canContainUndefs: false, variables: []}); + .toMatchObject({ cardinality: { type: 'exact', value: 1 }, canContainUndefs: false, variables: []}); }); }); }); diff --git a/packages/actor-rdf-join-inner-symmetrichash/test/ActorRdfJoinSymmetricHash-test.ts b/packages/actor-rdf-join-inner-symmetrichash/test/ActorRdfJoinSymmetricHash-test.ts index 338c4b9a71..632a20145b 100644 --- a/packages/actor-rdf-join-inner-symmetrichash/test/ActorRdfJoinSymmetricHash-test.ts +++ b/packages/actor-rdf-join-inner-symmetrichash/test/ActorRdfJoinSymmetricHash-test.ts @@ -4,6 +4,7 @@ import { ActorRdfJoin } from '@comunica/bus-rdf-join'; import type { IActionRdfJoinSelectivity, IActorRdfJoinSelectivityOutput } from '@comunica/bus-rdf-join-selectivity'; import type { Actor, IActorTest, Mediator } from '@comunica/core'; import { ActionContext, Bus } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IQueryOperationResultBindings, Bindings, IActionContext } from '@comunica/types'; import type * as RDF from '@rdfjs/types'; import arrayifyStream from 'arrayify-stream'; @@ -68,6 +69,7 @@ describe('ActorRdfJoinSymmetricHash', () => { output: { bindingsStream: new ArrayIterator([], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 4 }, canContainUndefs: false, variables: variables0, @@ -80,6 +82,7 @@ describe('ActorRdfJoinSymmetricHash', () => { output: { bindingsStream: new ArrayIterator([], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: false, variables: variables1, @@ -98,42 +101,113 @@ describe('ActorRdfJoinSymmetricHash', () => { action.entries.forEach(({ output }) => output?.bindingsStream.destroy()); }); - it('should only handle 2 streams', () => { - action.entries.push( {}); - return expect(actor.test(action)).rejects.toBeTruthy(); - }); - it('should fail on undefs in left stream', () => { - action.entries[0].output.metadata = () => Promise.resolve({ - cardinality: { type: 'estimate', value: 4 }, - canContainUndefs: true, - variables: [], - }); + action = { + type: 'inner', + entries: [ + { + output: { + bindingsStream: new ArrayIterator([], { autoStart: false }), + metadata: () => Promise.resolve({ + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 4 }, + canContainUndefs: true, + variables: [], + }), + type: 'bindings', + }, + operation: {}, + }, + { + output: { + bindingsStream: new ArrayIterator([], { autoStart: false }), + metadata: () => Promise.resolve({ + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 5 }, + canContainUndefs: false, + variables: [], + }), + type: 'bindings', + }, + operation: {}, + }, + ], + context, + }; return expect(actor.test(action)).rejects .toThrow(new Error('Actor actor can not join streams containing undefs')); }); it('should fail on undefs in right stream', () => { - action.entries[1].output.metadata = () => Promise.resolve({ - cardinality: { type: 'estimate', value: 4 }, - canContainUndefs: true, - variables: [], - }); + action = { + type: 'inner', + entries: [ + { + output: { + bindingsStream: new ArrayIterator([], { autoStart: false }), + metadata: () => Promise.resolve({ + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 4 }, + canContainUndefs: false, + variables: [], + }), + type: 'bindings', + }, + operation: {}, + }, + { + output: { + bindingsStream: new ArrayIterator([], { autoStart: false }), + metadata: () => Promise.resolve({ + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 5 }, + canContainUndefs: true, + variables: [], + }), + type: 'bindings', + }, + operation: {}, + }, + ], + context, + }; return expect(actor.test(action)).rejects .toThrow(new Error('Actor actor can not join streams containing undefs')); }); it('should fail on undefs in left and right stream', () => { - action.entries[0].output.metadata = () => Promise.resolve({ - cardinality: { type: 'estimate', value: 4 }, - canContainUndefs: true, - variables: [], - }); - action.entries[1].output.metadata = () => Promise.resolve({ - cardinality: { type: 'estimate', value: 4 }, - canContainUndefs: true, - variables: [], - }); + action = { + type: 'inner', + entries: [ + { + output: { + bindingsStream: new ArrayIterator([], { autoStart: false }), + metadata: () => Promise.resolve({ + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 4 }, + canContainUndefs: true, + variables: [], + }), + type: 'bindings', + }, + operation: {}, + }, + { + output: { + bindingsStream: new ArrayIterator([], { autoStart: false }), + metadata: () => Promise.resolve({ + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 5 }, + canContainUndefs: true, + variables: [], + }), + type: 'bindings', + }, + operation: {}, + }, + ], + context, + }; return expect(actor.test(action)).rejects .toThrow(new Error('Actor actor can not join streams containing undefs')); }); @@ -161,6 +235,7 @@ describe('ActorRdfJoinSymmetricHash', () => { it('should return an empty stream for empty input', () => { return actor.run(action).then(async(output: IQueryOperationResultBindings) => { expect(await output.metadata()).toEqual({ + state: expect.any(MetadataValidationState), canContainUndefs: false, cardinality: { type: 'estimate', value: 20 }, variables: [], @@ -189,6 +264,7 @@ describe('ActorRdfJoinSymmetricHash', () => { variables1 = [ DF.variable('a'), DF.variable('c') ]; return actor.run(action).then(async(output: IQueryOperationResultBindings) => { expect(await output.metadata()).toEqual({ + state: expect.any(MetadataValidationState), canContainUndefs: false, cardinality: { type: 'estimate', value: 20 }, variables: [ DF.variable('a'), DF.variable('b'), DF.variable('c') ], @@ -223,6 +299,7 @@ describe('ActorRdfJoinSymmetricHash', () => { variables1 = [ DF.variable('a'), DF.variable('c') ]; return actor.run(action).then(async(output: IQueryOperationResultBindings) => { expect(await output.metadata()).toEqual({ + state: expect.any(MetadataValidationState), canContainUndefs: false, cardinality: { type: 'estimate', value: 20 }, variables: [ DF.variable('a'), DF.variable('b'), DF.variable('c') ], @@ -333,6 +410,7 @@ describe('ActorRdfJoinSymmetricHash', () => { ]), ]; expect(await output.metadata()).toEqual({ + state: expect.any(MetadataValidationState), canContainUndefs: false, cardinality: { type: 'estimate', value: 20 }, variables: [ DF.variable('a'), DF.variable('b'), DF.variable('c') ], diff --git a/packages/actor-rdf-join-optional-bind/test/ActorRdfJoinOptionalBind-test.ts b/packages/actor-rdf-join-optional-bind/test/ActorRdfJoinOptionalBind-test.ts index 24e40029a1..6c0920894b 100644 --- a/packages/actor-rdf-join-optional-bind/test/ActorRdfJoinOptionalBind-test.ts +++ b/packages/actor-rdf-join-optional-bind/test/ActorRdfJoinOptionalBind-test.ts @@ -5,6 +5,7 @@ import type { IActionRdfJoinSelectivity, IActorRdfJoinSelectivityOutput } from ' import { KeysQueryOperation } from '@comunica/context-entries'; import type { Actor, IActorTest, Mediator } from '@comunica/core'; import { ActionContext, Bus } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IQueryOperationResultBindings, Bindings, IActionContext } from '@comunica/types'; import { ArrayIterator } from 'asynciterator'; import { DataFactory } from 'rdf-data-factory'; @@ -61,6 +62,7 @@ describe('ActorRdfJoinOptionalBind', () => { return { bindingsStream: new ArrayIterator(data, { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: data.length }, canContainUndefs: false, variables: [ DF.variable('bound') ], @@ -98,6 +100,7 @@ describe('ActorRdfJoinOptionalBind', () => { }, [ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, pageSize: 100, requestTime: 10, @@ -105,6 +108,7 @@ describe('ActorRdfJoinOptionalBind', () => { variables: [ DF.variable('a') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, pageSize: 100, requestTime: 20, @@ -138,6 +142,7 @@ describe('ActorRdfJoinOptionalBind', () => { }, [ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, pageSize: 100, requestTime: 10, @@ -145,6 +150,7 @@ describe('ActorRdfJoinOptionalBind', () => { variables: [ DF.variable('a'), DF.variable('b'), DF.variable('d') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, pageSize: 100, requestTime: 20, @@ -178,6 +184,7 @@ describe('ActorRdfJoinOptionalBind', () => { }, [ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, pageSize: 100, requestTime: 10, @@ -185,6 +192,7 @@ describe('ActorRdfJoinOptionalBind', () => { variables: [ DF.variable('a') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, pageSize: 100, requestTime: 20, @@ -213,6 +221,7 @@ describe('ActorRdfJoinOptionalBind', () => { }, [ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, pageSize: 100, requestTime: 10, @@ -220,6 +229,7 @@ describe('ActorRdfJoinOptionalBind', () => { variables: [ DF.variable('a') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, pageSize: 100, requestTime: 20, @@ -248,6 +258,7 @@ describe('ActorRdfJoinOptionalBind', () => { }, [ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, pageSize: 100, requestTime: 10, @@ -255,6 +266,7 @@ describe('ActorRdfJoinOptionalBind', () => { variables: [ DF.variable('a') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, pageSize: 100, requestTime: 20, @@ -286,6 +298,7 @@ describe('ActorRdfJoinOptionalBind', () => { BF.bindings([[ DF.variable('a'), DF.literal('3') ]]), ], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -312,6 +325,7 @@ describe('ActorRdfJoinOptionalBind', () => { ]), ], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -344,6 +358,7 @@ describe('ActorRdfJoinOptionalBind', () => { ]), ]); expect(await result.metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 7.2 }, canContainUndefs: true, variables: [ DF.variable('a'), DF.variable('b') ], @@ -356,11 +371,13 @@ describe('ActorRdfJoinOptionalBind', () => { context: new ActionContext({ a: 'b', [KeysQueryOperation.joinLeftMetadata.name]: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], }, [KeysQueryOperation.joinRightMetadatas.name]: [{ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -373,11 +390,13 @@ describe('ActorRdfJoinOptionalBind', () => { context: new ActionContext({ a: 'b', [KeysQueryOperation.joinLeftMetadata.name]: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], }, [KeysQueryOperation.joinRightMetadatas.name]: [{ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -390,11 +409,13 @@ describe('ActorRdfJoinOptionalBind', () => { context: new ActionContext({ a: 'b', [KeysQueryOperation.joinLeftMetadata.name]: { + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], }, [KeysQueryOperation.joinRightMetadatas.name]: [{ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], diff --git a/packages/actor-rdf-join-optional-nestedloop/test/ActorRdfJoinOptionalNestedLoop-test.ts b/packages/actor-rdf-join-optional-nestedloop/test/ActorRdfJoinOptionalNestedLoop-test.ts index ea654989dd..deafc9ae5b 100644 --- a/packages/actor-rdf-join-optional-nestedloop/test/ActorRdfJoinOptionalNestedLoop-test.ts +++ b/packages/actor-rdf-join-optional-nestedloop/test/ActorRdfJoinOptionalNestedLoop-test.ts @@ -3,6 +3,7 @@ import type { IActionRdfJoin } from '@comunica/bus-rdf-join'; import type { IActionRdfJoinSelectivity, IActorRdfJoinSelectivityOutput } from '@comunica/bus-rdf-join-selectivity'; import type { Actor, IActorTest, Mediator } from '@comunica/core'; import { ActionContext, Bus } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IActionContext } from '@comunica/types'; import { ArrayIterator } from 'asynciterator'; import { DataFactory } from 'rdf-data-factory'; @@ -115,6 +116,7 @@ describe('ActorRdfJoinOptionalNestedLoop', () => { BF.bindings([[ DF.variable('a'), DF.literal('3') ]]), ], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a') ], @@ -140,6 +142,7 @@ describe('ActorRdfJoinOptionalNestedLoop', () => { ]), ], { autoStart: false }), metadata: () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 3 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -157,6 +160,7 @@ describe('ActorRdfJoinOptionalNestedLoop', () => { expect(result.type).toEqual('bindings'); expect(await result.metadata()) .toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 9 }, canContainUndefs: true, variables: [ DF.variable('a'), DF.variable('b') ], diff --git a/packages/actor-rdf-metadata-accumulate-cancontainundefs/.npmignore b/packages/actor-rdf-metadata-accumulate-cancontainundefs/.npmignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/actor-rdf-metadata-accumulate-cancontainundefs/README.md b/packages/actor-rdf-metadata-accumulate-cancontainundefs/README.md new file mode 100644 index 0000000000..3f369ec123 --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-cancontainundefs/README.md @@ -0,0 +1,36 @@ +# Comunica CanContainUndefs RDF Metadata Accumulate Actor + +[![npm version](https://badge.fury.io/js/%40comunica%2Factor-rdf-metadata-accumulate-cancontainundefs.svg)](https://www.npmjs.com/package/@comunica/actor-rdf-metadata-accumulate-cancontainundefs) + +An [RDF Metadata Accumulate](https://github.com/comunica/comunica/tree/master/packages/bus-rdf-metadata-accumulate) actor that +handles the `canContainUndefs` field. + +This module is part of the [Comunica framework](https://github.com/comunica/comunica), +and should only be used by [developers that want to build their own query engine](https://comunica.dev/docs/modify/). + +[Click here if you just want to query with Comunica](https://comunica.dev/docs/query/). + +## Install + +```bash +$ yarn add @comunica/actor-rdf-metadata-accumulate-cancontainundefs +``` + +## Configure + +After installing, this package can be added to your engine's configuration as follows: +```text +{ + "@context": [ + ... + "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/actor-rdf-metadata-accumulate-cancontainundefs/^1.0.0/components/context.jsonld" + ], + "actors": [ + ... + { + "@id": "urn:comunica:default:rdf-metadata-accumulate/actors#cancontainundefs", + "@type": "ActorRdfMetadataAccumulateCanContainUndefs" + } + ] +} +``` diff --git a/packages/actor-rdf-metadata-accumulate-cancontainundefs/lib/ActorRdfMetadataAccumulateCanContainUndefs.ts b/packages/actor-rdf-metadata-accumulate-cancontainundefs/lib/ActorRdfMetadataAccumulateCanContainUndefs.ts new file mode 100644 index 0000000000..be6e8485c6 --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-cancontainundefs/lib/ActorRdfMetadataAccumulateCanContainUndefs.ts @@ -0,0 +1,31 @@ +import type { IActionRdfMetadataAccumulate, IActorRdfMetadataAccumulateOutput, + IActorRdfMetadataAccumulateArgs } from '@comunica/bus-rdf-metadata-accumulate'; +import { ActorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate'; +import type { IActorTest } from '@comunica/core'; + +/** + * A comunica CanContainUndefs RDF Metadata Accumulate Actor. + */ +export class ActorRdfMetadataAccumulateCanContainUndefs extends ActorRdfMetadataAccumulate { + public constructor(args: IActorRdfMetadataAccumulateArgs) { + super(args); + } + + public async test(action: IActionRdfMetadataAccumulate): Promise { + return true; + } + + public async run(action: IActionRdfMetadataAccumulate): Promise { + // Return default value on initialize + if (action.mode === 'initialize') { + return { metadata: { canContainUndefs: false }}; + } + + // Otherwise, attempt to increment existing value + let canContainUndefs = action.accumulatedMetadata.canContainUndefs; + if (action.appendingMetadata.canContainUndefs) { + canContainUndefs = true; + } + return { metadata: { canContainUndefs }}; + } +} diff --git a/packages/actor-rdf-metadata-accumulate-cancontainundefs/lib/index.ts b/packages/actor-rdf-metadata-accumulate-cancontainundefs/lib/index.ts new file mode 100644 index 0000000000..7d08be7cbe --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-cancontainundefs/lib/index.ts @@ -0,0 +1 @@ +export * from './ActorRdfMetadataAccumulateCanContainUndefs'; diff --git a/packages/actor-rdf-metadata-accumulate-cancontainundefs/package.json b/packages/actor-rdf-metadata-accumulate-cancontainundefs/package.json new file mode 100644 index 0000000000..90760f1ebe --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-cancontainundefs/package.json @@ -0,0 +1,43 @@ +{ + "name": "@comunica/actor-rdf-metadata-accumulate-cancontainundefs", + "version": "2.5.0", + "description": "A cancontainundefs rdf-metadata-accumulate actor", + "lsd:module": true, + "main": "lib/index.js", + "typings": "lib/index", + "repository": { + "type": "git", + "url": "https://github.com/comunica/comunica.git", + "directory": "packages/actor-rdf-metadata-accumulate-cancontainundefs" + }, + "publishConfig": { + "access": "public" + }, + "sideEffects": false, + "keywords": [ + "comunica", + "actor", + "rdf-metadata-accumulate", + "cancontainundefs" + ], + "license": "MIT", + "bugs": { + "url": "https://github.com/comunica/comunica/issues" + }, + "homepage": "https://comunica.dev/", + "files": [ + "components", + "lib/**/*.d.ts", + "lib/**/*.js", + "lib/**/*.js.map" + ], + "dependencies": { + "@comunica/core": "^2.5.0", + "@comunica/bus-rdf-metadata-accumulate": "^2.5.0" + }, + "scripts": { + "build": "npm run build:ts && npm run build:components", + "build:ts": "node \"../../node_modules/typescript/bin/tsc\"", + "build:components": "componentsjs-generator" + } +} diff --git a/packages/actor-rdf-metadata-accumulate-cancontainundefs/test/ActorRdfMetadataAccumulateCanContainUndefs-test.ts b/packages/actor-rdf-metadata-accumulate-cancontainundefs/test/ActorRdfMetadataAccumulateCanContainUndefs-test.ts new file mode 100644 index 0000000000..363dc265ba --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-cancontainundefs/test/ActorRdfMetadataAccumulateCanContainUndefs-test.ts @@ -0,0 +1,67 @@ +import { ActionContext, Bus } from '@comunica/core'; +import type { IActionContext } from '@comunica/types'; +import { ActorRdfMetadataAccumulateCanContainUndefs } from '../lib/ActorRdfMetadataAccumulateCanContainUndefs'; + +describe('ActorRdfMetadataAccumulateCanContainUndefs', () => { + let bus: any; + let context: IActionContext; + + beforeEach(() => { + bus = new Bus({ name: 'bus' }); + context = new ActionContext(); + }); + + describe('An ActorRdfMetadataAccumulateCanContainUndefs instance', () => { + let actor: ActorRdfMetadataAccumulateCanContainUndefs; + + beforeEach(() => { + actor = new ActorRdfMetadataAccumulateCanContainUndefs({ name: 'actor', bus }); + }); + + describe('test', () => { + it('should always pass', async() => { + await expect(actor.test({ context, mode: 'initialize' })).resolves.toBeTruthy(); + }); + }); + + describe('run', () => { + it('should handle initialization', async() => { + expect(await actor.run({ context, mode: 'initialize' })) + .toEqual({ metadata: { canContainUndefs: false }}); + }); + + it('should handle appending with two false entries', async() => { + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { canContainUndefs: false }, + appendingMetadata: { canContainUndefs: false }, + })).toEqual({ metadata: { canContainUndefs: false }}); + }); + + it('should handle appending with one true entry', async() => { + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { canContainUndefs: true }, + appendingMetadata: { canContainUndefs: false }, + })).toEqual({ metadata: { canContainUndefs: true }}); + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { canContainUndefs: false }, + appendingMetadata: { canContainUndefs: true }, + })).toEqual({ metadata: { canContainUndefs: true }}); + }); + + it('should handle appending with two true entries', async() => { + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { canContainUndefs: true }, + appendingMetadata: { canContainUndefs: true }, + })).toEqual({ metadata: { canContainUndefs: true }}); + }); + }); + }); +}); diff --git a/packages/actor-rdf-metadata-accumulate-cardinality/.npmignore b/packages/actor-rdf-metadata-accumulate-cardinality/.npmignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/actor-rdf-metadata-accumulate-cardinality/README.md b/packages/actor-rdf-metadata-accumulate-cardinality/README.md new file mode 100644 index 0000000000..fce29da967 --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-cardinality/README.md @@ -0,0 +1,36 @@ +# Comunica Cardinality RDF Metadata Accumulate Actor + +[![npm version](https://badge.fury.io/js/%40comunica%2Factor-rdf-metadata-accumulate-cardinality.svg)](https://www.npmjs.com/package/@comunica/actor-rdf-metadata-accumulate-cardinality) + +An [RDF Metadata Accumulate](https://github.com/comunica/comunica/tree/master/packages/bus-rdf-metadata-accumulate) actor that +handles the `cardinality` field. + +This module is part of the [Comunica framework](https://github.com/comunica/comunica), +and should only be used by [developers that want to build their own query engine](https://comunica.dev/docs/modify/). + +[Click here if you just want to query with Comunica](https://comunica.dev/docs/query/). + +## Install + +```bash +$ yarn add @comunica/actor-rdf-metadata-accumulate-cardinality +``` + +## Configure + +After installing, this package can be added to your engine's configuration as follows: +```text +{ + "@context": [ + ... + "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/actor-rdf-metadata-accumulate-cardinality/^1.0.0/components/context.jsonld" + ], + "actors": [ + ... + { + "@id": "urn:comunica:default:rdf-metadata-accumulate/actors#cardinality", + "@type": "ActorRdfMetadataAccumulateCardinality" + } + ] +} +``` diff --git a/packages/actor-rdf-metadata-accumulate-cardinality/lib/ActorRdfMetadataAccumulateCardinality.ts b/packages/actor-rdf-metadata-accumulate-cardinality/lib/ActorRdfMetadataAccumulateCardinality.ts new file mode 100644 index 0000000000..6517ba91e4 --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-cardinality/lib/ActorRdfMetadataAccumulateCardinality.ts @@ -0,0 +1,66 @@ +import type { IActionRdfMetadataAccumulate, IActorRdfMetadataAccumulateOutput, + IActorRdfMetadataAccumulateArgs } from '@comunica/bus-rdf-metadata-accumulate'; +import { ActorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate'; +import type { IActorTest } from '@comunica/core'; +import type { QueryResultCardinality } from '@comunica/types'; + +/** + * A comunica Cardinality RDF Metadata Accumulate Actor. + */ +export class ActorRdfMetadataAccumulateCardinality extends ActorRdfMetadataAccumulate { + public constructor(args: IActorRdfMetadataAccumulateArgs) { + super(args); + } + + public async test(action: IActionRdfMetadataAccumulate): Promise { + return true; + } + + public async run(action: IActionRdfMetadataAccumulate): Promise { + // Return default value on initialize + if (action.mode === 'initialize') { + return { metadata: { cardinality: { type: 'exact', value: 0 }}}; + } + + // Otherwise, attempt to update existing value + const cardinality: QueryResultCardinality = { ...action.accumulatedMetadata.cardinality }; + + if (cardinality.dataset) { + if (action.appendingMetadata.cardinality.dataset) { + // If the accumulated cardinality is dataset-wide + if (cardinality.dataset !== action.appendingMetadata.cardinality.dataset && + action.appendingMetadata.subsetOf === cardinality.dataset) { + // If the appending cardinality refers to the subset of a dataset, + // use the cardinality of the subset. + return { metadata: { cardinality: action.appendingMetadata.cardinality }}; + } + if (cardinality.dataset !== action.appendingMetadata.cardinality.dataset) { + // If the appending cardinality refers to another dataset, + // remove the dataset scopes. + delete cardinality.dataset; + } else { + // If the appending cardinality is for the same dataset, + // keep the accumulated cardinality unchanged. + return { metadata: { cardinality }}; + } + } else { + // If the appending cardinality refers to a dataset subset, + // keep the accumulated cardinality unchanged. + return { metadata: { cardinality }}; + } + } + + if (!action.appendingMetadata.cardinality || !Number.isFinite(action.appendingMetadata.cardinality.value)) { + // We're already at infinite, so ignore any later metadata + cardinality.type = 'estimate'; + cardinality.value = Number.POSITIVE_INFINITY; + } else { + if (action.appendingMetadata.cardinality.type === 'estimate') { + cardinality.type = 'estimate'; + } + cardinality.value += action.appendingMetadata.cardinality.value; + } + + return { metadata: { cardinality }}; + } +} diff --git a/packages/actor-rdf-metadata-accumulate-cardinality/lib/index.ts b/packages/actor-rdf-metadata-accumulate-cardinality/lib/index.ts new file mode 100644 index 0000000000..4a91c11d77 --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-cardinality/lib/index.ts @@ -0,0 +1 @@ +export * from './ActorRdfMetadataAccumulateCardinality'; diff --git a/packages/actor-rdf-metadata-accumulate-cardinality/package.json b/packages/actor-rdf-metadata-accumulate-cardinality/package.json new file mode 100644 index 0000000000..9091718101 --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-cardinality/package.json @@ -0,0 +1,44 @@ +{ + "name": "@comunica/actor-rdf-metadata-accumulate-cardinality", + "version": "2.5.0", + "description": "A cardinality rdf-metadata-accumulate actor", + "lsd:module": true, + "main": "lib/index.js", + "typings": "lib/index", + "repository": { + "type": "git", + "url": "https://github.com/comunica/comunica.git", + "directory": "packages/actor-rdf-metadata-accumulate-cardinality" + }, + "publishConfig": { + "access": "public" + }, + "sideEffects": false, + "keywords": [ + "comunica", + "actor", + "rdf-metadata-accumulate", + "cardinality" + ], + "license": "MIT", + "bugs": { + "url": "https://github.com/comunica/comunica/issues" + }, + "homepage": "https://comunica.dev/", + "files": [ + "components", + "lib/**/*.d.ts", + "lib/**/*.js", + "lib/**/*.js.map" + ], + "dependencies": { + "@comunica/core": "^2.5.0", + "@comunica/bus-rdf-metadata-accumulate": "^2.5.0", + "@comunica/types": "^2.5.0" + }, + "scripts": { + "build": "npm run build:ts && npm run build:components", + "build:ts": "node \"../../node_modules/typescript/bin/tsc\"", + "build:components": "componentsjs-generator" + } +} diff --git a/packages/actor-rdf-metadata-accumulate-cardinality/test/ActorRdfMetadataAccumulateCardinality-test.ts b/packages/actor-rdf-metadata-accumulate-cardinality/test/ActorRdfMetadataAccumulateCardinality-test.ts new file mode 100644 index 0000000000..9438eea721 --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-cardinality/test/ActorRdfMetadataAccumulateCardinality-test.ts @@ -0,0 +1,121 @@ +import { ActionContext, Bus } from '@comunica/core'; +import type { IActionContext } from '@comunica/types'; +import { ActorRdfMetadataAccumulateCardinality } from '../lib/ActorRdfMetadataAccumulateCardinality'; + +describe('ActorRdfMetadataAccumulateCardinality', () => { + let bus: any; + let context: IActionContext; + + beforeEach(() => { + bus = new Bus({ name: 'bus' }); + context = new ActionContext(); + }); + + describe('An ActorRdfMetadataAccumulateCardinality instance', () => { + let actor: ActorRdfMetadataAccumulateCardinality; + + beforeEach(() => { + actor = new ActorRdfMetadataAccumulateCardinality({ name: 'actor', bus }); + }); + + describe('test', () => { + it('should always pass', async() => { + await expect(actor.test({ context, mode: 'initialize' })).resolves.toBeTruthy(); + }); + }); + + describe('run', () => { + it('should handle initialization', async() => { + expect(await actor.run({ context, mode: 'initialize' })) + .toEqual({ metadata: { cardinality: { type: 'exact', value: 0 }}}); + }); + + it('should handle appending with exact cardinalities', async() => { + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { cardinality: { type: 'exact', value: 2 }}, + appendingMetadata: { cardinality: { type: 'exact', value: 3 }}, + })).toEqual({ metadata: { cardinality: { type: 'exact', value: 5 }}}); + }); + + it('should handle appending with estimate cardinalities', async() => { + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { cardinality: { type: 'estimate', value: 2 }}, + appendingMetadata: { cardinality: { type: 'estimate', value: 3 }}, + })).toEqual({ metadata: { cardinality: { type: 'estimate', value: 5 }}}); + }); + + it('should handle appending with exact and estimate cardinalities', async() => { + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { cardinality: { type: 'exact', value: 2 }}, + appendingMetadata: { cardinality: { type: 'estimate', value: 3 }}, + })).toEqual({ metadata: { cardinality: { type: 'estimate', value: 5 }}}); + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { cardinality: { type: 'estimate', value: 2 }}, + appendingMetadata: { cardinality: { type: 'exact', value: 3 }}, + })).toEqual({ metadata: { cardinality: { type: 'estimate', value: 5 }}}); + }); + + it('should handle appending with undefined cardinality', async() => { + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { cardinality: { type: 'estimate', value: 2 }}, + appendingMetadata: {}, + })).toEqual({ metadata: { cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }}}); + }); + + it('should handle appending with infinite cardinality', async() => { + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { cardinality: { type: 'estimate', value: 2 }}, + appendingMetadata: { cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }}, + })).toEqual({ metadata: { cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }}}); + }); + + it('should handle appending with dataset-wide cardinality', async() => { + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { cardinality: { type: 'estimate', value: 200, dataset: 'abc' }}, + appendingMetadata: { cardinality: { type: 'estimate', value: 3 }}, + })).toEqual({ metadata: { cardinality: { type: 'estimate', value: 200, dataset: 'abc' }}}); + }); + + it('should handle appending with the same dataset-wide cardinality', async() => { + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { cardinality: { type: 'estimate', value: 200, dataset: 'abc' }}, + appendingMetadata: { cardinality: { type: 'estimate', value: 3, dataset: 'abc' }}, + })).toEqual({ metadata: { cardinality: { type: 'estimate', value: 200, dataset: 'abc' }}}); + }); + + it('should handle appending with different dataset-wide cardinalities', async() => { + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { cardinality: { type: 'estimate', value: 200, dataset: 'abc' }}, + appendingMetadata: { cardinality: { type: 'estimate', value: 3, dataset: 'def' }}, + })).toEqual({ metadata: { cardinality: { type: 'estimate', value: 203 }}}); + }); + + it('should handle appending with different dataset-wide cardinalities that are subsets', async() => { + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { cardinality: { type: 'estimate', value: 200, dataset: 'abc' }}, + appendingMetadata: { cardinality: { type: 'estimate', value: 3, dataset: 'def' }, subsetOf: 'abc' }, + })).toEqual({ metadata: { cardinality: { type: 'estimate', value: 3, dataset: 'def' }}}); + }); + }); + }); +}); diff --git a/packages/actor-rdf-metadata-accumulate-pagesize/.npmignore b/packages/actor-rdf-metadata-accumulate-pagesize/.npmignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/actor-rdf-metadata-accumulate-pagesize/README.md b/packages/actor-rdf-metadata-accumulate-pagesize/README.md new file mode 100644 index 0000000000..86f1821371 --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-pagesize/README.md @@ -0,0 +1,36 @@ +# Comunica PageSize RDF Metadata Accumulate Actor + +[![npm version](https://badge.fury.io/js/%40comunica%2Factor-rdf-metadata-accumulate-pagesize.svg)](https://www.npmjs.com/package/@comunica/actor-rdf-metadata-accumulate-pagesize) + +An [RDF Metadata Accumulate](https://github.com/comunica/comunica/tree/master/packages/bus-rdf-metadata-accumulate) actor that +handles the `pageSize` field. + +This module is part of the [Comunica framework](https://github.com/comunica/comunica), +and should only be used by [developers that want to build their own query engine](https://comunica.dev/docs/modify/). + +[Click here if you just want to query with Comunica](https://comunica.dev/docs/query/). + +## Install + +```bash +$ yarn add @comunica/actor-rdf-metadata-accumulate-pagesize +``` + +## Configure + +After installing, this package can be added to your engine's configuration as follows: +```text +{ + "@context": [ + ... + "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/actor-rdf-metadata-accumulate-pagesize/^1.0.0/components/context.jsonld" + ], + "actors": [ + ... + { + "@id": "urn:comunica:default:rdf-metadata-accumulate/actors#pagesize", + "@type": "ActorRdfMetadataAccumulatePageSize" + } + ] +} +``` diff --git a/packages/actor-rdf-metadata-accumulate-pagesize/lib/ActorRdfMetadataAccumulatePageSize.ts b/packages/actor-rdf-metadata-accumulate-pagesize/lib/ActorRdfMetadataAccumulatePageSize.ts new file mode 100644 index 0000000000..93db6fc5a0 --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-pagesize/lib/ActorRdfMetadataAccumulatePageSize.ts @@ -0,0 +1,35 @@ +import type { IActionRdfMetadataAccumulate, IActorRdfMetadataAccumulateOutput, + IActorRdfMetadataAccumulateArgs } from '@comunica/bus-rdf-metadata-accumulate'; +import { ActorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate'; +import type { IActorTest } from '@comunica/core'; + +/** + * A comunica PageSize RDF Metadata Accumulate Actor. + */ +export class ActorRdfMetadataAccumulatePageSize extends ActorRdfMetadataAccumulate { + public constructor(args: IActorRdfMetadataAccumulateArgs) { + super(args); + } + + public async test(action: IActionRdfMetadataAccumulate): Promise { + return true; + } + + public async run(action: IActionRdfMetadataAccumulate): Promise { + // Return nothing on initialize + if (action.mode === 'initialize') { + return { metadata: {}}; + } + + // Otherwise, attempt to increment existing value + return { + metadata: { + ...('pageSize' in action.accumulatedMetadata) || ('pageSize' in action.appendingMetadata) ? + { + pageSize: (action.accumulatedMetadata.pageSize || 0) + (action.appendingMetadata.pageSize || 0), + } : + {}, + }, + }; + } +} diff --git a/packages/actor-rdf-metadata-accumulate-pagesize/lib/index.ts b/packages/actor-rdf-metadata-accumulate-pagesize/lib/index.ts new file mode 100644 index 0000000000..9474bf9902 --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-pagesize/lib/index.ts @@ -0,0 +1 @@ +export * from './ActorRdfMetadataAccumulatePageSize'; diff --git a/packages/actor-rdf-metadata-accumulate-pagesize/package.json b/packages/actor-rdf-metadata-accumulate-pagesize/package.json new file mode 100644 index 0000000000..1204c2ad3a --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-pagesize/package.json @@ -0,0 +1,43 @@ +{ + "name": "@comunica/actor-rdf-metadata-accumulate-pagesize", + "version": "2.5.0", + "description": "A pagesize rdf-metadata-accumulate actor", + "lsd:module": true, + "main": "lib/index.js", + "typings": "lib/index", + "repository": { + "type": "git", + "url": "https://github.com/comunica/comunica.git", + "directory": "packages/actor-rdf-metadata-accumulate-pagesize" + }, + "publishConfig": { + "access": "public" + }, + "sideEffects": false, + "keywords": [ + "comunica", + "actor", + "rdf-metadata-accumulate", + "pagesize" + ], + "license": "MIT", + "bugs": { + "url": "https://github.com/comunica/comunica/issues" + }, + "homepage": "https://comunica.dev/", + "files": [ + "components", + "lib/**/*.d.ts", + "lib/**/*.js", + "lib/**/*.js.map" + ], + "dependencies": { + "@comunica/core": "^2.5.0", + "@comunica/bus-rdf-metadata-accumulate": "^2.5.0" + }, + "scripts": { + "build": "npm run build:ts && npm run build:components", + "build:ts": "node \"../../node_modules/typescript/bin/tsc\"", + "build:components": "componentsjs-generator" + } +} diff --git a/packages/actor-rdf-metadata-accumulate-pagesize/test/ActorRdfMetadataAccumulatePageSize-test.ts b/packages/actor-rdf-metadata-accumulate-pagesize/test/ActorRdfMetadataAccumulatePageSize-test.ts new file mode 100644 index 0000000000..631c9665fd --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-pagesize/test/ActorRdfMetadataAccumulatePageSize-test.ts @@ -0,0 +1,64 @@ +import { ActionContext, Bus } from '@comunica/core'; +import type { IActionContext } from '@comunica/types'; +import { ActorRdfMetadataAccumulatePageSize } from '../lib/ActorRdfMetadataAccumulatePageSize'; + +describe('ActorRdfMetadataAccumulatePageSize', () => { + let bus: any; + let context: IActionContext; + + beforeEach(() => { + bus = new Bus({ name: 'bus' }); + context = new ActionContext(); + }); + + describe('An ActorRdfMetadataAccumulatePageSize instance', () => { + let actor: ActorRdfMetadataAccumulatePageSize; + + beforeEach(() => { + actor = new ActorRdfMetadataAccumulatePageSize({ name: 'actor', bus }); + }); + + describe('test', () => { + it('should always pass', async() => { + await expect(actor.test({ context, mode: 'initialize' })).resolves.toBeTruthy(); + }); + }); + + describe('run', () => { + it('should handle initialization', async() => { + expect(await actor.run({ context, mode: 'initialize' })) + .toEqual({ metadata: {}}); + }); + + it('should handle appending with two entries', async() => { + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { pageSize: 2 }, + appendingMetadata: { pageSize: 3 }, + })).toEqual({ metadata: { pageSize: 5 }}); + }); + + it('should handle appending with undefined entries', async() => { + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: {}, + appendingMetadata: { pageSize: 3 }, + })).toEqual({ metadata: { pageSize: 3 }}); + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { pageSize: 2 }, + appendingMetadata: {}, + })).toEqual({ metadata: { pageSize: 2 }}); + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: {}, + appendingMetadata: {}, + })).toEqual({ metadata: {}}); + }); + }); + }); +}); diff --git a/packages/actor-rdf-metadata-accumulate-requesttime/.npmignore b/packages/actor-rdf-metadata-accumulate-requesttime/.npmignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/actor-rdf-metadata-accumulate-requesttime/README.md b/packages/actor-rdf-metadata-accumulate-requesttime/README.md new file mode 100644 index 0000000000..4e68418180 --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-requesttime/README.md @@ -0,0 +1,36 @@ +# Comunica RequestTime RDF Metadata Accumulate Actor + +[![npm version](https://badge.fury.io/js/%40comunica%2Factor-rdf-metadata-accumulate-requesttime.svg)](https://www.npmjs.com/package/@comunica/actor-rdf-metadata-accumulate-requesttime) + +An [RDF Metadata Accumulate](https://github.com/comunica/comunica/tree/master/packages/bus-rdf-metadata-accumulate) actor that +handles the `requestTime` field. + +This module is part of the [Comunica framework](https://github.com/comunica/comunica), +and should only be used by [developers that want to build their own query engine](https://comunica.dev/docs/modify/). + +[Click here if you just want to query with Comunica](https://comunica.dev/docs/query/). + +## Install + +```bash +$ yarn add @comunica/actor-rdf-metadata-accumulate-requesttime +``` + +## Configure + +After installing, this package can be added to your engine's configuration as follows: +```text +{ + "@context": [ + ... + "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/actor-rdf-metadata-accumulate-requesttime/^1.0.0/components/context.jsonld" + ], + "actors": [ + ... + { + "@id": "urn:comunica:default:rdf-metadata-accumulate/actors#requesttime", + "@type": "ActorRdfMetadataAccumulateRequestTime" + } + ] +} +``` diff --git a/packages/actor-rdf-metadata-accumulate-requesttime/lib/ActorRdfMetadataAccumulateRequestTime.ts b/packages/actor-rdf-metadata-accumulate-requesttime/lib/ActorRdfMetadataAccumulateRequestTime.ts new file mode 100644 index 0000000000..560a62f428 --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-requesttime/lib/ActorRdfMetadataAccumulateRequestTime.ts @@ -0,0 +1,35 @@ +import type { IActionRdfMetadataAccumulate, IActorRdfMetadataAccumulateOutput, + IActorRdfMetadataAccumulateArgs } from '@comunica/bus-rdf-metadata-accumulate'; +import { ActorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate'; +import type { IActorTest } from '@comunica/core'; + +/** + * A comunica RequestTime RDF Metadata Accumulate Actor. + */ +export class ActorRdfMetadataAccumulateRequestTime extends ActorRdfMetadataAccumulate { + public constructor(args: IActorRdfMetadataAccumulateArgs) { + super(args); + } + + public async test(action: IActionRdfMetadataAccumulate): Promise { + return true; + } + + public async run(action: IActionRdfMetadataAccumulate): Promise { + // Return nothing on initialize + if (action.mode === 'initialize') { + return { metadata: {}}; + } + + // Otherwise, attempt to increment existing value + return { + metadata: { + ...('requestTime' in action.accumulatedMetadata) || ('requestTime' in action.appendingMetadata) ? + { + requestTime: (action.accumulatedMetadata.requestTime || 0) + (action.appendingMetadata.requestTime || 0), + } : + {}, + }, + }; + } +} diff --git a/packages/actor-rdf-metadata-accumulate-requesttime/lib/index.ts b/packages/actor-rdf-metadata-accumulate-requesttime/lib/index.ts new file mode 100644 index 0000000000..87c55ea562 --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-requesttime/lib/index.ts @@ -0,0 +1 @@ +export * from './ActorRdfMetadataAccumulateRequestTime'; diff --git a/packages/actor-rdf-metadata-accumulate-requesttime/package.json b/packages/actor-rdf-metadata-accumulate-requesttime/package.json new file mode 100644 index 0000000000..93cc3f57f5 --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-requesttime/package.json @@ -0,0 +1,43 @@ +{ + "name": "@comunica/actor-rdf-metadata-accumulate-requesttime", + "version": "2.5.0", + "description": "A requesttime rdf-metadata-accumulate actor", + "lsd:module": true, + "main": "lib/index.js", + "typings": "lib/index", + "repository": { + "type": "git", + "url": "https://github.com/comunica/comunica.git", + "directory": "packages/actor-rdf-metadata-accumulate-requesttime" + }, + "publishConfig": { + "access": "public" + }, + "sideEffects": false, + "keywords": [ + "comunica", + "actor", + "rdf-metadata-accumulate", + "requesttime" + ], + "license": "MIT", + "bugs": { + "url": "https://github.com/comunica/comunica/issues" + }, + "homepage": "https://comunica.dev/", + "files": [ + "components", + "lib/**/*.d.ts", + "lib/**/*.js", + "lib/**/*.js.map" + ], + "dependencies": { + "@comunica/core": "^2.5.0", + "@comunica/bus-rdf-metadata-accumulate": "^2.5.0" + }, + "scripts": { + "build": "npm run build:ts && npm run build:components", + "build:ts": "node \"../../node_modules/typescript/bin/tsc\"", + "build:components": "componentsjs-generator" + } +} diff --git a/packages/actor-rdf-metadata-accumulate-requesttime/test/ActorRdfMetadataAccumulateRequestTime-test.ts b/packages/actor-rdf-metadata-accumulate-requesttime/test/ActorRdfMetadataAccumulateRequestTime-test.ts new file mode 100644 index 0000000000..4d1dc9caeb --- /dev/null +++ b/packages/actor-rdf-metadata-accumulate-requesttime/test/ActorRdfMetadataAccumulateRequestTime-test.ts @@ -0,0 +1,64 @@ +import { ActionContext, Bus } from '@comunica/core'; +import type { IActionContext } from '@comunica/types'; +import { ActorRdfMetadataAccumulateRequestTime } from '../lib/ActorRdfMetadataAccumulateRequestTime'; + +describe('ActorRdfMetadataAccumulateRequestTime', () => { + let bus: any; + let context: IActionContext; + + beforeEach(() => { + bus = new Bus({ name: 'bus' }); + context = new ActionContext(); + }); + + describe('An ActorRdfMetadataAccumulateRequestTime instance', () => { + let actor: ActorRdfMetadataAccumulateRequestTime; + + beforeEach(() => { + actor = new ActorRdfMetadataAccumulateRequestTime({ name: 'actor', bus }); + }); + + describe('test', () => { + it('should always pass', async() => { + await expect(actor.test({ context, mode: 'initialize' })).resolves.toBeTruthy(); + }); + }); + + describe('run', () => { + it('should handle initialization', async() => { + expect(await actor.run({ context, mode: 'initialize' })) + .toEqual({ metadata: {}}); + }); + + it('should handle appending with two entries', async() => { + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { requestTime: 2 }, + appendingMetadata: { requestTime: 3 }, + })).toEqual({ metadata: { requestTime: 5 }}); + }); + + it('should handle appending with undefined entries', async() => { + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: {}, + appendingMetadata: { requestTime: 3 }, + })).toEqual({ metadata: { requestTime: 3 }}); + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: { requestTime: 2 }, + appendingMetadata: {}, + })).toEqual({ metadata: { requestTime: 2 }}); + expect(await actor.run({ + context, + mode: 'append', + accumulatedMetadata: {}, + appendingMetadata: {}, + })).toEqual({ metadata: {}}); + }); + }); + }); +}); diff --git a/packages/actor-rdf-metadata-extract-hydra-count/lib/ActorRdfMetadataExtractHydraCount.ts b/packages/actor-rdf-metadata-extract-hydra-count/lib/ActorRdfMetadataExtractHydraCount.ts index ecd4e9316f..bd4ec68035 100644 --- a/packages/actor-rdf-metadata-extract-hydra-count/lib/ActorRdfMetadataExtractHydraCount.ts +++ b/packages/actor-rdf-metadata-extract-hydra-count/lib/ActorRdfMetadataExtractHydraCount.ts @@ -26,13 +26,21 @@ export class ActorRdfMetadataExtractHydraCount extends ActorRdfMetadataExtract // Immediately resolve when a value has been found. action.metadata.on('data', quad => { if (this.predicates.includes(quad.predicate.value)) { - resolve({ metadata: { cardinality: { type: 'estimate', value: Number.parseInt(quad.object.value, 10) }}}); + resolve({ + metadata: { + cardinality: { + type: 'estimate', + value: Number.parseInt(quad.object.value, 10), + dataset: quad.subject.value, + }, + }, + }); } }); // If no value has been found, assume infinity. action.metadata.on('end', () => { - resolve({ metadata: { cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }}}); + resolve({ metadata: { cardinality: { type: 'estimate', value: 0 }}}); }); }); } diff --git a/packages/actor-rdf-metadata-extract-hydra-count/test/ActorRdfMetadataExtractHydraCount-test.ts b/packages/actor-rdf-metadata-extract-hydra-count/test/ActorRdfMetadataExtractHydraCount-test.ts index afea65c5b6..77220255a5 100644 --- a/packages/actor-rdf-metadata-extract-hydra-count/test/ActorRdfMetadataExtractHydraCount-test.ts +++ b/packages/actor-rdf-metadata-extract-hydra-count/test/ActorRdfMetadataExtractHydraCount-test.ts @@ -57,12 +57,12 @@ describe('ActorRdfMetadataExtractHydraCount', () => { it('should run on a stream where count is given', () => { return expect(actor.run({ url: '', metadata: input, requestTime: 0, context })).resolves - .toEqual({ metadata: { cardinality: { type: 'estimate', value: 12_345 }}}); + .toEqual({ metadata: { cardinality: { type: 'estimate', value: 12_345, dataset: 'g1' }}}); }); it('should run on a stream where count is not given', () => { return expect(actor.run({ url: '', metadata: inputNone, requestTime: 0, context })).resolves - .toEqual({ metadata: { cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }}}); + .toEqual({ metadata: { cardinality: { type: 'estimate', value: 0 }}}); }); }); }); diff --git a/packages/actor-rdf-resolve-hypermedia-none/test/ActorRdfResolveHypermediaNone-test.ts b/packages/actor-rdf-resolve-hypermedia-none/test/ActorRdfResolveHypermediaNone-test.ts index cfc78614fc..da1ba85580 100644 --- a/packages/actor-rdf-resolve-hypermedia-none/test/ActorRdfResolveHypermediaNone-test.ts +++ b/packages/actor-rdf-resolve-hypermedia-none/test/ActorRdfResolveHypermediaNone-test.ts @@ -2,6 +2,7 @@ import { Readable } from 'stream'; import { ActorRdfResolveHypermedia } from '@comunica/bus-rdf-resolve-hypermedia'; import { ActionContext, Bus } from '@comunica/core'; import 'jest-rdf'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IActionContext } from '@comunica/types'; import type * as RDF from '@rdfjs/types'; import arrayifyStream from 'arrayify-stream'; @@ -64,7 +65,11 @@ describe('ActorRdfResolveHypermediaNone', () => { await expect(new Promise((resolve, reject) => { stream = source.match(v, v, v, v); stream.getProperty('metadata', resolve); - })).resolves.toEqual({ cardinality: { type: 'exact', value: 2 }, canContainUndefs: false }); + })).resolves.toEqual({ + state: expect.any(MetadataValidationState), + cardinality: { type: 'exact', value: 2 }, + canContainUndefs: false, + }); expect(await arrayifyStream(stream!)).toEqualRdfQuadArray([ quad('s1', 'p1', 'o1'), quad('s2', 'p2', 'o2'), diff --git a/packages/actor-rdf-resolve-hypermedia-qpf/lib/ActorRdfResolveHypermediaQpf.ts b/packages/actor-rdf-resolve-hypermedia-qpf/lib/ActorRdfResolveHypermediaQpf.ts index 0fa2e93c7c..48612bdd14 100644 --- a/packages/actor-rdf-resolve-hypermedia-qpf/lib/ActorRdfResolveHypermediaQpf.ts +++ b/packages/actor-rdf-resolve-hypermedia-qpf/lib/ActorRdfResolveHypermediaQpf.ts @@ -27,7 +27,7 @@ export class ActorRdfResolveHypermediaQpf extends ActorRdfResolveHypermedia } public async testMetadata(action: IActionRdfResolveHypermedia): Promise { - const { searchForm } = this.createSource(action.metadata, action.context); + const { searchForm } = this.createSource(action.url, action.metadata, action.context); if (action.handledDatasets && action.handledDatasets[searchForm.dataset]) { throw new Error(`Actor ${this.name} can only be applied for the first page of a QPF dataset.`); } @@ -41,11 +41,16 @@ export class ActorRdfResolveHypermediaQpf extends ActorRdfResolveHypermedia */ public async run(action: IActionRdfResolveHypermedia): Promise { this.logInfo(action.context, `Identified as qpf source: ${action.url}`); - const source = this.createSource(action.metadata, action.context, action.quads); + const source = this.createSource(action.url, action.metadata, action.context, action.quads); return { source, dataset: source.searchForm.dataset }; } - protected createSource(metadata: Record, context: IActionContext, quads?: RDF.Stream): RdfSourceQpf { + protected createSource( + url: string, + metadata: Record, + context: IActionContext, + quads?: RDF.Stream, + ): RdfSourceQpf { return new RdfSourceQpf( this.mediatorMetadata, this.mediatorMetadataExtract, @@ -54,6 +59,7 @@ export class ActorRdfResolveHypermediaQpf extends ActorRdfResolveHypermedia this.predicateUri, this.objectUri, this.graphUri, + url, metadata, context, quads, diff --git a/packages/actor-rdf-resolve-hypermedia-qpf/lib/RdfSourceQpf.ts b/packages/actor-rdf-resolve-hypermedia-qpf/lib/RdfSourceQpf.ts index cdeed7bc03..dac77a83e8 100644 --- a/packages/actor-rdf-resolve-hypermedia-qpf/lib/RdfSourceQpf.ts +++ b/packages/actor-rdf-resolve-hypermedia-qpf/lib/RdfSourceQpf.ts @@ -29,6 +29,7 @@ export class RdfSourceQpf implements IQuadSource { private readonly predicateUri: string; private readonly objectUri: string; private readonly graphUri?: string; + private readonly url: string; private readonly defaultGraph?: RDF.NamedNode; private readonly context: IActionContext; private readonly cachedQuads: Record>; @@ -37,6 +38,7 @@ export class RdfSourceQpf implements IQuadSource { mediatorMetadataExtract: MediatorRdfMetadataExtract, mediatorDereferenceRdf: MediatorDereferenceRdf, subjectUri: string, predicateUri: string, objectUri: string, graphUri: string | undefined, + url: string, metadata: Record, context: IActionContext, initialQuads?: RDF.Stream) { this.mediatorMetadata = mediatorMetadata; this.mediatorMetadataExtract = mediatorMetadataExtract; @@ -45,6 +47,7 @@ export class RdfSourceQpf implements IQuadSource { this.predicateUri = predicateUri; this.objectUri = objectUri; this.graphUri = graphUri; + this.url = url; this.context = context; this.cachedQuads = {}; const searchForm = this.getSearchForm(metadata); @@ -170,7 +173,7 @@ export class RdfSourceQpf implements IQuadSource { requestTime: dereferenceRdfOutput.requestTime, }) .then(({ metadata }) => quads - .setProperty('metadata', { ...metadata, canContainUndefs: false })); + .setProperty('metadata', { ...metadata, canContainUndefs: false, subsetOf: this.url })); // The server is free to send any data in its response (such as metadata), // including quads that do not match the given matter. diff --git a/packages/actor-rdf-resolve-hypermedia-qpf/test/ActorRdfResolveHypermediaQpf-test.ts b/packages/actor-rdf-resolve-hypermedia-qpf/test/ActorRdfResolveHypermediaQpf-test.ts index 8a9231de35..4051800006 100644 --- a/packages/actor-rdf-resolve-hypermedia-qpf/test/ActorRdfResolveHypermediaQpf-test.ts +++ b/packages/actor-rdf-resolve-hypermedia-qpf/test/ActorRdfResolveHypermediaQpf-test.ts @@ -102,7 +102,7 @@ describe('ActorRdfResolveHypermediaQpf', () => { it('should create an RdfSourceQpf', () => { const context = {}; const quads = empty(); - const source = actor.createSource(metadata, context, quads); + const source = actor.createSource('url', metadata, context, quads); expect(source).toBeInstanceOf(RdfSourceQpf); expect(source.mediatorMetadata).toBe(mediatorMetadata); expect(source.mediatorMetadataExtract).toBe(mediatorMetadataExtract); diff --git a/packages/actor-rdf-resolve-hypermedia-qpf/test/RdfSourceQpf-test.ts b/packages/actor-rdf-resolve-hypermedia-qpf/test/RdfSourceQpf-test.ts index d6bd066842..ac734b2267 100644 --- a/packages/actor-rdf-resolve-hypermedia-qpf/test/RdfSourceQpf-test.ts +++ b/packages/actor-rdf-resolve-hypermedia-qpf/test/RdfSourceQpf-test.ts @@ -89,6 +89,7 @@ describe('RdfSourceQpf', () => { 'p', 's', 'g', + 'url', metadata, new ActionContext(), undefined, @@ -106,6 +107,7 @@ describe('RdfSourceQpf', () => { 'p', 's', 'g', + 'url', metadata, new ActionContext(), streamifyArray([ @@ -131,6 +133,7 @@ describe('RdfSourceQpf', () => { 'p', 'o', 'g', + 'url', metadata, new ActionContext(), streamifyArray([ @@ -200,6 +203,7 @@ describe('RdfSourceQpf', () => { 'p', 'o', 'g', + 'url', metadata, new ActionContext(), streamifyArray([ @@ -238,6 +242,7 @@ describe('RdfSourceQpf', () => { 'p', 'o', 'g', + 'url', metadata, new ActionContext(), streamifyArray([ @@ -285,6 +290,7 @@ describe('RdfSourceQpf', () => { 'p', 'o', undefined, + 'url', metadata, new ActionContext(), streamifyArray([ @@ -348,7 +354,7 @@ describe('RdfSourceQpf', () => { const output = source.match(DF.namedNode('s1'), v, DF.namedNode('o1'), v); const metadataPromise = new Promise(resolve => output.getProperty('metadata', resolve)); await arrayifyStream(output); - expect(await metadataPromise).toEqual({ next: 'NEXT', canContainUndefs: false }); + expect(await metadataPromise).toEqual({ next: 'NEXT', canContainUndefs: false, subsetOf: 'url' }); }); // The following test is not applicable anymore. @@ -495,6 +501,7 @@ describe('RdfSourceQpf with a custom default graph', () => { 'p', 'o', 'g', + 'url', metadata, new ActionContext(), streamifyArray([ diff --git a/packages/actor-rdf-resolve-quad-pattern-federated/lib/ActorRdfResolveQuadPatternFederated.ts b/packages/actor-rdf-resolve-quad-pattern-federated/lib/ActorRdfResolveQuadPatternFederated.ts index 028fdb70b4..9a8d276bc2 100644 --- a/packages/actor-rdf-resolve-quad-pattern-federated/lib/ActorRdfResolveQuadPatternFederated.ts +++ b/packages/actor-rdf-resolve-quad-pattern-federated/lib/ActorRdfResolveQuadPatternFederated.ts @@ -1,3 +1,5 @@ +import type { MediatorRdfMetadataAccumulate, + IActionRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate'; import type { IActionRdfResolveQuadPattern, IActorRdfResolveQuadPatternArgs, IQuadSource, MediatorRdfResolveQuadPattern, @@ -16,12 +18,52 @@ import { FederatedQuadSource } from './FederatedQuadSource'; export class ActorRdfResolveQuadPatternFederated extends ActorRdfResolveQuadPatternSource implements IActorRdfResolveQuadPatternFederatedArgs { public readonly mediatorResolveQuadPattern: MediatorRdfResolveQuadPattern; + public readonly mediatorRdfMetadataAccumulate: MediatorRdfMetadataAccumulate; public readonly skipEmptyPatterns: boolean; protected readonly emptyPatterns: Map = new Map(); public constructor(args: IActorRdfResolveQuadPatternFederatedArgs) { super(args); + + // TODO: remove this backwards-compatibility in the next major version, and make the param mandatory + if (!args.mediatorRdfMetadataAccumulate) { + this.mediatorRdfMetadataAccumulate = { + async mediate(action: IActionRdfMetadataAccumulate) { + if (action.mode === 'initialize') { + return { metadata: { cardinality: { type: 'exact', value: 0 }, canContainUndefs: false }}; + } + + const metadata = { ...action.accumulatedMetadata }; + const subMetadata = action.appendingMetadata; + if (!subMetadata.cardinality || !Number.isFinite(subMetadata.cardinality.value)) { + // We're already at infinite, so ignore any later metadata + metadata.cardinality.type = 'estimate'; + metadata.cardinality.value = Number.POSITIVE_INFINITY; + } else { + if (subMetadata.cardinality.type === 'estimate') { + metadata.cardinality.type = 'estimate'; + } + metadata.cardinality.value += subMetadata.cardinality.value; + } + if (metadata.requestTime || subMetadata.requestTime) { + metadata.requestTime = metadata.requestTime || 0; + subMetadata.requestTime = subMetadata.requestTime || 0; + metadata.requestTime += subMetadata.requestTime; + } + if (metadata.pageSize || subMetadata.pageSize) { + metadata.pageSize = metadata.pageSize || 0; + subMetadata.pageSize = subMetadata.pageSize || 0; + metadata.pageSize += subMetadata.pageSize; + } + if (subMetadata.canContainUndefs) { + metadata.canContainUndefs = true; + } + + return { metadata }; + }, + }; + } } public async test(action: IActionRdfResolveQuadPattern): Promise { @@ -35,6 +77,7 @@ export class ActorRdfResolveQuadPatternFederated extends ActorRdfResolveQuadPatt protected async getSource(context: IActionContext): Promise { return new FederatedQuadSource( this.mediatorResolveQuadPattern, + this.mediatorRdfMetadataAccumulate, context, this.emptyPatterns, this.skipEmptyPatterns, @@ -47,6 +90,10 @@ export interface IActorRdfResolveQuadPatternFederatedArgs extends IActorRdfResol * The quad pattern resolve mediator. */ mediatorResolveQuadPattern: MediatorRdfResolveQuadPattern; + /** + * The RDF metadata accumulate mediator. + */ + mediatorRdfMetadataAccumulate?: MediatorRdfMetadataAccumulate; /** * If quad patterns that are sub-patterns of empty quad patterns should be skipped. * This assumes that sources remain static during query evaluation. diff --git a/packages/actor-rdf-resolve-quad-pattern-federated/lib/FederatedQuadSource.ts b/packages/actor-rdf-resolve-quad-pattern-federated/lib/FederatedQuadSource.ts index c91d5b6c1e..a91a393107 100644 --- a/packages/actor-rdf-resolve-quad-pattern-federated/lib/FederatedQuadSource.ts +++ b/packages/actor-rdf-resolve-quad-pattern-federated/lib/FederatedQuadSource.ts @@ -1,4 +1,5 @@ import { ClosableTransformIterator } from '@comunica/bus-query-operation'; +import type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate'; import type { IActorRdfResolveQuadPatternOutput, IQuadSource, @@ -7,6 +8,7 @@ import type { import { getDataSourceContext } from '@comunica/bus-rdf-resolve-quad-pattern'; import { KeysRdfResolveQuadPattern } from '@comunica/context-entries'; import { BlankNodeScoped } from '@comunica/data-factory'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IActionContext, DataSources, IDataSource, MetadataQuads } from '@comunica/types'; import type * as RDF from '@rdfjs/types'; import type { AsyncIterator } from 'asynciterator'; @@ -26,6 +28,7 @@ export class FederatedQuadSource implements IQuadSource { private static readonly SKOLEM_PREFIX = 'urn:comunica_skolem:source_'; protected readonly mediatorResolveQuadPattern: MediatorRdfResolveQuadPattern; + protected readonly mediatorRdfMetadataAccumulate: MediatorRdfMetadataAccumulate; protected readonly sources: DataSources; protected readonly contextDefault: IActionContext; @@ -34,10 +37,15 @@ export class FederatedQuadSource implements IQuadSource { protected readonly skipEmptyPatterns: boolean; protected readonly algebraFactory: Factory; - public constructor(mediatorResolveQuadPattern: MediatorRdfResolveQuadPattern, - context: IActionContext, emptyPatterns: Map, - skipEmptyPatterns: boolean) { + public constructor( + mediatorResolveQuadPattern: MediatorRdfResolveQuadPattern, + mediatorRdfMetadataAccumulate: MediatorRdfMetadataAccumulate, + context: IActionContext, + emptyPatterns: Map, + skipEmptyPatterns: boolean, + ) { this.mediatorResolveQuadPattern = mediatorResolveQuadPattern; + this.mediatorRdfMetadataAccumulate = mediatorRdfMetadataAccumulate; this.sources = context.get(KeysRdfResolveQuadPattern.sources)!; this.contextDefault = context.delete(KeysRdfResolveQuadPattern.sources); this.emptyPatterns = emptyPatterns; @@ -187,27 +195,39 @@ export class FederatedQuadSource implements IQuadSource { } public match(subject: RDF.Term, predicate: RDF.Term, object: RDF.Term, graph: RDF.Term): AsyncIterator { - // Counters for our metadata - const metadata: MetadataQuads = { cardinality: { type: 'exact', value: 0 }, canContainUndefs: false }; - let remainingSources = this.sources.length; - - // Anonymous function to handle cardinality from metadata - const checkEmitMetadata = (currentTotalItems: number, source: IDataSource, - pattern: RDF.BaseQuad | undefined, lastMetadata?: MetadataQuads): void => { - if (this.skipEmptyPatterns && !currentTotalItems && pattern && !this.isSourceEmpty(source, pattern)) { - this.emptyPatterns.get(source)!.push(pattern); - } - if (!remainingSources) { - if (lastMetadata && this.sources.length === 1) { - // If we only had one source, emit the metadata as-is. - it.setProperty('metadata', lastMetadata); - } else { - it.setProperty('metadata', metadata); + // A function that is called every time metadata is found, + // and will accumulate all metadata if the metadata from all sources have been defined. + const tryAccumulatingMetadata = async(): Promise => { + // Only accumulate when the metadata of all sources have been defined + if (accumulatingMetadata.size === this.sources.length) { + // Accumulate metadata using mediator + let accumulatedMetadata: MetadataQuads = (await this.mediatorRdfMetadataAccumulate + .mediate({ mode: 'initialize', context: this.contextDefault })).metadata; + for (const appendingMetadata of accumulatingMetadata.values()) { + accumulatedMetadata = { + ...appendingMetadata, + ...(await this.mediatorRdfMetadataAccumulate + .mediate({ + mode: 'append', + accumulatedMetadata, + appendingMetadata, + context: this.contextDefault, + })).metadata, + }; } + // Create new metadata state + accumulatedMetadata.state = new MetadataValidationState(); + + // Emit metadata, and invalidate any metadata that was set before + const metadataToInvalidate = it.getProperty('metadata'); + it.setProperty('metadata', accumulatedMetadata); + metadataToInvalidate?.state.invalidate(); } }; - const proxyIt: Promise[]> = Promise.all(this.sources.map(async source => { + // Immediately start sub-iterators for each source, so that their metadata can be collected. + const accumulatingMetadata: Map = new Map(); + const proxyIt: Promise[]> = Promise.all(this.sources.map(async(source, sourceIndex) => { const sourceId = this.getSourceId(source); // Deskolemize terms, so we send the original blank nodes to each source. @@ -232,40 +252,44 @@ export class FederatedQuadSource implements IQuadSource { this.isSourceEmpty(source, pattern = this.algebraFactory .createPattern(patternS, patternP, patternO, patternG))) { output = { data: new ArrayIterator([], { autoStart: false }) }; - output.data.setProperty('metadata', { cardinality: 0, canContainUndefs: false }); + // Return the default metadata + output.data.setProperty('metadata', { + state: new MetadataValidationState(), + ...(await this.mediatorRdfMetadataAccumulate + .mediate({ mode: 'initialize', context: this.contextDefault })).metadata, + }); } else { output = await this.mediatorResolveQuadPattern.mediate({ pattern, context }); } // Handle the metadata from this source - output.data.getProperty('metadata', (subMetadata: MetadataQuads) => { - if (!subMetadata.cardinality || !Number.isFinite(subMetadata.cardinality.value)) { - // We're already at infinite, so ignore any later metadata - metadata.cardinality.type = 'estimate'; - metadata.cardinality.value = Number.POSITIVE_INFINITY; - remainingSources = 0; - } else { - if (subMetadata.cardinality.type === 'estimate') { - metadata.cardinality.type = 'estimate'; + const addMetadataPropertyListener = (): void => { + output.data.getProperty('metadata', (subMetadata: MetadataQuads) => { + accumulatingMetadata.set(`${sourceIndex}`, subMetadata); + + // Save empty patterns + if (this.skipEmptyPatterns && + !subMetadata.cardinality?.value && + pattern && + !this.isSourceEmpty(source, pattern)) { + this.emptyPatterns.get(source)!.push(pattern); } - metadata.cardinality.value += subMetadata.cardinality.value; - remainingSources--; - } - if (metadata.requestTime || subMetadata.requestTime) { - metadata.requestTime = metadata.requestTime || 0; - subMetadata.requestTime = subMetadata.requestTime || 0; - metadata.requestTime += subMetadata.requestTime; - } - if (metadata.pageSize || subMetadata.pageSize) { - metadata.pageSize = metadata.pageSize || 0; - subMetadata.pageSize = subMetadata.pageSize || 0; - metadata.pageSize += subMetadata.pageSize; - } - if (subMetadata.canContainUndefs) { - metadata.canContainUndefs = true; - } - checkEmitMetadata(metadata.cardinality.value, source, pattern, subMetadata); - }); + + // Accumulate metadata + tryAccumulatingMetadata() + .catch(error => it.emit('error', error)); + + // Re-accumulate metadata if this metadata changes + subMetadata.state?.addInvalidateListener(() => { + // Remove this source's metadata in the array + accumulatingMetadata.delete(`${sourceIndex}`); + + // Listen to new metadata property changes + addMetadataPropertyListener(); + }); + }); + }; + addMetadataPropertyListener(); // Determine the data stream from this source const data = output.data.map(quad => FederatedQuadSource.skolemizeQuad(quad, sourceId)); @@ -293,7 +317,10 @@ export class FederatedQuadSource implements IQuadSource { // If we have 0 sources, immediately emit metadata if (this.sources.length === 0) { - it.setProperty('metadata', metadata); + this.mediatorRdfMetadataAccumulate + .mediate({ mode: 'initialize', context: this.contextDefault }) + .then(result => it.setProperty('metadata', { state: new MetadataValidationState(), ...result.metadata })) + .catch(error => it.emit('error', error)); } return it; diff --git a/packages/actor-rdf-resolve-quad-pattern-federated/package.json b/packages/actor-rdf-resolve-quad-pattern-federated/package.json index f6406cb1d0..57d5bf0e50 100644 --- a/packages/actor-rdf-resolve-quad-pattern-federated/package.json +++ b/packages/actor-rdf-resolve-quad-pattern-federated/package.json @@ -33,10 +33,12 @@ ], "dependencies": { "@comunica/bus-query-operation": "^2.6.8", + "@comunica/bus-rdf-metadata-accumulate": "^2.5.0", "@comunica/bus-rdf-resolve-quad-pattern": "^2.6.8", "@comunica/context-entries": "^2.6.8", "@comunica/core": "^2.6.8", "@comunica/data-factory": "^2.5.1", + "@comunica/metadata": "^2.5.0", "@comunica/types": "^2.6.8", "@rdfjs/types": "*", "asynciterator": "^3.8.0", diff --git a/packages/actor-rdf-resolve-quad-pattern-federated/test/ActorRdfResolveQuadPatternFederated-test.ts b/packages/actor-rdf-resolve-quad-pattern-federated/test/ActorRdfResolveQuadPatternFederated-test.ts index f23cb8a1fd..2eece43031 100644 --- a/packages/actor-rdf-resolve-quad-pattern-federated/test/ActorRdfResolveQuadPatternFederated-test.ts +++ b/packages/actor-rdf-resolve-quad-pattern-federated/test/ActorRdfResolveQuadPatternFederated-test.ts @@ -1,5 +1,6 @@ import { ActorRdfResolveQuadPattern } from '@comunica/bus-rdf-resolve-quad-pattern'; import { ActionContext, Bus } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IActionContext } from '@comunica/types'; import arrayifyStream from 'arrayify-stream'; import { ArrayIterator } from 'asynciterator'; @@ -24,8 +25,11 @@ describe('ActorRdfResolveQuadPatternFederated', () => { squad('s1', 'p1', 'o2'), ], { autoStart: false }); data.setProperty('metadata', { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, - canContainUndefs: false, + requestTime: 10, + pageSize: 100, + canContainUndefs: true, }); return Promise.resolve({ data }); }, @@ -96,7 +100,13 @@ describe('ActorRdfResolveQuadPatternFederated', () => { return actor.run({ pattern, context }) .then(async output => { expect(await new Promise(resolve => output.data.getProperty('metadata', resolve))) - .toEqual({ cardinality: { type: 'estimate', value: 4 }, canContainUndefs: false }); + .toEqual({ + state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: 4 }, + canContainUndefs: true, + pageSize: 200, + requestTime: 20, + }); expect(await arrayifyStream(output.data)).toBeRdfIsomorphic([ squad('s1', 'p1', 'o1'), squad('s1', 'p1', 'o1'), @@ -118,7 +128,13 @@ describe('ActorRdfResolveQuadPatternFederated', () => { return actor.run({ pattern, context }) .then(async output => { expect(await new Promise(resolve => output.data.getProperty('metadata', resolve))) - .toEqual({ cardinality: { type: 'estimate', value: 4 }, canContainUndefs: false }); + .toEqual({ + state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: 4 }, + canContainUndefs: true, + pageSize: 200, + requestTime: 20, + }); expect(await arrayifyStream(output.data)).toBeRdfIsomorphic([ squad('s1', 'p1', 'o1'), squad('s1', 'p1', 'o1'), @@ -146,7 +162,13 @@ describe('ActorRdfResolveQuadPatternFederated', () => { ]); await expect(new Promise(resolve => data.getProperty('metadata', resolve))) - .resolves.toEqual({ cardinality: { type: 'estimate', value: 4 }, canContainUndefs: false }); + .resolves.toEqual({ + state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: 4 }, + canContainUndefs: true, + pageSize: 200, + requestTime: 20, + }); }); it('should run when only data is called', () => { @@ -168,5 +190,93 @@ describe('ActorRdfResolveQuadPatternFederated', () => { ]); }); }); + + it('should run if cardinality is Infinity', () => { + mediatorResolveQuadPattern.mediate = () => { + const data = new ArrayIterator([ + squad('s1', 'p1', 'o1'), + squad('s1', 'p1', 'o2'), + ], { autoStart: false }); + data.setProperty('metadata', { + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, + requestTime: 10, + pageSize: 100, + canContainUndefs: true, + }); + return Promise.resolve({ data }); + }; + + const pattern = squad('?s', 'p', 'o'); + context = new ActionContext({ + '@comunica/bus-rdf-resolve-quad-pattern:sources': + [ + { type: 'nonEmptySource', value: 'I will not be empty' }, + { type: 'nonEmptySource', value: 'I will not be empty' }, + ], + }); + return actor.run({ pattern, context }) + .then(async output => { + expect(await new Promise(resolve => output.data.getProperty('metadata', resolve))) + .toEqual({ + state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, + canContainUndefs: true, + pageSize: 200, + requestTime: 20, + }); + expect(await arrayifyStream(output.data)).toBeRdfIsomorphic([ + squad('s1', 'p1', 'o1'), + squad('s1', 'p1', 'o1'), + squad('s1', 'p1', 'o2'), + squad('s1', 'p1', 'o2'), + ]); + }); + }); + + it('should run if metadata is only set the first time', () => { + let call = 0; + mediatorResolveQuadPattern.mediate = () => { + const data = new ArrayIterator([ + squad('s1', 'p1', 'o1'), + squad('s1', 'p1', 'o2'), + ], { autoStart: false }); + data.setProperty('metadata', { + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, + requestTime: call === 0 ? 10 : undefined, + pageSize: call === 0 ? 100 : undefined, + canContainUndefs: true, + }); + call++; + return Promise.resolve({ data }); + }; + + const pattern = squad('?s', 'p', 'o'); + context = new ActionContext({ + '@comunica/bus-rdf-resolve-quad-pattern:sources': + [ + { type: 'nonEmptySource', value: 'I will not be empty' }, + { type: 'nonEmptySource', value: 'I will not be empty' }, + ], + }); + return actor.run({ pattern, context }) + .then(async output => { + expect(await new Promise(resolve => output.data.getProperty('metadata', resolve))) + .toEqual({ + state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, + canContainUndefs: true, + pageSize: 100, + requestTime: 10, + }); + expect(await arrayifyStream(output.data)).toBeRdfIsomorphic([ + squad('s1', 'p1', 'o1'), + squad('s1', 'p1', 'o1'), + squad('s1', 'p1', 'o2'), + squad('s1', 'p1', 'o2'), + ]); + }); + }); }); }); diff --git a/packages/actor-rdf-resolve-quad-pattern-federated/test/FederatedQuadSource-test.ts b/packages/actor-rdf-resolve-quad-pattern-federated/test/FederatedQuadSource-test.ts index a241755655..d732a179bc 100644 --- a/packages/actor-rdf-resolve-quad-pattern-federated/test/FederatedQuadSource-test.ts +++ b/packages/actor-rdf-resolve-quad-pattern-federated/test/FederatedQuadSource-test.ts @@ -1,6 +1,8 @@ +import type { IActionRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate'; import { KeysRdfResolveQuadPattern } from '@comunica/context-entries'; import { ActionContext } from '@comunica/core'; import { BlankNodeScoped } from '@comunica/data-factory'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IActionContext } from '@comunica/types'; import type * as RDF from '@rdfjs/types'; import arrayifyStream from 'arrayify-stream'; @@ -19,18 +21,23 @@ const DF = new DataFactory(); const v = DF.variable('v'); describe('FederatedQuadSource', () => { - let mediator: any; + let mediatorResolveQuadPattern: any; + let mediatorRdfMetadataAccumulate: any; let context: IActionContext; let returnedIterators: AsyncIterator[]; beforeEach(() => { returnedIterators = []; - mediator = { + mediatorResolveQuadPattern = { mediate(action: any) { const type = action.context.get(KeysRdfResolveQuadPattern.source).type; if (type === 'emptySource') { const data = new ArrayIterator([], { autoStart: false }); - data.setProperty('metadata', { cardinality: { type: 'estimate', value: 0 }, canContainUndefs: false }); + data.setProperty('metadata', { + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 0 }, + canContainUndefs: false, + }); return Promise.resolve({ data }); } if (type === 'nonEmptySourceNoMeta') { @@ -47,6 +54,7 @@ describe('FederatedQuadSource', () => { squad('s1', 'p1', 'o2'), ], { autoStart: false }); data.setProperty('metadata', { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false, }); @@ -58,6 +66,7 @@ describe('FederatedQuadSource', () => { squad('_:s2', '_:p2', '_:o2'), ], { autoStart: false }); data.setProperty('metadata', { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false, }); @@ -75,6 +84,7 @@ describe('FederatedQuadSource', () => { return squad('_:s1', '_:p1', '_:o1'); }; data.setProperty('metadata', { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false, }); @@ -90,6 +100,7 @@ describe('FederatedQuadSource', () => { squad('s3', 'p1', 'o2', 'g2'), ], { autoStart: false }); data.setProperty('metadata', { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 6 }, canContainUndefs: false, }); @@ -101,6 +112,7 @@ describe('FederatedQuadSource', () => { squad('s1', 'p1', 'o2'), ], { autoStart: false }); data.setProperty('metadata', { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, canContainUndefs: true, }); @@ -111,6 +123,7 @@ describe('FederatedQuadSource', () => { squad('s1', 'p1', 'o2'), ], { autoStart: false }); data.setProperty('metadata', { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 2 }, requestTime: 10, pageSize: 100, @@ -122,6 +135,41 @@ describe('FederatedQuadSource', () => { return Promise.resolve({ data }); }, }; + mediatorRdfMetadataAccumulate = { + async mediate(action: IActionRdfMetadataAccumulate) { + if (action.mode === 'initialize') { + return { metadata: { cardinality: { type: 'exact', value: 0 }, canContainUndefs: false }}; + } + + const metadata = { ...action.accumulatedMetadata }; + const subMetadata = action.appendingMetadata; + if (!subMetadata.cardinality || !Number.isFinite(subMetadata.cardinality.value)) { + // We're already at infinite, so ignore any later metadata + metadata.cardinality.type = 'estimate'; + metadata.cardinality.value = Number.POSITIVE_INFINITY; + } else { + if (subMetadata.cardinality.type === 'estimate') { + metadata.cardinality.type = 'estimate'; + } + metadata.cardinality.value += subMetadata.cardinality.value; + } + if (metadata.requestTime || subMetadata.requestTime) { + metadata.requestTime = metadata.requestTime || 0; + subMetadata.requestTime = subMetadata.requestTime || 0; + metadata.requestTime += subMetadata.requestTime; + } + if (metadata.pageSize || subMetadata.pageSize) { + metadata.pageSize = metadata.pageSize || 0; + subMetadata.pageSize = subMetadata.pageSize || 0; + metadata.pageSize += subMetadata.pageSize; + } + if (subMetadata.canContainUndefs) { + metadata.canContainUndefs = true; + } + + return { metadata }; + }, + }; context = new ActionContext({ [KeysRdfResolveQuadPattern.sources.name]: [{ type: 'myType', value: 'myValue' }], }); @@ -133,11 +181,23 @@ describe('FederatedQuadSource', () => { }); it('should be a FederatedQuadSource constructor', () => { - expect(new FederatedQuadSource(mediator, context, new Map(), true)).toBeInstanceOf(FederatedQuadSource); + expect(new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + context, + new Map(), + true, + )).toBeInstanceOf(FederatedQuadSource); }); it('should be a FederatedQuadSource constructor with optional bufferSize argument', () => { - expect(new FederatedQuadSource(mediator, context, new Map(), true)).toBeInstanceOf(FederatedQuadSource); + expect(new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + context, + new Map(), + true, + )).toBeInstanceOf(FederatedQuadSource); }); }); @@ -456,7 +516,13 @@ describe('FederatedQuadSource', () => { subSource = {}; emptyPatterns = new Map(); emptyPatterns.set(subSource, [ squad('?a', '?b', '?c', '"d"') ]); - source = new FederatedQuadSource(mediator, context, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + context, + emptyPatterns, + true, + ); }); it('should return an AsyncIterator', async() => { @@ -500,7 +566,13 @@ describe('FederatedQuadSource', () => { subSource = {}; emptyPatterns = new Map(); emptyPatterns.set(subSource, [ squad('?a', '?b', '?c', '"d"') ]); - source = new FederatedQuadSource(mediator, context, emptyPatterns, false); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + context, + emptyPatterns, + false, + ); }); describe('when calling isSourceEmpty', () => { @@ -534,7 +606,13 @@ describe('FederatedQuadSource', () => { beforeEach(() => { emptyPatterns = new Map(); contextEmpty = new ActionContext({ [KeysRdfResolveQuadPattern.sources.name]: []}); - source = new FederatedQuadSource(mediator, contextEmpty, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextEmpty, + emptyPatterns, + true, + ); }); it('should return an empty array of results using #toArray', async() => { @@ -548,10 +626,18 @@ describe('FederatedQuadSource', () => { it('should emit metadata with 0 cardinality', async() => { const stream = source.match(v, v, v, v); await expect(new Promise(resolve => stream.getProperty('metadata', resolve))) - .resolves.toEqual({ cardinality: { type: 'exact', value: 0 }, canContainUndefs: false }); - + .resolves.toEqual({ + state: expect.any(MetadataValidationState), + cardinality: { type: 'exact', value: 0 }, + canContainUndefs: false, + }); expect(await arrayifyStream(stream)).toEqual([]); }); + + it('should reject when mediatorRdfMetadataAccumulate rejects', async() => { + mediatorRdfMetadataAccumulate.mediate = () => Promise.reject(new Error(`mediatorRdfMetadataAccumulate error in FederatedQuadSource-test`)); + await expect(arrayifyStream(source.match(v, v, v, v))).rejects.toThrow(`mediatorRdfMetadataAccumulate error in FederatedQuadSource-test`); + }); }); describe('A FederatedQuadSource instance over one empty source', () => { @@ -564,7 +650,13 @@ describe('FederatedQuadSource', () => { subSource = { type: 'emptySource', value: 'I will be empty' }; emptyPatterns = new Map(); contextSingleEmpty = new ActionContext({ [KeysRdfResolveQuadPattern.sources.name]: [ subSource ]}); - source = new FederatedQuadSource(mediator, contextSingleEmpty, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingleEmpty, + emptyPatterns, + true, + ); }); it('should return an empty AsyncIterator', async() => { @@ -574,7 +666,11 @@ describe('FederatedQuadSource', () => { it('should emit metadata with 0 cardinality', async() => { const stream = source.match(v, v, v, v); await expect(new Promise(resolve => stream.getProperty('metadata', resolve))) - .resolves.toEqual({ cardinality: { type: 'estimate', value: 0 }, canContainUndefs: false }); + .resolves.toEqual({ + state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: 0 }, + canContainUndefs: false, + }); expect(await arrayifyStream(stream)).toEqual([]); }); @@ -606,7 +702,13 @@ describe('FederatedQuadSource', () => { subSource = { type: 'nonEmptySource', value: 'I will not be empty' }; emptyPatterns = new Map(); contextSingle = new ActionContext({ [KeysRdfResolveQuadPattern.sources.name]: [ subSource ]}); - source = new FederatedQuadSource(mediator, contextSingle, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingle, + emptyPatterns, + true, + ); }); it('should return a non-empty AsyncIterator in the default graph', async() => { @@ -627,6 +729,7 @@ describe('FederatedQuadSource', () => { const stream = source.match(v, v, v, v); await expect(new Promise(resolve => stream.getProperty('metadata', resolve))) .resolves.toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 2 }, requestTime: 10, pageSize: 100, @@ -662,7 +765,7 @@ describe('FederatedQuadSource', () => { }); it('destroyed before started should void proxyIt errors', async() => { - mediator.mediate = () => Promise.reject(new Error('ignored error')); + mediatorResolveQuadPattern.mediate = () => Promise.reject(new Error('ignored error')); const it = source.match(DF.variable('s'), DF.literal('p'), DF.variable('o'), DF.variable('g')); it.destroy(); @@ -680,7 +783,13 @@ describe('FederatedQuadSource', () => { subSource = { type: 'graphs', value: 'I will contain named graphs' }; emptyPatterns = new Map(); contextSingle = new ActionContext({ [KeysRdfResolveQuadPattern.sources.name]: [ subSource ]}); - source = new FederatedQuadSource(mediator, contextSingle, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingle, + emptyPatterns, + true, + ); }); it('should return a non-empty AsyncIterator in the default graph', async() => { @@ -711,8 +820,11 @@ describe('FederatedQuadSource', () => { it('should emit metadata with 6 cardinality', async() => { const stream = source.match(v, v, v, v); await expect(new Promise(resolve => stream.getProperty('metadata', resolve))) - .resolves.toEqual({ cardinality: { type: 'estimate', value: 6 }, canContainUndefs: false }); - + .resolves.toEqual({ + state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: 6 }, + canContainUndefs: false, + }); expect(await arrayifyStream(stream)).toEqual([ squad('s1', 'p1', 'o1'), squad('s1', 'p1', 'o2'), @@ -742,7 +854,13 @@ describe('FederatedQuadSource', () => { subSource2, ], }); - source = new FederatedQuadSource(mediator, contextSingleEmpty, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingleEmpty, + emptyPatterns, + true, + ); }); it('should return a non-empty AsyncIterator in the default graph', async() => { @@ -765,10 +883,12 @@ describe('FederatedQuadSource', () => { const stream = source.match(v, v, v, v); await expect(new Promise(resolve => stream.getProperty('metadata', resolve))) .resolves.toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 2 }, requestTime: 10, pageSize: 100, canContainUndefs: false, + otherMetadata: true, }); expect(await stream.toArray()).toEqual([ @@ -812,7 +932,13 @@ describe('FederatedQuadSource', () => { subSource2, ], }); - source = new FederatedQuadSource(mediator, contextSingleEmpty, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingleEmpty, + emptyPatterns, + true, + ); }); it('should return an empty AsyncIterator for the default graph', async() => { @@ -828,8 +954,11 @@ describe('FederatedQuadSource', () => { it('should emit metadata with 0 cardinality', async() => { const stream = source.match(v, v, v, v); await expect(new Promise(resolve => stream.getProperty('metadata', resolve))) - .resolves.toEqual({ cardinality: { type: 'estimate', value: 0 }, canContainUndefs: false }); - + .resolves.toEqual({ + state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: 0 }, + canContainUndefs: false, + }); expect(await stream.toArray()).toEqual([]); }); @@ -870,7 +999,13 @@ describe('FederatedQuadSource', () => { subSource, ], }); - source = new FederatedQuadSource(mediator, contextSingleEmpty, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingleEmpty, + emptyPatterns, + true, + ); }); it('should return an empty AsyncIterator in the default graph', async() => { @@ -886,7 +1021,11 @@ describe('FederatedQuadSource', () => { it('should emit metadata with 0 cardinality', async() => { const stream = source.match(v, v, v, v); await expect(new Promise(resolve => stream.getProperty('metadata', resolve))) - .resolves.toEqual({ cardinality: { type: 'estimate', value: 0 }, canContainUndefs: false }); + .resolves.toEqual({ + state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: 0 }, + canContainUndefs: false, + }); expect(await stream.toArray()).toEqual([]); }); @@ -921,7 +1060,13 @@ describe('FederatedQuadSource', () => { { type: 'emptySource', value: 'I will be empty' }, ], }); - source = new FederatedQuadSource(mediator, contextSingleEmpty, emptyPatterns, false); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingleEmpty, + emptyPatterns, + false, + ); }); it('should return an empty AsyncIterator in the default graph', async() => { @@ -937,8 +1082,11 @@ describe('FederatedQuadSource', () => { it('should emit metadata with 0 cardinality', async() => { const stream = source.match(v, v, v, v); await expect(new Promise(resolve => stream.getProperty('metadata', resolve))) - .resolves.toEqual({ cardinality: { type: 'estimate', value: 0 }, canContainUndefs: false }); - + .resolves.toEqual({ + state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: 0 }, + canContainUndefs: false, + }); expect(await stream.toArray()).toEqual([]); }); @@ -972,7 +1120,13 @@ describe('FederatedQuadSource', () => { subSource2, ], }); - source = new FederatedQuadSource(mediator, contextSingleEmpty, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingleEmpty, + emptyPatterns, + true, + ); }); it('should return a non-empty AsyncIterator in the default graph', async() => { @@ -999,10 +1153,12 @@ describe('FederatedQuadSource', () => { const stream = source.match(v, v, v, v); await expect(new Promise(resolve => stream.getProperty('metadata', resolve))) .resolves.toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 4 }, requestTime: 20, pageSize: 200, canContainUndefs: false, + otherMetadata: true, }); expect(await stream.toArray()).toEqual([ @@ -1026,6 +1182,11 @@ describe('FederatedQuadSource', () => { [ subSource2, []], ]); }); + + it('should reject when mediatorRdfMetadataAccumulate rejects', async() => { + mediatorRdfMetadataAccumulate.mediate = () => Promise.reject(new Error(`mediatorRdfMetadataAccumulate error in FederatedQuadSource-test`)); + await expect(arrayifyStream(source.match(v, v, v, v))).rejects.toThrow(`mediatorRdfMetadataAccumulate error in FederatedQuadSource-test`); + }); }); describe('A FederatedQuadSource instance over two non-empty sources, one without metadata', () => { @@ -1042,7 +1203,13 @@ describe('FederatedQuadSource', () => { { type: 'nonEmptySourceNoMeta', value: 'I will also not be empty, but have no metadata' }, ], }); - source = new FederatedQuadSource(mediator, contextSingleEmpty, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingleEmpty, + emptyPatterns, + true, + ); }); it('should return a non-empty AsyncIterator for the default graph', async() => { @@ -1069,10 +1236,12 @@ describe('FederatedQuadSource', () => { const stream = source.match(v, v, v, v); await expect(new Promise(resolve => stream.getProperty('metadata', resolve))) .resolves.toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, requestTime: 10, pageSize: 100, canContainUndefs: false, + otherMetadata: true, }); expect(await stream.toArray()).toEqual([ @@ -1098,7 +1267,13 @@ describe('FederatedQuadSource', () => { { type: 'nonEmptySourceNoMeta', value: 'I will also not be empty, but have no metadata' }, ], }); - source = new FederatedQuadSource(mediator, contextSingleEmpty, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingleEmpty, + emptyPatterns, + true, + ); }); it('should return a non-empty AsyncIterator for the default graph', async() => { @@ -1125,6 +1300,7 @@ describe('FederatedQuadSource', () => { const stream = source.match(v, v, v, v); await expect(new Promise(resolve => stream.getProperty('metadata', resolve))) .resolves.toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false, }); @@ -1152,7 +1328,13 @@ describe('FederatedQuadSource', () => { { type: 'nonEmptySourceInfMeta', value: 'I will also not be empty, but have inf metadata' }, ], }); - source = new FederatedQuadSource(mediator, contextSingleEmpty, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingleEmpty, + emptyPatterns, + true, + ); }); it('should return a non-empty AsyncIterator for the default graph', async() => { @@ -1179,10 +1361,12 @@ describe('FederatedQuadSource', () => { const stream = source.match(v, v, v, v); await expect(new Promise(resolve => stream.getProperty('metadata', resolve))) .resolves.toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, requestTime: 10, pageSize: 100, canContainUndefs: false, + otherMetadata: true, }); expect(await stream.toArray()).toEqual([ @@ -1208,7 +1392,13 @@ describe('FederatedQuadSource', () => { { type: 'nonEmptySourceInfMeta', value: 'I will also not be empty, but have inf metadata' }, ], }); - source = new FederatedQuadSource(mediator, contextSingleEmpty, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingleEmpty, + emptyPatterns, + true, + ); }); it('should return a non-empty AsyncIterator for the default graph', async() => { @@ -1235,6 +1425,7 @@ describe('FederatedQuadSource', () => { const stream = source.match(v, v, v, v); await expect(new Promise(resolve => stream.getProperty('metadata', resolve))) .resolves.toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, canContainUndefs: false, }); @@ -1266,7 +1457,13 @@ describe('FederatedQuadSource', () => { subSource2, ], }); - source = new FederatedQuadSource(mediator, contextSingleEmpty, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingleEmpty, + emptyPatterns, + true, + ); }); it('should return a non-empty AsyncIterator', async() => { @@ -1512,17 +1709,25 @@ describe('FederatedQuadSource', () => { subSource2, ], }); - source = new FederatedQuadSource(mediator, contextSingleEmpty, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingleEmpty, + emptyPatterns, + true, + ); }); it('should emit metadata with canContainUndefs true', async() => { const stream = source.match(v, v, v, v); await expect(new Promise(resolve => stream.getProperty('metadata', resolve))) .resolves.toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 4 }, pageSize: 100, requestTime: 10, canContainUndefs: true, + otherMetadata: true, }); expect(await stream.toArray()).toEqual([ @@ -1552,17 +1757,25 @@ describe('FederatedQuadSource', () => { subSource2, ], }); - source = new FederatedQuadSource(mediator, contextSingleEmpty, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingleEmpty, + emptyPatterns, + true, + ); }); it('should emit metadata with canContainUndefs true', async() => { const stream = source.match(v, v, v, v); await expect(new Promise(resolve => stream.getProperty('metadata', resolve))) .resolves.toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 4 }, pageSize: 100, requestTime: 10, canContainUndefs: true, + otherMetadata: true, }); expect(await stream.toArray()).toEqual([ @@ -1592,14 +1805,23 @@ describe('FederatedQuadSource', () => { subSource2, ], }); - source = new FederatedQuadSource(mediator, contextSingleEmpty, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingleEmpty, + emptyPatterns, + true, + ); }); it('should emit metadata with canContainUndefs true', async() => { const stream = source.match(v, v, v, v); await expect(new Promise(resolve => stream.getProperty('metadata', resolve))) - .resolves.toEqual({ cardinality: { type: 'estimate', value: 4 }, canContainUndefs: true }); - + .resolves.toEqual({ + state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: 4 }, + canContainUndefs: true, + }); expect(await stream.toArray()).toEqual([ squad('s1', 'p1', 'o1'), squad('s1', 'p1', 'o1'), @@ -1624,7 +1846,13 @@ describe('FederatedQuadSource', () => { subSource1, ], }); - source = new FederatedQuadSource(mediator, contextSingleEmpty, emptyPatterns, true); + source = new FederatedQuadSource( + mediatorResolveQuadPattern, + mediatorRdfMetadataAccumulate, + contextSingleEmpty, + emptyPatterns, + true, + ); }); it('should emit an error', async() => { diff --git a/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/ActorRdfResolveQuadPatternHypermedia.ts b/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/ActorRdfResolveQuadPatternHypermedia.ts index 0ba01c0a24..06e976e02b 100644 --- a/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/ActorRdfResolveQuadPatternHypermedia.ts +++ b/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/ActorRdfResolveQuadPatternHypermedia.ts @@ -1,6 +1,8 @@ import type { MediatorDereferenceRdf } from '@comunica/bus-dereference-rdf'; import type { ActorHttpInvalidateListenable, IActionHttpInvalidate } from '@comunica/bus-http-invalidate'; import type { MediatorRdfMetadata } from '@comunica/bus-rdf-metadata'; +import type { MediatorRdfMetadataAccumulate, + IActionRdfMetadataAccumulateAppend } from '@comunica/bus-rdf-metadata-accumulate'; import type { MediatorRdfMetadataExtract } from '@comunica/bus-rdf-metadata-extract'; import type { MediatorRdfResolveHypermedia } from '@comunica/bus-rdf-resolve-hypermedia'; import type { MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links'; @@ -26,6 +28,7 @@ export class ActorRdfResolveQuadPatternHypermedia extends ActorRdfResolveQuadPat public readonly mediatorDereferenceRdf: MediatorDereferenceRdf; public readonly mediatorMetadata: MediatorRdfMetadata; public readonly mediatorMetadataExtract: MediatorRdfMetadataExtract; + public readonly mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate; public readonly mediatorRdfResolveHypermedia: MediatorRdfResolveHypermedia; public readonly mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks; public readonly mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue; @@ -44,6 +47,16 @@ export class ActorRdfResolveQuadPatternHypermedia extends ActorRdfResolveQuadPat ({ url }: IActionHttpInvalidate) => url ? cache.delete(url) : cache.clear(), ); } + + // TODO: remove this backwards-compatibility in the next major version, and make the param mandatory + if (!args.mediatorMetadataAccumulate) { + this.mediatorMetadataAccumulate = { + async mediate(action: IActionRdfMetadataAccumulateAppend) { + // 'initialize' mode is not used in this actor, so we can assume 'append'. + return { metadata: { ...action.accumulatedMetadata, ...action.appendingMetadata }}; + }, + }; + } } public async test(action: IActionRdfResolveQuadPattern): Promise { @@ -73,6 +86,7 @@ export class ActorRdfResolveQuadPatternHypermedia extends ActorRdfResolveQuadPat { mediatorMetadata: this.mediatorMetadata, mediatorMetadataExtract: this.mediatorMetadataExtract, + mediatorMetadataAccumulate: this.mediatorMetadataAccumulate, mediatorDereferenceRdf: this.mediatorDereferenceRdf, mediatorRdfResolveHypermedia: this.mediatorRdfResolveHypermedia, mediatorRdfResolveHypermediaLinks: this.mediatorRdfResolveHypermediaLinks, @@ -128,6 +142,10 @@ export interface IActorRdfResolveQuadPatternHypermediaArgs extends IActorRdfReso * The metadata extract mediator */ mediatorMetadataExtract: MediatorRdfMetadataExtract; + /** + * The metadata accumulate mediator + */ + mediatorMetadataAccumulate?: MediatorRdfMetadataAccumulate; /** * The hypermedia resolve mediator */ diff --git a/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/LinkedRdfSourcesAsyncRdfIterator.ts b/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/LinkedRdfSourcesAsyncRdfIterator.ts index e0600ca547..21761c10fd 100644 --- a/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/LinkedRdfSourcesAsyncRdfIterator.ts +++ b/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/LinkedRdfSourcesAsyncRdfIterator.ts @@ -1,8 +1,10 @@ import type { ILink } from '@comunica/bus-rdf-resolve-hypermedia-links'; import type { ILinkQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue'; import type { IQuadSource } from '@comunica/bus-rdf-resolve-quad-pattern'; +import { MetadataValidationState } from '@comunica/metadata'; +import type { MetadataQuads } from '@comunica/types'; import type * as RDF from '@rdfjs/types'; -import type { AsyncIterator } from 'asynciterator'; +import type { AsyncIterator, BufferedIteratorOptions } from 'asynciterator'; import { BufferedIterator } from 'asynciterator'; import LRUCache = require('lru-cache'); @@ -13,7 +15,6 @@ export abstract class LinkedRdfSourcesAsyncRdfIterator extends BufferedIterator< protected readonly predicate: RDF.Term; protected readonly object: RDF.Term; protected readonly graph: RDF.Term; - protected nextSource: ISourceState | undefined; private readonly cacheSize: number; protected readonly firstUrl: string; @@ -22,10 +23,11 @@ export abstract class LinkedRdfSourcesAsyncRdfIterator extends BufferedIterator< private started = false; private readonly currentIterators: AsyncIterator[]; private iteratorsPendingCreation: number; + private accumulatedMetadata: Promise; public constructor(cacheSize: number, subject: RDF.Term, predicate: RDF.Term, object: RDF.Term, graph: RDF.Term, - firstUrl: string, maxIterators: number) { - super({ autoStart: true }); + firstUrl: string, maxIterators: number, options?: BufferedIteratorOptions) { + super({ autoStart: true, ...options }); this.cacheSize = cacheSize; this.subject = subject; this.predicate = predicate; @@ -40,6 +42,8 @@ export abstract class LinkedRdfSourcesAsyncRdfIterator extends BufferedIterator< this.currentIterators = []; this.iteratorsPendingCreation = 0; + // eslint-disable-next-line unicorn/no-useless-undefined + this.accumulatedMetadata = Promise.resolve(undefined); } protected _end(destroy?: boolean): void { @@ -177,13 +181,24 @@ export abstract class LinkedRdfSourcesAsyncRdfIterator extends BufferedIterator< return (this.currentIterators.length + this.iteratorsPendingCreation) > 0; } + /** + * Append the fields from appendingMetadata into accumulatedMetadata. + * @param accumulatedMetadata The fields to append to. + * @param appendingMetadata The fields to append. + * @protected + */ + protected abstract accumulateMetadata( + accumulatedMetadata: MetadataQuads, + appendingMetadata: MetadataQuads, + ): Promise; + /** * Start a new iterator for the given source. * Once the iterator is done, it will either determine a new source, or it will close the linked iterator. * @param {ISourceState} startSource The start source state. - * @param {boolean} emitMetadata If the metadata event should be emitted. + * @param {boolean} firstPage If this is the first iterator that is being started. */ - protected startIterator(startSource: ISourceState, emitMetadata: boolean): void { + protected startIterator(startSource: ISourceState, firstPage: boolean): void { // Delegate the quad pattern query to the given source const iterator = startSource.source! .match(this.subject, this.predicate, this.object, this.graph); @@ -206,30 +221,53 @@ export abstract class LinkedRdfSourcesAsyncRdfIterator extends BufferedIterator< // Listen for the metadata of the source // The metadata property is guaranteed to be set - iterator.getProperty('metadata', (metadata: Record) => { - startSource.metadata = { ...startSource.metadata, ...metadata }; - - // Emit metadata if needed - if (emitMetadata) { - this.setProperty('metadata', startSource.metadata); - } - - // Determine next urls, which will eventually become a next-next source. - this.getSourceLinks(startSource.metadata) - .then((nextUrls: ILink[]) => Promise.all(nextUrls)) - .then(async(nextUrls: ILink[]) => { - // Append all next URLs to our queue - const linkQueue = await this.getLinkQueue(); - for (const nextUrl of nextUrls) { - linkQueue.push(nextUrl, startSource.link); + iterator.getProperty('metadata', (metadata: MetadataQuads) => { + // Accumulate the metadata object + this.accumulatedMetadata = this.accumulatedMetadata + .then(previousMetadata => (async() => { + if (!previousMetadata) { + previousMetadata = startSource.metadata; } - - receivedMetadata = true; - this.startIteratorsForNextUrls(startSource.handledDatasets, true); - }).catch(error => this.destroy(error)); + return this.accumulateMetadata(previousMetadata, metadata); + })() + .then(accumulatedMetadata => { + // Also merge fields that were not explicitly accumulated + const returnMetadata = { ...startSource.metadata, ...metadata, ...accumulatedMetadata }; + + // Create new metadata state + returnMetadata.state = new MetadataValidationState(); + + // Emit metadata, and invalidate any metadata that was set before + this.updateMetadata(returnMetadata); + + // Determine next urls, which will eventually become a next-next source. + this.getSourceLinks(returnMetadata) + .then((nextUrls: ILink[]) => Promise.all(nextUrls)) + .then(async(nextUrls: ILink[]) => { + // Append all next URLs to our queue + const linkQueue = await this.getLinkQueue(); + for (const nextUrl of nextUrls) { + linkQueue.push(nextUrl, startSource.link); + } + + receivedMetadata = true; + this.startIteratorsForNextUrls(startSource.handledDatasets, true); + }).catch(error => this.destroy(error)); + + return returnMetadata; + })).catch(error => { + this.destroy(error); + return {}; + }); }); } + protected updateMetadata(metadataNew: MetadataQuads): void { + const metadataToInvalidate = this.getProperty('metadata'); + this.setProperty('metadata', metadataNew); + metadataToInvalidate?.state.invalidate(); + } + protected isRunning(): boolean { return !this.done; } @@ -261,12 +299,16 @@ export abstract class LinkedRdfSourcesAsyncRdfIterator extends BufferedIterator< } // Close, only if no other iterators are still running - if (canClose && linkQueue.isEmpty() && !this.areIteratorsRunning()) { + if (canClose && this.isCloseable(linkQueue)) { this.close(); } }) .catch(error => this.destroy(error)); } + + protected isCloseable(linkQueue: ILinkQueue): boolean { + return linkQueue.isEmpty() && !this.areIteratorsRunning(); + } } /** @@ -296,7 +338,7 @@ export interface ISourceState { /** * The source's initial metadata. */ - metadata: Record; + metadata: MetadataQuads; /** * All dataset identifiers that have been passed for this source. */ diff --git a/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/MediatedLinkedRdfSourcesAsyncRdfIterator.ts b/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/MediatedLinkedRdfSourcesAsyncRdfIterator.ts index 126c86bd8d..f51230a837 100644 --- a/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/MediatedLinkedRdfSourcesAsyncRdfIterator.ts +++ b/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/MediatedLinkedRdfSourcesAsyncRdfIterator.ts @@ -1,12 +1,13 @@ import type { IActorDereferenceRdfOutput, MediatorDereferenceRdf } from '@comunica/bus-dereference-rdf'; import type { IActorRdfMetadataOutput, MediatorRdfMetadata } from '@comunica/bus-rdf-metadata'; +import type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate'; import type { MediatorRdfMetadataExtract } from '@comunica/bus-rdf-metadata-extract'; import type { MediatorRdfResolveHypermedia } from '@comunica/bus-rdf-resolve-hypermedia'; import type { ILink, MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links'; import type { ILinkQueue, MediatorRdfResolveHypermediaLinksQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue'; -import type { IActionContext, IAggregatedStore } from '@comunica/types'; +import type { IActionContext, IAggregatedStore, MetadataQuads } from '@comunica/types'; import type * as RDF from '@rdfjs/types'; import { Readable } from 'readable-stream'; import type { ISourceState } from './LinkedRdfSourcesAsyncRdfIterator'; @@ -22,6 +23,7 @@ export class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAs private readonly mediatorDereferenceRdf: MediatorDereferenceRdf; private readonly mediatorMetadata: MediatorRdfMetadata; private readonly mediatorMetadataExtract: MediatorRdfMetadataExtract; + private readonly mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate; private readonly mediatorRdfResolveHypermedia: MediatorRdfResolveHypermedia; private readonly mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks; private readonly mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue; @@ -30,17 +32,30 @@ export class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAs private readonly handledUrls: Record; private readonly aggregatedStore: IAggregatedStore | undefined; private linkQueue: Promise | undefined; + private wasForcefullyClosed = false; public constructor(cacheSize: number, context: IActionContext, forceSourceType: string | undefined, subject: RDF.Term, predicate: RDF.Term, object: RDF.Term, graph: RDF.Term, firstUrl: string, maxIterators: number, aggregatedStore: IAggregatedStore | undefined, mediators: IMediatorArgs) { - super(cacheSize, subject, predicate, object, graph, firstUrl, maxIterators); + super( + cacheSize, + subject, + predicate, + object, + graph, + firstUrl, + maxIterators, + // Buffersize must be infinite for an aggregated store because it must keep filling until there are no more + // derived iterators in the aggregated store. + aggregatedStore ? { maxBufferSize: Number.POSITIVE_INFINITY } : undefined, + ); this.context = context; this.forceSourceType = forceSourceType; this.mediatorDereferenceRdf = mediators.mediatorDereferenceRdf; this.mediatorMetadata = mediators.mediatorMetadata; this.mediatorMetadataExtract = mediators.mediatorMetadataExtract; + this.mediatorMetadataAccumulate = mediators.mediatorMetadataAccumulate; this.mediatorRdfResolveHypermedia = mediators.mediatorRdfResolveHypermedia; this.mediatorRdfResolveHypermediaLinks = mediators.mediatorRdfResolveHypermediaLinks; this.mediatorRdfResolveHypermediaLinksQueue = mediators.mediatorRdfResolveHypermediaLinksQueue; @@ -53,19 +68,40 @@ export class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAs // until the buffer of this iterator must be fully consumed, which will not always be the case. public close(): void { - this.aggregatedStore?.end(); - super.close(); + this.getLinkQueue() + .then(linkQueue => { + if (this.isCloseable(linkQueue)) { + this.aggregatedStore?.end(); + super.close(); + } else { + this.wasForcefullyClosed = true; + } + }) + .catch(error => super.destroy(error)); } public destroy(cause?: Error): void { - this.aggregatedStore?.end(); - super.destroy(cause); + this.getLinkQueue() + .then(linkQueue => { + if (this.isCloseable(linkQueue)) { + this.aggregatedStore?.end(); + super.destroy(cause); + } else { + this.wasForcefullyClosed = true; + } + }) + .catch(error => super.destroy(error)); + } + + protected isCloseable(linkQueue: ILinkQueue): boolean { + return (this.wasForcefullyClosed || linkQueue.isEmpty()) && !this.areIteratorsRunning(); } protected override canStartNewIterator(): boolean { // Also allow sub-iterators to be started if the aggregated store has at least one running iterator. // We need this because there are cases where these running iterators will be consumed before this linked iterator. - return (this.aggregatedStore && this.aggregatedStore.hasRunningIterators()) || super.canStartNewIterator(); + return !this.wasForcefullyClosed && + (this.aggregatedStore && this.aggregatedStore.hasRunningIterators()) || super.canStartNewIterator(); } protected override isRunning(): boolean { @@ -158,6 +194,7 @@ export class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAs } // Aggregate all discovered quads into a store. + this.aggregatedStore?.setBaseMetadata( metadata, false); this.aggregatedStore?.import(quads); // Determine the source @@ -177,7 +214,24 @@ export class MediatedLinkedRdfSourcesAsyncRdfIterator extends LinkedRdfSourcesAs handledDatasets[dataset] = true; } - return { link, source, metadata, handledDatasets }; + return { link, source, metadata: metadata, handledDatasets }; + } + + public async accumulateMetadata( + accumulatedMetadata: MetadataQuads, + appendingMetadata: MetadataQuads, + ): Promise { + return (await this.mediatorMetadataAccumulate.mediate({ + mode: 'append', + accumulatedMetadata, + appendingMetadata, + context: this.context, + })).metadata; + } + + protected updateMetadata(metadataNew: MetadataQuads): void { + super.updateMetadata(metadataNew); + this.aggregatedStore?.setBaseMetadata(metadataNew, true); } } @@ -185,6 +239,7 @@ export interface IMediatorArgs { mediatorDereferenceRdf: MediatorDereferenceRdf; mediatorMetadata: MediatorRdfMetadata; mediatorMetadataExtract: MediatorRdfMetadataExtract; + mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate; mediatorRdfResolveHypermedia: MediatorRdfResolveHypermedia; mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks; mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue; diff --git a/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/MediatedQuadSource.ts b/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/MediatedQuadSource.ts index 003cf8d3bc..55cb13c463 100644 --- a/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/MediatedQuadSource.ts +++ b/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/MediatedQuadSource.ts @@ -53,7 +53,10 @@ export class MediatedQuadSource implements IQuadSource { if (aggregatedStores) { aggregatedStore = aggregatedStores.get(this.firstUrl); if (!aggregatedStore) { - aggregatedStore = new StreamingStoreMetadata(); + aggregatedStore = new StreamingStoreMetadata( + undefined, + (acc, app) => it.accumulateMetadata(acc, app), + ); aggregatedStores.set(this.firstUrl, aggregatedStore); } if (aggregatedStore.started) { @@ -67,7 +70,7 @@ export class MediatedQuadSource implements IQuadSource { } } - const it = new MediatedLinkedRdfSourcesAsyncRdfIterator( + const it: MediatedLinkedRdfSourcesAsyncRdfIterator = new MediatedLinkedRdfSourcesAsyncRdfIterator( this.cacheSize, context, this.forceSourceType, diff --git a/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/StreamingStoreMetadata.ts b/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/StreamingStoreMetadata.ts index 95e0d8be96..549dedcbce 100644 --- a/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/StreamingStoreMetadata.ts +++ b/packages/actor-rdf-resolve-quad-pattern-hypermedia/lib/StreamingStoreMetadata.ts @@ -1,4 +1,7 @@ +// eslint-disable-next-line import/no-nodejs-modules +import type { EventEmitter } from 'events'; import { ClosableTransformIterator } from '@comunica/bus-query-operation'; +import { MetadataValidationState } from '@comunica/metadata'; import type { MetadataQuads, IAggregatedStore } from '@comunica/types'; import type * as RDF from '@rdfjs/types'; import type { AsyncIterator } from 'asynciterator'; @@ -10,6 +13,30 @@ import { StreamingStore } from 'rdf-streaming-store'; export class StreamingStoreMetadata extends StreamingStore implements IAggregatedStore { public started = false; public readonly runningIterators: Set> = new Set>(); + protected readonly metadataAccumulator: + (accumulatedMetadata: MetadataQuads, appendingMetadata: MetadataQuads) => Promise; + + protected baseMetadata: MetadataQuads = { + state: new MetadataValidationState(), + cardinality: { type: 'exact', value: 0 }, + canContainUndefs: false, + }; + + public constructor( + store: RDF.Store | undefined, + metadataAccumulator: + (accumulatedMetadata: MetadataQuads, appendingMetadata: MetadataQuads) => Promise, + ) { + super(store); + this.metadataAccumulator = metadataAccumulator; + } + + public import(stream: RDF.Stream): EventEmitter { + if (!this.ended) { + super.import(stream); + } + return stream; + } public hasRunningIterators(): boolean { return this.runningIterators.size > 0; @@ -22,8 +49,9 @@ export class StreamingStoreMetadata extends StreamingStore implements IAggregate graph?: RDF.Term | null, ): AsyncIterator { // Wrap the raw stream in an AsyncIterator + const rawStream = super.match(subject, predicate, object, graph); const iterator = new ClosableTransformIterator( - super.match(subject, predicate, object, graph), { + rawStream, { autoStart: false, onClose: () => { // Running iterators are deleted once closed or destroyed @@ -33,18 +61,64 @@ export class StreamingStoreMetadata extends StreamingStore implements IAggregate ); // Expose the metadata property containing the cardinality + let count = this.getStore().countQuads(subject!, predicate!, object!, graph!); const metadata: MetadataQuads = { + state: new MetadataValidationState(), cardinality: { - type: 'exact', - value: this.getStore().countQuads(subject!, predicate!, object!, graph!), + type: 'estimate', + value: count, }, canContainUndefs: false, }; iterator.setProperty('metadata', metadata); + iterator.setProperty('lastCount', count); + + // Ever time a new quad is pushed into the iterator, update the metadata + rawStream.on('quad', () => { + iterator.setProperty('lastCount', ++count); + this.updateMetadataState(iterator, count); + }); // Store all running iterators until they close or are destroyed this.runningIterators.add(iterator); return iterator; } + + public setBaseMetadata(metadata: MetadataQuads, updateStates: boolean): void { + this.baseMetadata = { ...metadata }; + this.baseMetadata.cardinality = { type: 'exact', value: 0 }; + + if (updateStates) { + for (const iterator of this.runningIterators) { + const count: number = iterator.getProperty('lastCount')!; + this.updateMetadataState(iterator, count); + } + } + } + + protected updateMetadataState(iterator: AsyncIterator, count: number): void { + // Append the given cardinality to the base metadata + const metadataNew: MetadataQuads = { + state: new MetadataValidationState(), + cardinality: { + type: 'estimate', + value: count, + }, + canContainUndefs: false, + }; + + this.metadataAccumulator(this.baseMetadata, metadataNew) + .then(accumulatedMetadata => { + accumulatedMetadata.state = new MetadataValidationState(); + + // Set the new metadata, and invalidate the previous state + const metadataToInvalidate = iterator.getProperty('metadata'); + iterator.setProperty('metadata', accumulatedMetadata); + metadataToInvalidate?.state.invalidate(); + }) + .catch(() => { + // Void errors + }); + } } diff --git a/packages/actor-rdf-resolve-quad-pattern-hypermedia/package.json b/packages/actor-rdf-resolve-quad-pattern-hypermedia/package.json index 99f0b49f15..0b7fb07eda 100644 --- a/packages/actor-rdf-resolve-quad-pattern-hypermedia/package.json +++ b/packages/actor-rdf-resolve-quad-pattern-hypermedia/package.json @@ -37,18 +37,20 @@ "@comunica/bus-query-operation": "^2.6.8", "@comunica/bus-rdf-metadata": "^2.6.8", "@comunica/bus-rdf-metadata-extract": "^2.6.8", + "@comunica/bus-rdf-metadata-accumulate": "^2.5.0", "@comunica/bus-rdf-resolve-hypermedia": "^2.6.8", "@comunica/bus-rdf-resolve-hypermedia-links": "^2.6.8", "@comunica/bus-rdf-resolve-hypermedia-links-queue": "^2.6.8", "@comunica/bus-rdf-resolve-quad-pattern": "^2.6.8", "@comunica/context-entries": "^2.6.8", "@comunica/core": "^2.6.8", + "@comunica/metadata": "^2.5.0", "@comunica/types": "^2.6.8", "@rdfjs/types": "*", "@types/lru-cache": "^7.0.0", "asynciterator": "^3.8.0", "lru-cache": "^7.0.0", - "rdf-streaming-store": "^1.0.2", + "rdf-streaming-store": "^1.1.0", "readable-stream": "^4.2.0", "sparqlalgebrajs": "^4.0.5" }, diff --git a/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/ActorRdfResolveQuadPatternHypermedia-test.ts b/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/ActorRdfResolveQuadPatternHypermedia-test.ts index b09f6b2090..a1ebf55a2b 100644 --- a/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/ActorRdfResolveQuadPatternHypermedia-test.ts +++ b/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/ActorRdfResolveQuadPatternHypermedia-test.ts @@ -1,11 +1,13 @@ import type { MediatorDereferenceRdf } from '@comunica/bus-dereference-rdf'; import type { MediatorRdfMetadata } from '@comunica/bus-rdf-metadata'; +import type { MediatorRdfMetadataAccumulate } from '@comunica/bus-rdf-metadata-accumulate'; import type { MediatorRdfMetadataExtract } from '@comunica/bus-rdf-metadata-extract'; import type { MediatorRdfResolveHypermedia } from '@comunica/bus-rdf-resolve-hypermedia'; import type { MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links'; import type { MediatorRdfResolveHypermediaLinksQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue'; import { KeysRdfResolveQuadPattern } from '@comunica/context-entries'; import { ActionContext, Bus } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IActionContext } from '@comunica/types'; import 'jest-rdf'; import arrayifyStream from 'arrayify-stream'; @@ -20,6 +22,7 @@ describe('ActorRdfResolveQuadPatternHypermedia', () => { let mediatorDereferenceRdf: MediatorDereferenceRdf; let mediatorMetadata: MediatorRdfMetadata; let mediatorMetadataExtract: MediatorRdfMetadataExtract; + let mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate; let mediatorRdfResolveHypermedia: MediatorRdfResolveHypermedia; let mediatorRdfResolveHypermediaLinks: MediatorRdfResolveHypermediaLinks; let mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQueue; @@ -29,6 +32,7 @@ describe('ActorRdfResolveQuadPatternHypermedia', () => { mediatorDereferenceRdf = utilMediators.mediatorDereferenceRdf; mediatorMetadata = utilMediators.mediatorMetadata; mediatorMetadataExtract = utilMediators.mediatorMetadataExtract; + mediatorMetadataAccumulate = utilMediators.mediatorMetadataAccumulate; mediatorRdfResolveHypermedia = utilMediators.mediatorRdfResolveHypermedia; mediatorRdfResolveHypermediaLinks = utilMediators.mediatorRdfResolveHypermediaLinks; mediatorRdfResolveHypermediaLinksQueue = utilMediators.mediatorRdfResolveHypermediaLinksQueue; @@ -45,6 +49,7 @@ describe('ActorRdfResolveQuadPatternHypermedia', () => { mediatorDereferenceRdf, mediatorMetadata, mediatorMetadataExtract, + mediatorMetadataAccumulate, mediatorRdfResolveHypermedia, mediatorRdfResolveHypermediaLinks, })).toBeInstanceOf(ActorRdfResolveQuadPatternHypermedia); @@ -62,6 +67,7 @@ describe('ActorRdfResolveQuadPatternHypermedia', () => { mediatorDereferenceRdf, mediatorMetadata, mediatorMetadataExtract, + mediatorMetadataAccumulate, mediatorRdfResolveHypermedia, mediatorRdfResolveHypermediaLinks, })).toBeInstanceOf(ActorRdfResolveQuadPatternHypermedia); @@ -92,6 +98,7 @@ describe('ActorRdfResolveQuadPatternHypermedia', () => { httpInvalidator, mediatorMetadata, mediatorMetadataExtract, + mediatorMetadataAccumulate, mediatorDereferenceRdf, mediatorRdfResolveHypermedia, mediatorRdfResolveHypermediaLinks, @@ -231,6 +238,7 @@ describe('ActorRdfResolveQuadPatternHypermedia', () => { httpInvalidator, mediatorMetadata, mediatorMetadataExtract, + mediatorMetadataAccumulate, mediatorDereferenceRdf, mediatorRdfResolveHypermedia, mediatorRdfResolveHypermediaLinks, @@ -263,13 +271,21 @@ describe('ActorRdfResolveQuadPatternHypermedia', () => { quad('s4', 'p4', 'o4'), ]); expect(await new Promise(resolve => data.getProperty('metadata', resolve))) - .toEqual({ firstMeta: true, a: 1 }); + .toEqual({ state: expect.any(MetadataValidationState), + cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, + firstMeta: true, + a: 1 }); }); it('should return a quad stream and metadata, with metadata resolving first', async() => { const { data } = await actor.run({ context, pattern }); expect(await new Promise(resolve => data.getProperty('metadata', resolve))) - .toEqual({ firstMeta: true, a: 1 }); + .toEqual({ + state: expect.any(MetadataValidationState), + firstMeta: true, + a: 1, + cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, + }); expect(await arrayifyStream(data)).toEqualRdfQuadArray([ quad('s1', 'p1', 'o1'), quad('s2', 'p2', 'o2'), @@ -279,4 +295,60 @@ describe('ActorRdfResolveQuadPatternHypermedia', () => { }); }); }); + + describe('An ActorRdfResolveQuadPatternHypermedia instance without mediatorMetadataAccumulate', () => { + let actor: any; + let context: IActionContext; + let pattern: any; + let httpInvalidator: any; + let listener: any; + + beforeEach(() => { + httpInvalidator = { + addInvalidateListener: (l: any) => listener = l, + }; + actor = new ActorRdfResolveQuadPatternHypermedia({ + bus, + cacheSize: 10, + maxIterators: 64, + aggregateStore: false, + httpInvalidator, + mediatorMetadata, + mediatorMetadataExtract, + mediatorDereferenceRdf, + mediatorRdfResolveHypermedia, + mediatorRdfResolveHypermediaLinks, + mediatorRdfResolveHypermediaLinksQueue, + name: 'actor', + }); + context = new ActionContext({ [KeysRdfResolveQuadPattern.source.name]: { value: 'firstUrl' }}); + pattern = quad('?s', 'p1', '?o'); + }); + + it('should return a quad stream and metadata', async() => { + const { data } = await actor.run({ context, pattern }); + expect(await arrayifyStream(data)).toEqualRdfQuadArray([ + quad('s1', 'p1', 'o1'), + quad('s2', 'p2', 'o2'), + quad('s3', 'p3', 'o3'), + quad('s4', 'p4', 'o4'), + ]); + expect(await new Promise(resolve => data.getProperty('metadata', resolve))) + .toEqual({ state: expect.any(MetadataValidationState), + firstMeta: true, + a: 1 }); + }); + + it('should return a quad stream and metadata, with metadata resolving first', async() => { + const { data } = await actor.run({ context, pattern }); + expect(await new Promise(resolve => data.getProperty('metadata', resolve))) + .toEqual({ state: expect.any(MetadataValidationState), firstMeta: true, a: 1 }); + expect(await arrayifyStream(data)).toEqualRdfQuadArray([ + quad('s1', 'p1', 'o1'), + quad('s2', 'p2', 'o2'), + quad('s3', 'p3', 'o3'), + quad('s4', 'p4', 'o4'), + ]); + }); + }); }); diff --git a/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/LinkedRdfSourcesAsyncRdfIterator-test.ts b/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/LinkedRdfSourcesAsyncRdfIterator-test.ts index 8f0e40b2b0..d3d2a14a94 100644 --- a/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/LinkedRdfSourcesAsyncRdfIterator-test.ts +++ b/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/LinkedRdfSourcesAsyncRdfIterator-test.ts @@ -3,6 +3,8 @@ import { Readable } from 'stream'; import { LinkQueueFifo } from '@comunica/actor-rdf-resolve-hypermedia-links-queue-fifo'; import type { ILink } from '@comunica/bus-rdf-resolve-hypermedia-links'; import type { ILinkQueue } from '@comunica/bus-rdf-resolve-hypermedia-links-queue'; +import { MetadataValidationState } from '@comunica/metadata'; +import type { MetadataQuads } from '@comunica/types'; import type * as RDF from '@rdfjs/types'; import { ArrayIterator, wrap } from 'asynciterator'; import { DataFactory } from 'rdf-data-factory'; @@ -43,11 +45,11 @@ class Dummy extends LinkedRdfSourcesAsyncRdfIterator { return { link, handledDatasets: { [link.url]: true }, - metadata: { requestedPage }, + metadata: { requestedPage }, source: { match() { const it = new ArrayIterator([], { autoStart: false }); - it.setProperty('metadata', { subseq: true }); + it.setProperty('metadata', { subseq: true, next: undefined }); if (this.createdSubIterator) { this.createdSubIterator.emit('data', it); } @@ -59,11 +61,11 @@ class Dummy extends LinkedRdfSourcesAsyncRdfIterator { return { link, handledDatasets: { [link.url]: true }, - metadata: { requestedPage, firstPageToken: true, next: `P${requestedPage + 1}` }, + metadata: { requestedPage, firstPageToken: true, next: `P${requestedPage + 1}` }, source: { match: () => { const it = new ArrayIterator([ ...this.data[requestedPage] ], { autoStart: false }); - it.setProperty('metadata', { subseq: true }); + it.setProperty('metadata', { subseq: true, next: `P${requestedPage + 1}` }); if (this.createdSubIterator) { this.createdSubIterator.emit('data', it); } @@ -72,6 +74,13 @@ class Dummy extends LinkedRdfSourcesAsyncRdfIterator { }, }; } + + protected async accumulateMetadata( + accumulatedMetadata: MetadataQuads, + appendingMetadata: MetadataQuads, + ): Promise { + return { ...accumulatedMetadata, ...appendingMetadata }; + } } // Dummy class with a rejecting getNextSource @@ -110,11 +119,11 @@ class DummyMetaOverride extends Dummy { return { link, handledDatasets: { [link.url]: true }, - metadata: { requestedPage }, + metadata: { requestedPage }, source: { match() { const it = new ArrayIterator([], { autoStart: false }); - it.setProperty('metadata', { subseq: true }); + it.setProperty('metadata', { subseq: true, next: undefined }); return it; }, }, @@ -123,7 +132,7 @@ class DummyMetaOverride extends Dummy { return { link, handledDatasets: { [link.url]: true }, - metadata: { firstPageToken: true, next: `P${requestedPage + 1}` }, + metadata: { firstPageToken: true, next: `P${requestedPage + 1}` }, source: { match: () => { const quads = new ArrayIterator([ ...this.data[requestedPage] ], { autoStart: false }); @@ -141,7 +150,7 @@ class DummyMetaOverrideEarly extends Dummy { return { link, handledDatasets: { [link.url]: true }, - metadata: { firstPageToken: true }, + metadata: { firstPageToken: true }, source: { match() { const slowReadable = new Readable(); @@ -163,7 +172,7 @@ class DummyMetaOverrideLate extends Dummy { return { link, handledDatasets: { [link.url]: true }, - metadata: { firstPageToken: true }, + metadata: { firstPageToken: true }, source: { match() { const quads = new ArrayIterator([], { autoStart: false }); @@ -192,7 +201,7 @@ class DummyError extends Dummy { return { link, handledDatasets: { [link.url]: true }, - metadata: { next: 'NEXT' }, + metadata: { next: 'NEXT' }, source: { match() { const quads = new ArrayIterator([], { autoStart: false }); @@ -234,6 +243,16 @@ class DummyErrorLinkQueueLater extends Dummy { } } +// Dummy class with a rejecting accumulateMetadata +class DummyErrorAccumulate extends Dummy { + protected accumulateMetadata( + accumulatedMetadata: MetadataQuads, + appendingMetadata: MetadataQuads, + ): Promise { + return Promise.reject(new Error('accumulateMetadata error')); + } +} + describe('LinkedRdfSourcesAsyncRdfIterator', () => { describe('A LinkedRdfSourcesAsyncRdfIterator instance with negative maxIterators', () => { const data = [[]]; @@ -264,7 +283,7 @@ describe('LinkedRdfSourcesAsyncRdfIterator', () => { expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(4, { P1: true }, true); }); - it('handles metadata for a single page after consuming data', async() => { + it('handles metadata for a single page before consuming data', async() => { const data = [[ [ 'a', 'b', 'c' ], [ 'd', 'e', 'f' ], @@ -274,6 +293,14 @@ describe('LinkedRdfSourcesAsyncRdfIterator', () => { const it = new Dummy(quads, v, v, v, v, 'first'); jest.spyOn( it, 'startIteratorsForNextUrls'); + expect(await new Promise(resolve => it.getProperty('metadata', resolve))).toEqual({ + state: expect.any(MetadataValidationState), + firstPageToken: true, + next: 'P1', + requestedPage: 0, + subseq: true, + }); + await new Promise(resolve => { const result: any = []; it.on('data', d => result.push(d)); @@ -282,16 +309,9 @@ describe('LinkedRdfSourcesAsyncRdfIterator', () => { resolve(); }); }); - - expect(await new Promise(resolve => it.getProperty('metadata', resolve))).toEqual({ - firstPageToken: true, - next: 'P1', - requestedPage: 0, - subseq: true, - }); }); - it('handles metadata for a single page before consuming data', async() => { + it('handles metadata for a single page after consuming data', async() => { const data = [[ [ 'a', 'b', 'c' ], [ 'd', 'e', 'f' ], @@ -301,13 +321,6 @@ describe('LinkedRdfSourcesAsyncRdfIterator', () => { const it = new Dummy(quads, v, v, v, v, 'first'); jest.spyOn( it, 'startIteratorsForNextUrls'); - expect(await new Promise(resolve => it.getProperty('metadata', resolve))).toEqual({ - firstPageToken: true, - next: 'P1', - requestedPage: 0, - subseq: true, - }); - await new Promise(resolve => { const result: any = []; it.on('data', d => result.push(d)); @@ -316,6 +329,14 @@ describe('LinkedRdfSourcesAsyncRdfIterator', () => { resolve(); }); }); + + expect(await new Promise(resolve => it.getProperty('metadata', resolve))).toEqual({ + state: expect.any(MetadataValidationState), + firstPageToken: true, + next: undefined, + requestedPage: 0, + subseq: true, + }); }); it('handles a single empty page', done => { @@ -331,12 +352,20 @@ describe('LinkedRdfSourcesAsyncRdfIterator', () => { }); }); - it('handles metadata for a single empty page after consuming data', async() => { + it('handles metadata for a single empty page before consuming data', async() => { const data = [[]]; const quads = toTerms(data); const it = new Dummy(quads, v, v, v, v, 'first'); jest.spyOn( it, 'startIteratorsForNextUrls'); + expect(await new Promise(resolve => it.getProperty('metadata', resolve))).toEqual({ + state: expect.any(MetadataValidationState), + firstPageToken: true, + next: 'P1', + requestedPage: 0, + subseq: true, + }); + await new Promise(resolve => { const result: any = []; it.on('data', d => result.push(d)); @@ -345,28 +374,14 @@ describe('LinkedRdfSourcesAsyncRdfIterator', () => { resolve(); }); }); - - expect(await new Promise(resolve => it.getProperty('metadata', resolve))).toEqual({ - firstPageToken: true, - next: 'P1', - requestedPage: 0, - subseq: true, - }); }); - it('handles metadata for a single empty page before consuming data', async() => { + it('handles metadata for a single empty page after consuming data', async() => { const data = [[]]; const quads = toTerms(data); const it = new Dummy(quads, v, v, v, v, 'first'); jest.spyOn( it, 'startIteratorsForNextUrls'); - expect(await new Promise(resolve => it.getProperty('metadata', resolve))).toEqual({ - firstPageToken: true, - next: 'P1', - requestedPage: 0, - subseq: true, - }); - await new Promise(resolve => { const result: any = []; it.on('data', d => result.push(d)); @@ -375,6 +390,14 @@ describe('LinkedRdfSourcesAsyncRdfIterator', () => { resolve(); }); }); + + expect(await new Promise(resolve => it.getProperty('metadata', resolve))).toEqual({ + state: expect.any(MetadataValidationState), + firstPageToken: true, + next: undefined, + requestedPage: 0, + subseq: true, + }); }); it('handles a single page when the first source is pre-loaded', async() => { @@ -449,16 +472,11 @@ describe('LinkedRdfSourcesAsyncRdfIterator', () => { await new Promise(resolve => it.on('end', resolve)); expect(result).toEqual(quads.flat()); - expect(( it).startIteratorsForNextUrls).toHaveBeenCalledTimes(9); - expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(1, { first: true }, true); - expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(2, { first: true }, true); - expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(3, { first: true }, false); - expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(4, { first: true }, false); - expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(5, { P1: true }, true); - expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(6, { first: true }, false); - expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(7, { P2: true }, true); - expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(8, { first: true }, false); - expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(9, { P3: true }, true); + expect(( it).startIteratorsForNextUrls).toHaveBeenCalledWith({ first: true }, false); + expect(( it).startIteratorsForNextUrls).toHaveBeenCalledWith({ first: true }, true); + expect(( it).startIteratorsForNextUrls).toHaveBeenCalledWith({ P1: true }, true); + expect(( it).startIteratorsForNextUrls).toHaveBeenCalledWith({ P2: true }, true); + expect(( it).startIteratorsForNextUrls).toHaveBeenCalledWith({ P3: true }, true); }); it('handles multiple pages when the first source is pre-loaded', async() => { @@ -536,17 +554,15 @@ describe('LinkedRdfSourcesAsyncRdfIterator', () => { [ 'g', 'h', '3' ], ], ]).flat()); - expect(( it).startIteratorsForNextUrls).toHaveBeenCalledTimes(10); + expect(( it).startIteratorsForNextUrls).toHaveBeenCalledTimes(8); expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(1, { first: true }, false); expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(2, { first: true }, true); expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(3, { first: true }, false); expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(4, { P1: true }, true); - expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(5, { P1: true }, true); - expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(6, { first: true }, false); - expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(7, { P2: true }, true); + expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(5, { first: true }, false); + expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(6, { P1: true }, true); + expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(7, { first: true }, false); expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(8, { P2: true }, true); - expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(9, { P2: true }, true); - expect(( it).startIteratorsForNextUrls).toHaveBeenNthCalledWith(10, { P2: true }, true); }); it('destroys currentIterators when closed', async() => { @@ -725,6 +741,17 @@ describe('LinkedRdfSourcesAsyncRdfIterator', () => { }); })).rejects.toThrow(new Error('DummyErrorLinkQueueLater')); }); + + it('delegates error events from accumulateMetadata', async() => { + const it = new DummyErrorAccumulate([[], []], v, v, v, v, 'first'); + await expect(new Promise((resolve, reject) => { + it.on('error', reject); + it.on('end', resolve); + it.on('data', () => { + // Do nothing + }); + })).rejects.toThrow(new Error('accumulateMetadata error')); + }); }); }); diff --git a/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/MediatedLinkedRdfSourcesAsyncRdfIterator-test.ts b/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/MediatedLinkedRdfSourcesAsyncRdfIterator-test.ts index ec1a891109..93ac8cf2a4 100644 --- a/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/MediatedLinkedRdfSourcesAsyncRdfIterator-test.ts +++ b/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/MediatedLinkedRdfSourcesAsyncRdfIterator-test.ts @@ -101,12 +101,6 @@ describe('MediatedLinkedRdfSourcesAsyncRdfIterator', () => { ); }); - afterEach(() => { - // This is here since the MediatedLinkedRdfSourcesAsyncRdfIterator is never fully consumed - // This should not be needed, see https://github.com/comunica/comunica/issues/1197 - source.destroy(); - }); - describe('close', () => { it('should not end an undefined aggregated store', async() => { source.close(); @@ -115,6 +109,8 @@ describe('MediatedLinkedRdfSourcesAsyncRdfIterator', () => { it('should end a defined aggregated store', async() => { const aggregatedStore: any = { end: jest.fn(), + setBaseMetadata: jest.fn(), + import: jest.fn(), }; source = new MediatedLinkedRdfSourcesAsyncRdfIterator( 10, @@ -131,8 +127,58 @@ describe('MediatedLinkedRdfSourcesAsyncRdfIterator', () => { ); source.close(); + await new Promise(setImmediate); expect(aggregatedStore.end).toHaveBeenCalledTimes(1); }); + + it('should close if the iterator is closeable', async() => { + source.close(); + await new Promise(setImmediate); + expect(source.closed).toEqual(true); + expect(source.wasForcefullyClosed).toEqual(false); + }); + + it('should close if the iterator is closeable, and end the aggregated store', async() => { + const aggregatedStore: any = { + end: jest.fn(), + setBaseMetadata: jest.fn(), + import: jest.fn(), + }; + source = new MediatedLinkedRdfSourcesAsyncRdfIterator( + 10, + context, + 'forcedType', + s, + p, + o, + g, + 'first', + 64, + aggregatedStore, + mediators, + ); + + source.close(); + await new Promise(setImmediate); + expect(source.closed).toEqual(true); + expect(aggregatedStore.end).toHaveBeenCalled(); + expect(source.wasForcefullyClosed).toEqual(false); + }); + + it('should not close if the iterator is not closeable', async() => { + source.getLinkQueue = async() => ({ isEmpty: () => false }); + source.close(); + await new Promise(setImmediate); + expect(source.closed).toEqual(false); + expect(source.wasForcefullyClosed).toEqual(true); + }); + + it('should destroy if the link queue rejects', async() => { + source.getLinkQueue = () => Promise.reject(new Error('getLinkQueue reject')); + source.close(); + await expect(new Promise((resolve, reject) => source.on('error', reject))) + .rejects.toThrow('getLinkQueue reject'); + }); }); describe('destroy', () => { @@ -143,6 +189,8 @@ describe('MediatedLinkedRdfSourcesAsyncRdfIterator', () => { it('should end a defined aggregated store', async() => { const aggregatedStore: any = { end: jest.fn(), + setBaseMetadata: jest.fn(), + import: jest.fn(), }; source = new MediatedLinkedRdfSourcesAsyncRdfIterator( 10, @@ -159,8 +207,58 @@ describe('MediatedLinkedRdfSourcesAsyncRdfIterator', () => { ); source.destroy(); + await new Promise(setImmediate); expect(aggregatedStore.end).toHaveBeenCalledTimes(1); }); + + it('should close if the iterator is closeable', async() => { + source.destroy(); + await new Promise(setImmediate); + expect(source.closed).toEqual(true); + expect(source.wasForcefullyClosed).toEqual(false); + }); + + it('should close if the iterator is closeable, and end the aggregated store', async() => { + const aggregatedStore: any = { + end: jest.fn(), + setBaseMetadata: jest.fn(), + import: jest.fn(), + }; + source = new MediatedLinkedRdfSourcesAsyncRdfIterator( + 10, + context, + 'forcedType', + s, + p, + o, + g, + 'first', + 64, + aggregatedStore, + mediators, + ); + + source.destroy(); + await new Promise(setImmediate); + expect(source.closed).toEqual(true); + expect(aggregatedStore.end).toHaveBeenCalled(); + expect(source.wasForcefullyClosed).toEqual(false); + }); + + it('should not close if the iterator is not closeable', async() => { + source.getLinkQueue = async() => ({ isEmpty: () => false }); + source.destroy(); + await new Promise(setImmediate); + expect(source.closed).toEqual(false); + expect(source.wasForcefullyClosed).toEqual(true); + }); + + it('should destroy if the link queue rejects', async() => { + source.getLinkQueue = () => Promise.reject(new Error('getLinkQueue reject')); + source.destroy(); + await expect(new Promise((resolve, reject) => source.on('error', reject))) + .rejects.toThrow('getLinkQueue reject'); + }); }); describe('getLinkQueue', () => { @@ -178,8 +276,8 @@ describe('MediatedLinkedRdfSourcesAsyncRdfIterator', () => { it('should throw on a rejecting mediator', async() => { mediatorRdfResolveHypermediaLinksQueue.mediate = () => Promise - .reject(new Error('MediatedLinkRdfSourceAsyncRdfIterator-error')); - await expect(source.getLinkQueue()).rejects.toThrowError('MediatedLinkRdfSourceAsyncRdfIterator-error'); + .reject(new Error('mediatorRdfResolveHypermediaLinksQueue-error')); + await expect(source.getLinkQueue()).rejects.toThrowError('mediatorRdfResolveHypermediaLinksQueue-error'); }); }); @@ -341,5 +439,50 @@ describe('MediatedLinkedRdfSourcesAsyncRdfIterator', () => { await new Promise(setImmediate); }); }); + + describe('isCloseable', () => { + it('should be false for a non-empty link queue', async() => { + const linkQueue = { + isEmpty: () => false, + }; + expect(source.isCloseable(linkQueue)).toEqual(false); + }); + + it('should be true for an empty link queue', async() => { + const linkQueue = { + isEmpty: () => true, + }; + expect(source.isCloseable(linkQueue)).toEqual(true); + }); + + it('should be false for an empty link queue when sub-iterators are running', async() => { + const linkQueue = { + isEmpty: () => true, + }; + source.iteratorsPendingCreation++; + expect(source.isCloseable(linkQueue)).toEqual(false); + }); + + it('should be true for a non-empty link queue, but was forcefully closed', async() => { + const linkQueue = { + isEmpty: () => false, + }; + source.iteratorsPendingCreation++; + source.close(); + await new Promise(setImmediate); + source.iteratorsPendingCreation--; + expect(source.isCloseable(linkQueue)).toEqual(true); + }); + + it('should be false for non-empty link queue, was forcefully closed, and sub-iterators are running', async() => { + const linkQueue = { + isEmpty: () => true, + }; + source.iteratorsPendingCreation++; + source.close(); + await new Promise(setImmediate); + expect(source.isCloseable(linkQueue)).toEqual(false); + }); + }); }); }); diff --git a/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/MediatedQuadSource-test.ts b/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/MediatedQuadSource-test.ts index f5e69a45bd..d12691fc33 100644 --- a/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/MediatedQuadSource-test.ts +++ b/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/MediatedQuadSource-test.ts @@ -2,7 +2,8 @@ import type { MediatorDereferenceRdf, IActionDereferenceRdf, IActorDereferenceRdfOutput } from '@comunica/bus-dereference-rdf'; import { KeysInitQuery, KeysRdfResolveQuadPattern } from '@comunica/context-entries'; import { ActionContext } from '@comunica/core'; -import type { IActionContext } from '@comunica/types'; +import { MetadataValidationState } from '@comunica/metadata'; +import type { IActionContext, MetadataQuads } from '@comunica/types'; import type * as RDF from '@rdfjs/types'; import arrayifyStream from 'arrayify-stream'; import { ArrayIterator } from 'asynciterator'; @@ -79,7 +80,12 @@ describe('MediatedQuadSource', () => { quad('s3', 'p3', 'o3'), quad('s4', 'p4', 'o4'), ]); - expect(await metaPromise).toEqual({ firstMeta: true, a: 1 }); + expect(await metaPromise).toEqual({ + state: expect.any(MetadataValidationState), + firstMeta: true, + a: 1, + cardinality: { type: 'estimate', value: Number.POSITIVE_INFINITY }, + }); }); it('should set the first source after the first match call', async() => { @@ -95,7 +101,10 @@ describe('MediatedQuadSource', () => { source.sourcesState.sources.set('firstUrl', Promise.resolve({ link: { url: 'firstUrl' }, handledDatasets: {}, - metadata: { a: 2 }, + metadata: { state: new MetadataValidationState(), + cardinality: { type: 'exact', value: 0 }, + canContainUndefs: false, + a: 2 }, source: { match() { const it = new ArrayIterator([ @@ -122,14 +131,17 @@ describe('MediatedQuadSource', () => { source.sourcesState.sources.set('firstUrl', Promise.resolve({ link: { url: 'firstUrl' }, handledDatasets: {}, - metadata: { a: 2 }, + metadata: { state: new MetadataValidationState(), + cardinality: { type: 'exact', value: 1 }, + canContainUndefs: false, + a: 2 }, source: { match() { const it = new ArrayIterator([ quad('s1x', 'p1', 'o1'), quad('s2x', 'p2', 'o2'), ], { autoStart: false }); - it.setProperty('metadata', { firstMeta: true }); + it.setProperty('metadata', { firstMeta: true, cardinality: { type: 'exact', value: 1 }}); return it; }, }, @@ -142,7 +154,13 @@ describe('MediatedQuadSource', () => { quad('s3', 'p3', 'o3'), quad('s4', 'p4', 'o4'), ]); - expect(await metaPromise).toEqual({ firstMeta: true, a: 2 }); + expect(await metaPromise).toEqual({ + state: expect.any(MetadataValidationState), + canContainUndefs: false, + cardinality: { type: 'exact', value: 2 }, + firstMeta: true, + a: 2, + }); }); it('should match three chained sources', async() => { @@ -249,6 +267,188 @@ describe('MediatedQuadSource', () => { expect(mediatorRdfResolveHypermedia.mediate).toHaveBeenCalledTimes(4); }); + it('should match three chained sources when queried multiple times in parallel', async() => { + let i = 0; + mediatorRdfResolveHypermediaLinks.mediate = () => Promise.resolve({ links: [{ url: `next${i}` }]}); + mediatorRdfResolveHypermedia.mediate = jest.fn((args: any) => { + if (i < 3) { + i++; + } + return Promise.resolve({ + dataset: `MYDATASET${i}`, + source: { + match() { + const it = new ArrayIterator([ + quad(`s1${i}`, `p1${i}`, `o1${i}`), + quad(`s2${i}`, `p2${i}`, `o2${i}`), + ], { autoStart: false }); + it.setProperty('metadata', { firstMeta: true, cardinality: { type: 'exact', value: 2 }}); + return it; + }, + }, + }); + }); + let j = 0; + mediatorDereferenceRdf.mediate = async({ url }: IActionDereferenceRdf) => { + if (j < 3) { + j++; + } + const data: IActorDereferenceRdfOutput = { + data: new ArrayIterator([ + quad(`s1${j}`, `p1${j}`, `o1${j}`), + quad(`s2${j}`, `p2${j}`, `o2${j}`), + ], { autoStart: false }), + metadata: { triples: true }, + exists: true, + requestTime: 0, + url, + }; + // @ts-expect-error + data.data.setProperty('metadata', { firstMeta: true }); + return data; + }; + const it1 = source.match(v, v, undefined!, v, context); + const it2 = source.match(v, v, undefined!, v, context); + const it3 = source.match(DF.namedNode('s11'), v, undefined!, v, context); + + // Only dummy metadata has been defined yet at this stage + // It2 and 3 are not undefined, as they originate from the streamingstore. + expect(it1.getProperty('metadata')).toBeUndefined(); + expect(it2.getProperty('metadata')).toEqual({ + state: expect.any(MetadataValidationState), + cardinality: { + type: 'estimate', + value: 0, + }, + canContainUndefs: false, + }); + expect(it3.getProperty('metadata')).toEqual({ + state: expect.any(MetadataValidationState), + cardinality: { + type: 'estimate', + value: 0, + }, + canContainUndefs: false, + }); + + // Expect the metadata to be modified for every page + const it1Meta = jest.fn(); + const attachIt1MetaListener = () => { + it1.getProperty('metadata', (metadata: MetadataQuads) => { + it1Meta(metadata); + metadata.state.addInvalidateListener(attachIt1MetaListener); + }); + }; + attachIt1MetaListener(); + const it2Meta = jest.fn(); + const attachIt2MetaListener = () => { + it2.getProperty('metadata', (metadata: MetadataQuads) => { + it2Meta(metadata); + metadata.state.addInvalidateListener(attachIt2MetaListener); + }); + }; + attachIt2MetaListener(); + const it3Meta = jest.fn(); + const attachIt3MetaListener = () => { + it3.getProperty('metadata', (metadata: MetadataQuads) => { + it3Meta(metadata); + metadata.state.addInvalidateListener(attachIt3MetaListener); + }); + }; + attachIt3MetaListener(); + + expect(await arrayifyStream(it1)).toBeRdfIsomorphic([ + quad('s11', 'p11', 'o11'), + quad('s21', 'p21', 'o21'), + quad('s12', 'p12', 'o12'), + quad('s22', 'p22', 'o22'), + quad('s13', 'p13', 'o13'), + quad('s23', 'p23', 'o23'), + ]); + expect(mediatorRdfResolveHypermedia.mediate).toHaveBeenCalledTimes(4); + expect(await arrayifyStream(it2)).toBeRdfIsomorphic([ + quad('s11', 'p11', 'o11'), + quad('s21', 'p21', 'o21'), + quad('s12', 'p12', 'o12'), + quad('s22', 'p22', 'o22'), + quad('s13', 'p13', 'o13'), + quad('s23', 'p23', 'o23'), + ]); + expect(await arrayifyStream(it3)).toBeRdfIsomorphic([ + quad('s11', 'p11', 'o11'), + ]); + expect(mediatorRdfResolveHypermedia.mediate).toHaveBeenCalledTimes(4); + + expect(it1Meta).toHaveBeenCalledTimes(4); + expect(it1Meta).toHaveBeenNthCalledWith(1, { + state: expect.any(MetadataValidationState), + a: 1, + firstMeta: true, + cardinality: { type: 'exact', value: 2 }, + }); + expect(it1Meta).toHaveBeenNthCalledWith(2, { + state: expect.any(MetadataValidationState), + a: 1, + firstMeta: true, + cardinality: { type: 'exact', value: 4 }, + }); + expect(it1Meta).toHaveBeenNthCalledWith(3, { + state: expect.any(MetadataValidationState), + a: 1, + firstMeta: true, + cardinality: { type: 'exact', value: 6 }, + }); + expect(it1Meta).toHaveBeenNthCalledWith(4, { + state: expect.any(MetadataValidationState), + a: 1, + firstMeta: true, + cardinality: { type: 'exact', value: 8 }, + }); + expect(it2Meta).toHaveBeenCalledTimes(9); + expect(it2Meta).toHaveBeenNthCalledWith(1, { + state: expect.any(MetadataValidationState), + canContainUndefs: false, + cardinality: { type: 'estimate', value: 0 }, + }); + expect(it2Meta).toHaveBeenNthCalledWith(2, { + state: expect.any(MetadataValidationState), + a: 1, + cardinality: { type: 'estimate', value: 2 }, + }); + expect(it2Meta).toHaveBeenNthCalledWith(4, { + state: expect.any(MetadataValidationState), + a: 1, + cardinality: { type: 'estimate', value: 4 }, + }); + expect(it2Meta).toHaveBeenNthCalledWith(6, { + state: expect.any(MetadataValidationState), + a: 1, + cardinality: { type: 'estimate', value: 6 }, + }); + expect(it2Meta).toHaveBeenNthCalledWith(8, { + state: expect.any(MetadataValidationState), + a: 1, + cardinality: { type: 'estimate', value: 8 }, + }); + expect(it3Meta).toHaveBeenCalledTimes(6); + expect(it3Meta).toHaveBeenNthCalledWith(1, { + state: expect.any(MetadataValidationState), + canContainUndefs: false, + cardinality: { type: 'estimate', value: 0 }, + }); + expect(it3Meta).toHaveBeenNthCalledWith(2, { + state: expect.any(MetadataValidationState), + a: 1, + cardinality: { type: 'estimate', value: 1 }, + }); + expect(it3Meta).toHaveBeenNthCalledWith(6, { + state: expect.any(MetadataValidationState), + a: 1, + firstMeta: true, + cardinality: { type: 'estimate', value: 1 }, + }); + }); + it('should match three chained sources when queried multiple times for multiple queries', async() => { let i = 0; mediatorRdfResolveHypermediaLinks.mediate = () => Promise.resolve({ links: [{ url: `next${i}` }]}); diff --git a/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/MediatorDereferenceRdf-util.ts b/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/MediatorDereferenceRdf-util.ts index 469ad01570..b452bd4338 100644 --- a/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/MediatorDereferenceRdf-util.ts +++ b/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/MediatorDereferenceRdf-util.ts @@ -5,6 +5,8 @@ import type { MediatorDereferenceRdf, } from '@comunica/bus-dereference-rdf'; import type { MediatorRdfMetadata } from '@comunica/bus-rdf-metadata'; +import type { MediatorRdfMetadataAccumulate, + IActionRdfMetadataAccumulateAppend } from '@comunica/bus-rdf-metadata-accumulate'; import type { MediatorRdfMetadataExtract } from '@comunica/bus-rdf-metadata-extract'; import type { MediatorRdfResolveHypermedia } from '@comunica/bus-rdf-resolve-hypermedia'; import type { MediatorRdfResolveHypermediaLinks } from '@comunica/bus-rdf-resolve-hypermedia-links'; @@ -50,6 +52,30 @@ const mediatorMetadataExtract: MediatorRdfMetadataExtract = { mediate: ({ metadata }: any) => Promise.resolve({ metadata }), }; // @ts-expect-error +const mediatorMetadataAccumulate: MediatorRdfMetadataAccumulate = { + async mediate(action: IActionRdfMetadataAccumulateAppend) { + const metadata = { ...action.accumulatedMetadata }; + if (metadata.cardinality) { + metadata.cardinality = { ...metadata.cardinality }; + } + const subMetadata = action.appendingMetadata; + if (!subMetadata.cardinality) { + // We're already at infinite, so ignore any later metadata + metadata.cardinality = {}; + metadata.cardinality.type = 'estimate'; + metadata.cardinality.value = Number.POSITIVE_INFINITY; + } + if (metadata.cardinality?.value !== undefined && subMetadata.cardinality?.value !== undefined) { + metadata.cardinality.value += subMetadata.cardinality.value; + } + if (subMetadata.cardinality?.type === 'estimate') { + metadata.cardinality.type = 'estimate'; + } + + return { metadata }; + }, +}; +// @ts-expect-error const mediatorRdfResolveHypermedia: MediatorRdfResolveHypermedia = { mediate: ({ forceSourceType, handledDatasets, metadata, quads }: any) => Promise.resolve({ dataset: 'MYDATASET', @@ -69,6 +95,7 @@ const mediatorRdfResolveHypermediaLinksQueue: MediatorRdfResolveHypermediaLinksQ export const mediators = { mediatorMetadata, mediatorMetadataExtract, + mediatorMetadataAccumulate, mediatorDereferenceRdf, mediatorRdfResolveHypermedia, mediatorRdfResolveHypermediaLinks, diff --git a/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/StreamingStoreMetadata-test.ts b/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/StreamingStoreMetadata-test.ts new file mode 100644 index 0000000000..b77a369ce6 --- /dev/null +++ b/packages/actor-rdf-resolve-quad-pattern-hypermedia/test/StreamingStoreMetadata-test.ts @@ -0,0 +1,14 @@ +import { StreamingStoreMetadata } from '../lib/StreamingStoreMetadata'; + +describe('StreamingStoreMetadata', () => { + describe('setBaseMetadata', () => { + it('should ignore metadataAccumulator rejections', () => { + const store = new StreamingStoreMetadata( + undefined, + () => Promise.reject(new Error('StreamingStoreMetadata error')), + ); + const it1 = store.match(); + store.setBaseMetadata({}, true); + }); + }); +}); diff --git a/packages/actor-rdf-resolve-quad-pattern-rdfjs-source/lib/RdfJsQuadSource.ts b/packages/actor-rdf-resolve-quad-pattern-rdfjs-source/lib/RdfJsQuadSource.ts index baa2522984..7a69b63f80 100644 --- a/packages/actor-rdf-resolve-quad-pattern-rdfjs-source/lib/RdfJsQuadSource.ts +++ b/packages/actor-rdf-resolve-quad-pattern-rdfjs-source/lib/RdfJsQuadSource.ts @@ -1,4 +1,5 @@ import type { IQuadSource } from '@comunica/bus-rdf-resolve-quad-pattern'; +import { MetadataValidationState } from '@comunica/metadata'; import type * as RDF from '@rdfjs/types'; import type { AsyncIterator } from 'asynciterator'; import { wrap as wrapAsyncIterator } from 'asynciterator'; @@ -68,6 +69,10 @@ export class RdfJsQuadSource implements IQuadSource { matches.on('data', () => i++); }); } - it.setProperty('metadata', { cardinality: { type: 'exact', value: cardinality }, canContainUndefs: false }); + it.setProperty('metadata', { + state: new MetadataValidationState(), + cardinality: { type: 'exact', value: cardinality }, + canContainUndefs: false, + }); } } diff --git a/packages/actor-rdf-resolve-quad-pattern-rdfjs-source/package.json b/packages/actor-rdf-resolve-quad-pattern-rdfjs-source/package.json index a02734971b..1019869784 100644 --- a/packages/actor-rdf-resolve-quad-pattern-rdfjs-source/package.json +++ b/packages/actor-rdf-resolve-quad-pattern-rdfjs-source/package.json @@ -34,6 +34,7 @@ "dependencies": { "@comunica/bus-rdf-resolve-quad-pattern": "^2.6.8", "@comunica/core": "^2.6.8", + "@comunica/metadata": "^2.5.0", "@comunica/types": "^2.6.8", "@rdfjs/types": "*", "asynciterator": "^3.8.0" diff --git a/packages/actor-rdf-resolve-quad-pattern-rdfjs-source/test/ActorRdfResolveQuadPatternRdfJsSource-test.ts b/packages/actor-rdf-resolve-quad-pattern-rdfjs-source/test/ActorRdfResolveQuadPatternRdfJsSource-test.ts index 76512de699..895d4da03b 100644 --- a/packages/actor-rdf-resolve-quad-pattern-rdfjs-source/test/ActorRdfResolveQuadPatternRdfJsSource-test.ts +++ b/packages/actor-rdf-resolve-quad-pattern-rdfjs-source/test/ActorRdfResolveQuadPatternRdfJsSource-test.ts @@ -2,6 +2,7 @@ import { Readable } from 'stream'; import { ActorRdfResolveQuadPattern } from '@comunica/bus-rdf-resolve-quad-pattern'; import { KeysRdfResolveQuadPattern } from '@comunica/context-entries'; import { ActionContext, Bus } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IActionContext } from '@comunica/types'; import type * as RDF from '@rdfjs/types'; import arrayifyStream from 'arrayify-stream'; @@ -140,7 +141,9 @@ describe('ActorRdfResolveQuadPatternRdfJsSource', () => { DF.quad(DF.namedNode('s2'), DF.namedNode('p'), DF.namedNode('o2')), ]); expect(await new Promise(resolve => data.getProperty('metadata', resolve))) - .toEqual({ cardinality: { type: 'exact', value: 2 }, canContainUndefs: false }); + .toEqual({ cardinality: { type: 'exact', value: 2 }, + canContainUndefs: false, + state: expect.any(MetadataValidationState) }); }); it('should use countQuads for metadata if available', async() => { @@ -154,7 +157,9 @@ describe('ActorRdfResolveQuadPatternRdfJsSource', () => { }; const { data } = await actor.run({ pattern, context }); expect(await new Promise(resolve => data.getProperty('metadata', resolve))) - .toEqual({ cardinality: { type: 'exact', value: 123 }, canContainUndefs: false }); + .toEqual({ cardinality: { type: 'exact', value: 123 }, + canContainUndefs: false, + state: expect.any(MetadataValidationState) }); }); it('should use match for metadata if countQuads is not available', async() => { @@ -168,7 +173,9 @@ describe('ActorRdfResolveQuadPatternRdfJsSource', () => { }; const { data } = await actor.run({ pattern, context }); expect(await new Promise(resolve => data.getProperty('metadata', resolve))) - .toEqual({ cardinality: { type: 'exact', value: 3 }, canContainUndefs: false }); + .toEqual({ cardinality: { type: 'exact', value: 3 }, + canContainUndefs: false, + state: expect.any(MetadataValidationState) }); }); it('should delegate its error event', async() => { diff --git a/packages/bus-query-operation/lib/ActorQueryOperation.ts b/packages/bus-query-operation/lib/ActorQueryOperation.ts index 546ef43660..af4d6ec93f 100644 --- a/packages/bus-query-operation/lib/ActorQueryOperation.ts +++ b/packages/bus-query-operation/lib/ActorQueryOperation.ts @@ -93,9 +93,20 @@ export abstract class ActorQueryOperation extends Actor, T extends RDF.Variable | RDF.QuadTermName>( metadata: () => Promise, ): () => Promise { - let lastReturn: Promise; - // eslint-disable-next-line no-return-assign,@typescript-eslint/no-misused-promises - return () => (lastReturn || (lastReturn = metadata())); + let lastReturn: Promise | undefined; + return () => { + if (!lastReturn) { + lastReturn = metadata(); + lastReturn + .then(lastReturnValue => lastReturnValue.state.addInvalidateListener(() => { + lastReturn = undefined; + })) + .catch(() => { + // Ignore error + }); + } + return lastReturn; + }; } /** diff --git a/packages/bus-query-operation/test/ActorQueryOperation-test.ts b/packages/bus-query-operation/test/ActorQueryOperation-test.ts index e707ead92a..82cccc4347 100644 --- a/packages/bus-query-operation/test/ActorQueryOperation-test.ts +++ b/packages/bus-query-operation/test/ActorQueryOperation-test.ts @@ -1,6 +1,7 @@ import { BindingsFactory } from '@comunica/bindings-factory'; import { KeysInitQuery } from '@comunica/context-entries'; import { ActionContext, Bus } from '@comunica/core'; +import { MetadataValidationState } from '@comunica/metadata'; import type { FunctionArgumentsCache } from '@comunica/types'; import { ArrayIterator } from 'asynciterator'; import type { Algebra } from 'sparqlalgebrajs'; @@ -58,13 +59,29 @@ describe('ActorQueryOperation', () => { }); describe('#cachifyMetadata', () => { - it('should remember an instance', () => { - const cb = jest.fn(() => 'ABC'); + it('should remember an instance', async() => { + let counter = 0; + const cb = jest.fn(async() => ({ state: new MetadataValidationState(), value: counter++ })); const cached = ActorQueryOperation.cachifyMetadata( cb); - expect(cached()).toEqual('ABC'); - expect(cached()).toEqual('ABC'); + expect((await cached()).value).toEqual(0); + expect((await cached()).value).toEqual(0); expect(cb).toHaveBeenCalledTimes(1); }); + + it('should handle invalidation', async() => { + let counter = 0; + const state = new MetadataValidationState(); + const cb = jest.fn(async() => ({ state, value: counter++ })); + const cached = ActorQueryOperation.cachifyMetadata( cb); + expect((await cached()).value).toEqual(0); + expect((await cached()).value).toEqual(0); + expect(cb).toHaveBeenCalledTimes(1); + + state.invalidate(); + expect((await cached()).value).toEqual(1); + expect((await cached()).value).toEqual(1); + expect(cb).toHaveBeenCalledTimes(2); + }); }); describe('#validateQueryOutput', () => { diff --git a/packages/bus-rdf-join/lib/ActorRdfJoin.ts b/packages/bus-rdf-join/lib/ActorRdfJoin.ts index f1ced156cf..50c3521d11 100644 --- a/packages/bus-rdf-join/lib/ActorRdfJoin.ts +++ b/packages/bus-rdf-join/lib/ActorRdfJoin.ts @@ -6,6 +6,7 @@ import { KeysInitQuery } from '@comunica/context-entries'; import type { IAction, IActorArgs, Mediate } from '@comunica/core'; import { Actor } from '@comunica/core'; import type { IMediatorTypeJoinCoefficients } from '@comunica/mediatortype-join-coefficients'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IQueryOperationResultBindings, MetadataBindings, IPhysicalQueryPlanLogger, Bindings, IActionContext, IJoinEntry, IJoinEntryWithMetadata, @@ -200,7 +201,15 @@ export abstract class ActorRdfJoin cardinalityJoined.value *= (await this.mediatorJoinSelectivity.mediate({ entries, context })).selectivity; } + // Propagate metadata invalidations + const state = new MetadataValidationState(); + const invalidateListener = (): void => state.invalidate(); + for (const metadata of metadatas) { + metadata.state.addInvalidateListener(invalidateListener); + } + return { + state, ...partialMetadata, cardinality: { type: cardinalityJoined.type, diff --git a/packages/bus-rdf-join/package.json b/packages/bus-rdf-join/package.json index 2369b1b5a0..3210bf24fc 100644 --- a/packages/bus-rdf-join/package.json +++ b/packages/bus-rdf-join/package.json @@ -36,6 +36,7 @@ "@comunica/context-entries": "^2.6.8", "@comunica/core": "^2.6.8", "@comunica/mediatortype-join-coefficients": "^2.6.8", + "@comunica/metadata": "^2.5.0", "@comunica/types": "^2.6.8", "@rdfjs/types": "*", "rdf-data-factory": "^1.1.1", diff --git a/packages/bus-rdf-join/test/ActorRdfJoin-test.ts b/packages/bus-rdf-join/test/ActorRdfJoin-test.ts index 020f8e2908..f1a99e302e 100644 --- a/packages/bus-rdf-join/test/ActorRdfJoin-test.ts +++ b/packages/bus-rdf-join/test/ActorRdfJoin-test.ts @@ -4,6 +4,7 @@ import { KeysInitQuery } from '@comunica/context-entries'; import type { Actor, IActorTest, Mediator } from '@comunica/core'; import { ActionContext, Bus } from '@comunica/core'; import type { IMediatorTypeJoinCoefficients } from '@comunica/mediatortype-join-coefficients'; +import { MetadataValidationState } from '@comunica/metadata'; import type { IPhysicalQueryPlanLogger, MetadataBindings } from '@comunica/types'; import type * as RDF from '@rdfjs/types'; import { DataFactory } from 'rdf-data-factory'; @@ -83,6 +84,7 @@ describe('ActorRdfJoin', () => { bindingsStream: null, type: 'bindings', metadata: async() => ({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: variables0, @@ -95,6 +97,7 @@ describe('ActorRdfJoin', () => { bindingsStream: null, type: 'bindings', metadata: async() => ({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: false, variables: variables1, @@ -130,16 +133,28 @@ describe('ActorRdfJoin', () => { describe('overlappingVariables', () => { it('should return an empty array if there is no overlap', () => { expect(ActorRdfJoin.overlappingVariables([ - { cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: []}, - { cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: []}, + { + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 10 }, + canContainUndefs: false, + variables: [], + }, + { + state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 10 }, + canContainUndefs: false, + variables: [], + }, ])).toEqual([]); expect(ActorRdfJoin.overlappingVariables([ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('c'), DF.variable('d') ], @@ -150,11 +165,13 @@ describe('ActorRdfJoin', () => { it('should return a correct array if there is overlap', () => { expect(ActorRdfJoin.overlappingVariables([ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('d') ], @@ -163,11 +180,13 @@ describe('ActorRdfJoin', () => { expect(ActorRdfJoin.overlappingVariables([ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -176,11 +195,13 @@ describe('ActorRdfJoin', () => { expect(ActorRdfJoin.overlappingVariables([ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('c'), DF.variable('b') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], @@ -193,11 +214,13 @@ describe('ActorRdfJoin', () => { it('should join variables', () => { expect(ActorRdfJoin.joinVariables([ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [], @@ -206,11 +229,13 @@ describe('ActorRdfJoin', () => { expect(ActorRdfJoin.joinVariables([ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('c'), DF.variable('d') ], @@ -226,11 +251,13 @@ describe('ActorRdfJoin', () => { it('should deduplicate the result', () => { expect(ActorRdfJoin.joinVariables([ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('b'), DF.variable('d') ], @@ -239,11 +266,13 @@ describe('ActorRdfJoin', () => { expect(ActorRdfJoin.joinVariables([ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('a'), DF.variable('b') ], }, { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: [ DF.variable('b'), DF.variable('a') ], @@ -349,7 +378,7 @@ describe('ActorRdfJoin', () => { }); it('should handle entries', async() => { - expect(await ActorRdfJoin.getMetadatas(action.entries)).toEqual([ + expect(await ActorRdfJoin.getMetadatas(action.entries)).toMatchObject([ { cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: []}, { cardinality: { type: 'estimate', value: 5 }, canContainUndefs: false, variables: []}, ]); @@ -360,15 +389,27 @@ describe('ActorRdfJoin', () => { it('should calculate initial request times', async() => { expect(ActorRdfJoin.getRequestInitialTimes([ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, pageSize: 10, requestTime: 10, canContainUndefs: false, variables: [], }, - { cardinality: { type: 'estimate', value: 10 }, pageSize: 10, canContainUndefs: false, variables: []}, - { cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: []}, - { cardinality: { type: 'estimate', value: 10 }, requestTime: 10, canContainUndefs: false, variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 10 }, + pageSize: 10, + canContainUndefs: false, + variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 10 }, + canContainUndefs: false, + variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 10 }, + requestTime: 10, + canContainUndefs: false, + variables: []}, ])).toEqual([ 0, 0, @@ -382,15 +423,27 @@ describe('ActorRdfJoin', () => { it('should calculate item request times', async() => { expect(ActorRdfJoin.getRequestItemTimes([ { + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, pageSize: 10, requestTime: 10, canContainUndefs: false, variables: [], }, - { cardinality: { type: 'estimate', value: 10 }, pageSize: 10, canContainUndefs: false, variables: []}, - { cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: []}, - { cardinality: { type: 'estimate', value: 10 }, requestTime: 10, canContainUndefs: false, variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 10 }, + pageSize: 10, + canContainUndefs: false, + variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 10 }, + canContainUndefs: false, + variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 10 }, + requestTime: 10, + canContainUndefs: false, + variables: []}, ])).toEqual([ 1, 0, @@ -408,11 +461,14 @@ describe('ActorRdfJoin', () => { }); it('should return partial metadata if it is fully valid', async() => { + const state = new MetadataValidationState(); expect(await instance.constructResultMetadata([], [], action.context, { + state, cardinality: { type: 'estimate', value: 10 }, canContainUndefs: true, pageSize: 100, })).toEqual({ + state, cardinality: { type: 'estimate', value: 10 }, canContainUndefs: true, pageSize: 100, @@ -422,25 +478,46 @@ describe('ActorRdfJoin', () => { it('should return not use empty partial metadata', async() => { expect(await instance.constructResultMetadata([], [ - { cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: []}, - { cardinality: { type: 'estimate', value: 2 }, canContainUndefs: true, variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 10 }, + canContainUndefs: false, + variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 2 }, + canContainUndefs: true, + variables: []}, ], action.context, {})).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 20 * 0.8 }, canContainUndefs: true, variables: [], }); expect(await instance.constructResultMetadata([], [ - { cardinality: { type: 'estimate', value: 10 }, canContainUndefs: true, variables: []}, - { cardinality: { type: 'estimate', value: 2 }, canContainUndefs: true, variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 10 }, + canContainUndefs: true, + variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 2 }, + canContainUndefs: true, + variables: []}, ], action.context, {})).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 20 * 0.8 }, canContainUndefs: true, variables: [], }); expect(await instance.constructResultMetadata([], [ - { cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: []}, - { cardinality: { type: 'estimate', value: 2 }, canContainUndefs: false, variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 10 }, + canContainUndefs: false, + variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 2 }, + canContainUndefs: false, + variables: []}, ], action.context, {})).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 20 * 0.8 }, canContainUndefs: false, variables: [], @@ -449,9 +526,16 @@ describe('ActorRdfJoin', () => { it('should combine exact cardinalities', async() => { expect(await instance.constructResultMetadata([], [ - { cardinality: { type: 'exact', value: 10 }, canContainUndefs: false, variables: []}, - { cardinality: { type: 'exact', value: 2 }, canContainUndefs: true, variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'exact', value: 10 }, + canContainUndefs: false, + variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'exact', value: 2 }, + canContainUndefs: true, + variables: []}, ], action.context, {})).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'exact', value: 20 * 0.8 }, canContainUndefs: true, variables: [], @@ -460,9 +544,16 @@ describe('ActorRdfJoin', () => { it('should combine exact and estimate cardinalities', async() => { expect(await instance.constructResultMetadata([], [ - { cardinality: { type: 'estimate', value: 10 }, canContainUndefs: false, variables: []}, - { cardinality: { type: 'exact', value: 2 }, canContainUndefs: true, variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 10 }, + canContainUndefs: false, + variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'exact', value: 2 }, + canContainUndefs: true, + variables: []}, ], action.context, {})).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 20 * 0.8 }, canContainUndefs: true, variables: [], @@ -471,18 +562,47 @@ describe('ActorRdfJoin', () => { it('should join variables', async() => { expect(await instance.constructResultMetadata([], [ - { cardinality: { type: 'exact', value: 10 }, canContainUndefs: false, variables: [ DF.variable('a') ]}, + { state: new MetadataValidationState(), + cardinality: { type: 'exact', value: 10 }, + canContainUndefs: false, + variables: [ DF.variable('a') ]}, { + state: new MetadataValidationState(), cardinality: { type: 'exact', value: 2 }, canContainUndefs: true, variables: [ DF.variable('a'), DF.variable('b') ], }, ], action.context, {})).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'exact', value: 20 * 0.8 }, canContainUndefs: true, variables: [ DF.variable('a'), DF.variable('b') ], }); }); + + it('should handle metadata invalidation', async() => { + const state1 = new MetadataValidationState(); + const metadataOut = await instance.constructResultMetadata([], [ + { state: state1, + cardinality: { type: 'estimate', value: 10 }, + canContainUndefs: false, + variables: []}, + { state: new MetadataValidationState(), + cardinality: { type: 'estimate', value: 2 }, + canContainUndefs: true, + variables: []}, + ], action.context, {}); + expect(metadataOut.state.valid).toBeTruthy(); + + const invalidateLister = jest.fn(); + metadataOut.state.addInvalidateListener(invalidateLister); + expect(invalidateLister).not.toHaveBeenCalled(); + + // After invoking this, we expect the returned metadata to also be invalidated + state1.invalidate(); + expect(metadataOut.state.valid).toBeFalsy(); + expect(invalidateLister).toHaveBeenCalledTimes(1); + }); }); describe('test', () => { @@ -528,11 +648,13 @@ describe('ActorRdfJoin', () => { it('should return a value if both metadata objects are present', () => { action.entries[0].output.metadata = () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: false, variables: [], }); action.entries[1].output.metadata = () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: false, variables: [], @@ -542,6 +664,7 @@ describe('ActorRdfJoin', () => { it('should fail on undefs in left stream', () => { action.entries[0].output.metadata = () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: true, variables: [], @@ -552,6 +675,7 @@ describe('ActorRdfJoin', () => { it('should fail on undefs in right stream', () => { action.entries[1].output.metadata = () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: true, variables: [], @@ -562,11 +686,13 @@ describe('ActorRdfJoin', () => { it('should fail on undefs in left and right stream', () => { action.entries[0].output.metadata = () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: true, variables: [], }); action.entries[1].output.metadata = () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: true, variables: [], @@ -581,6 +707,7 @@ describe('ActorRdfJoin', () => { it('should handle undefs in left stream', () => { action.entries[0].output.metadata = () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: true, variables: [], @@ -596,6 +723,7 @@ describe('ActorRdfJoin', () => { it('should handle undefs in right stream', () => { action.entries[1].output.metadata = () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: true, variables: [], @@ -611,11 +739,13 @@ describe('ActorRdfJoin', () => { it('should handle undefs in left and right stream', () => { action.entries[0].output.metadata = () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: true, variables: [], }); action.entries[1].output.metadata = () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: true, variables: [], @@ -641,22 +771,28 @@ describe('ActorRdfJoin', () => { const runOutput = await instance.run(action); const innerOutput = (await instance.getOutput(action)).result; expect(( runOutput).dummy).toEqual(innerOutput.dummy); - expect(await runOutput.metadata()).toEqual(await innerOutput.metadata()); + expect(await runOutput.metadata()).toEqual({ + ...await innerOutput.metadata(), + state: expect.any(MetadataValidationState), + }); }); it('calculates cardinality if metadata is supplied', async() => { action.entries[0].output.metadata = () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 5 }, canContainUndefs: true, variables: [], }); action.entries[1].output.metadata = () => Promise.resolve({ + state: new MetadataValidationState(), cardinality: { type: 'estimate', value: 10 }, canContainUndefs: true, variables: [], }); await instance.run(action).then(async(result: any) => { return expect(await result.metadata()).toEqual({ + state: expect.any(MetadataValidationState), cardinality: { type: 'estimate', value: 40 }, canContainUndefs: true, variables: [], diff --git a/packages/bus-rdf-metadata-accumulate/.npmignore b/packages/bus-rdf-metadata-accumulate/.npmignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/bus-rdf-metadata-accumulate/README.md b/packages/bus-rdf-metadata-accumulate/README.md new file mode 100644 index 0000000000..9bde1c20c7 --- /dev/null +++ b/packages/bus-rdf-metadata-accumulate/README.md @@ -0,0 +1,25 @@ +# Comunica Bus RDF Metadata Accumulate + +[![npm version](https://badge.fury.io/js/%40comunica%2Fbus-rdf-metadata-accumulate.svg)](https://www.npmjs.com/package/@comunica/bus-rdf-metadata-accumulate) + +A bus for aggregating metadata objects together. + +This module is part of the [Comunica framework](https://github.com/comunica/comunica), +and should only be used by [developers that want to build their own query engine](https://comunica.dev/docs/modify/). + +[Click here if you just want to query with Comunica](https://comunica.dev/docs/query/). + +## Install + +```bash +$ yarn add @comunica/bus-rdf-metadata-accumulate +``` + +## Bus usage + +* **Context**: `"https://linkedsoftwaredependencies.org/bundles/npm/@comunica/bus-rdf-metadata-accumulate/^1.0.0/components/context.jsonld"` +* **Bus name**: `ActorRdfMetadataAccumulate:_default_bus` + +## Creating actors on this bus + +Actors extending [`ActorRdfMetadataAccumulate`](https://comunica.github.io/comunica/classes/bus_rdf_metadata_accumulate.ActorRdfMetadataAccumulate.html) are automatically subscribed to this bus. diff --git a/packages/bus-rdf-metadata-accumulate/lib/ActorRdfMetadataAccumulate.ts b/packages/bus-rdf-metadata-accumulate/lib/ActorRdfMetadataAccumulate.ts new file mode 100644 index 0000000000..a541cb653b --- /dev/null +++ b/packages/bus-rdf-metadata-accumulate/lib/ActorRdfMetadataAccumulate.ts @@ -0,0 +1,66 @@ +import type { IAction, IActorArgs, IActorOutput, IActorTest, Mediate } from '@comunica/core'; +import { Actor } from '@comunica/core'; +import type { MetadataQuads } from '@comunica/types'; + +/** + * A comunica actor for rdf-metadata-accumulate events. + * + * Actor types: + * * Input: IActionRdfMetadataAccumulate: The metadata objects to accumulate, + * or a trigger for initializing a new value. + * * Test: + * * Output: IActorRdfMetadataAccumulateOutput: The accumulated metadata object. + * + * @see IActionRdfMetadataAccumulate + * @see IActorRdfMetadataAccumulateOutput + */ +export abstract class ActorRdfMetadataAccumulate + extends Actor { + /** + * @param args - @defaultNested { a } bus + */ + public constructor(args: IActorRdfMetadataAccumulateArgs) { + super(args); + } +} + +export type IActionRdfMetadataAccumulate = IActionRdfMetadataAccumulateInitialize | IActionRdfMetadataAccumulateAppend; + +export interface IActionRdfMetadataAccumulateInitialize extends IAction { + /** + * Indicates that metadata fields should be initialized to default values. + */ + mode: 'initialize'; +} + +export interface IActionRdfMetadataAccumulateAppend extends IAction { + /** + * Indicates that metadata fields should be accumulated. + * The initialize step is guaranteed to have been called before this on the `accumulatedMetadata` object. + */ + mode: 'append'; + /** + * The metadata object that already has some accumulated fields. + * This object should not be mutated. + */ + accumulatedMetadata: MetadataQuads; + /** + * The metadata object with fields to append. + * This object should not be mutated. + */ + appendingMetadata: MetadataQuads; +} + +export interface IActorRdfMetadataAccumulateOutput extends IActorOutput { + /** + * The initialized or accumulated metadata object. + * This should only contain those fields that are applicable to this actor. + */ + metadata: Partial; +} + +export type IActorRdfMetadataAccumulateArgs = IActorArgs< +IActionRdfMetadataAccumulate, IActorTest, IActorRdfMetadataAccumulateOutput>; + +export type MediatorRdfMetadataAccumulate = Mediate< +IActionRdfMetadataAccumulate, IActorRdfMetadataAccumulateOutput>; diff --git a/packages/bus-rdf-metadata-accumulate/lib/index.ts b/packages/bus-rdf-metadata-accumulate/lib/index.ts new file mode 100644 index 0000000000..95d18587f2 --- /dev/null +++ b/packages/bus-rdf-metadata-accumulate/lib/index.ts @@ -0,0 +1 @@ +export * from './ActorRdfMetadataAccumulate'; diff --git a/packages/bus-rdf-metadata-accumulate/package.json b/packages/bus-rdf-metadata-accumulate/package.json new file mode 100644 index 0000000000..cfc788669f --- /dev/null +++ b/packages/bus-rdf-metadata-accumulate/package.json @@ -0,0 +1,42 @@ +{ + "name": "@comunica/bus-rdf-metadata-accumulate", + "version": "2.5.1", + "description": "A comunica bus for rdf-metadata-accumulate events.", + "lsd:module": true, + "main": "lib/index.js", + "typings": "lib/index", + "repository": { + "type": "git", + "url": "https://github.com/comunica/comunica.git", + "directory": "packages/bus-rdf-metadata-accumulate" + }, + "publishConfig": { + "access": "public" + }, + "sideEffects": false, + "keywords": [ + "comunica", + "bus", + "rdf-metadata-accumulate" + ], + "license": "MIT", + "bugs": { + "url": "https://github.com/comunica/comunica/issues" + }, + "homepage": "https://comunica.dev/", + "files": [ + "components", + "lib/**/*.d.ts", + "lib/**/*.js", + "lib/**/*.js.map" + ], + "dependencies": { + "@comunica/core": "^2.5.0", + "@comunica/types": "^2.5.0" + }, + "scripts": { + "build": "npm run build:ts && npm run build:components", + "build:ts": "node \"../../node_modules/typescript/bin/tsc\"", + "build:components": "componentsjs-generator" + } +} diff --git a/packages/bus-rdf-metadata-extract/README.md b/packages/bus-rdf-metadata-extract/README.md index 1e66fae63a..60eba3910b 100644 --- a/packages/bus-rdf-metadata-extract/README.md +++ b/packages/bus-rdf-metadata-extract/README.md @@ -22,5 +22,5 @@ $ yarn add @comunica/bus-rdf-metadata-extract ## Creating actors on this bus -Actors extending [`ActorRdfMetadataExtract`](https://comunica.github.io/comunica/classes/bus_rdf_metadata_extract.actorrdfmetadataextract.html) are automatically subscribed to this bus. +Actors extending [`ActorRdfMetadataExtract`](https://comunica.github.io/comunica/classes/bus_rdf_metadata_extract.ActorRdfMetadataExtract.html) are automatically subscribed to this bus. diff --git a/packages/bus-rdf-resolve-quad-pattern/lib/ActorRdfResolveQuadPatternSource.ts b/packages/bus-rdf-resolve-quad-pattern/lib/ActorRdfResolveQuadPatternSource.ts index 6817d61af7..9fca3891e7 100644 --- a/packages/bus-rdf-resolve-quad-pattern/lib/ActorRdfResolveQuadPatternSource.ts +++ b/packages/bus-rdf-resolve-quad-pattern/lib/ActorRdfResolveQuadPatternSource.ts @@ -58,9 +58,11 @@ export interface IQuadSource { /** * Returns a (possibly lazy) stream that processes all quads matching the pattern. * - * The returned stream MUST expose the property 'metadata'. + * The returned stream MUST expose the property 'metadata' of type `MetadataQuads`. * The implementor is reponsible for handling cases where 'metadata' * is being called without the stream being in flow-mode. + * This metadata object can become invalidated (see `metadata.state`), + * in which case the 'metadata' property must and will be updated. * * @param {RDF.Term} subject The exact subject to match, variable is wildcard. * @param {RDF.Term} predicate The exact predicate to match, variable is wildcard. diff --git a/packages/metadata/.npmignore b/packages/metadata/.npmignore new file mode 100644 index 0000000000..82f1b83e5a --- /dev/null +++ b/packages/metadata/.npmignore @@ -0,0 +1,2 @@ +bin/**/*.ts +index.ts diff --git a/packages/metadata/README.md b/packages/metadata/README.md new file mode 100644 index 0000000000..065423cc36 --- /dev/null +++ b/packages/metadata/README.md @@ -0,0 +1,16 @@ +# Comunica Metadata + +[![npm version](https://badge.fury.io/js/%40comunica%2Fmetadata.svg)](https://www.npmjs.com/package/@comunica/metadata) + +A collection of metadata helpers for Comunica. + +This module is part of the [Comunica framework](https://github.com/comunica/comunica), +and should only be used by [developers that want to build their own query engine](https://comunica.dev/docs/modify/). + +[Click here if you just want to query with Comunica](https://comunica.dev/docs/query/). + +## Install + +```bash +$ yarn add @comunica/metadata +``` diff --git a/packages/metadata/lib/MetadataValidationState.ts b/packages/metadata/lib/MetadataValidationState.ts new file mode 100644 index 0000000000..53cb70973b --- /dev/null +++ b/packages/metadata/lib/MetadataValidationState.ts @@ -0,0 +1,22 @@ +import type { IMetadataValidationState } from '@comunica/types'; + +/** + * Reusable implementation for metadata validation states. + */ +export class MetadataValidationState implements IMetadataValidationState { + private readonly invalidateListeners: (() => void)[] = []; + public valid = true; + + public addInvalidateListener(listener: () => void): void { + this.invalidateListeners.push(listener); + } + + public invalidate(): void { + if (this.valid) { + this.valid = false; + for (const invalidateListener of this.invalidateListeners) { + invalidateListener(); + } + } + } +} diff --git a/packages/metadata/lib/index.ts b/packages/metadata/lib/index.ts new file mode 100644 index 0000000000..27f2d54bc6 --- /dev/null +++ b/packages/metadata/lib/index.ts @@ -0,0 +1 @@ +export * from './MetadataValidationState'; diff --git a/packages/metadata/package.json b/packages/metadata/package.json new file mode 100644 index 0000000000..a8c95d10e3 --- /dev/null +++ b/packages/metadata/package.json @@ -0,0 +1,43 @@ +{ + "name": "@comunica/metadata", + "version": "2.5.0", + "description": "Metadata helpers for Comunica", + "lsd:module": true, + "main": "lib/index.js", + "typings": "lib/index", + "repository": { + "type": "git", + "url": "https://github.com/comunica/comunica.git", + "directory": "packages/metadata" + }, + "publishConfig": { + "access": "public" + }, + "sideEffects": false, + "keywords": [ + "comunica", + "metadata" + ], + "license": "MIT", + "bugs": { + "url": "https://github.com/comunica/comunica/issues" + }, + "homepage": "https://comunica.dev/", + "files": [ + "components", + "lib/**/*.d.ts", + "lib/**/*.js", + "lib/**/*.js.map", + "bin/**/*.d.ts", + "bin/**/*.js", + "bin/**/*.js.map" + ], + "dependencies": { + "@comunica/types": "^2.5.0" + }, + "scripts": { + "build": "npm run build:ts && npm run build:components", + "build:ts": "node \"../../node_modules/typescript/bin/tsc\"", + "build:components": "componentsjs-generator" + } +} diff --git a/packages/metadata/test/MetadataValidationState-test.ts b/packages/metadata/test/MetadataValidationState-test.ts new file mode 100644 index 0000000000..4dc6623034 --- /dev/null +++ b/packages/metadata/test/MetadataValidationState-test.ts @@ -0,0 +1,51 @@ +import type { IMetadataValidationState } from '@comunica/types'; +import { MetadataValidationState } from '../lib/MetadataValidationState'; + +describe('MetadataValidationState', () => { + let state: IMetadataValidationState; + + beforeEach(() => { + state = new MetadataValidationState(); + }); + + describe('without invalidate listeners', () => { + it('can be invalidated', () => { + expect(state.valid).toBeTruthy(); + state.invalidate(); + expect(state.valid).toBeFalsy(); + }); + }); + + describe('with invalidate listeners', () => { + let listener1: any; + let listener2: any; + beforeEach(() => { + listener1 = jest.fn(); + listener2 = jest.fn(); + state.addInvalidateListener(listener1); + state.addInvalidateListener(listener2); + }); + + it('can be invalidated', () => { + expect(state.valid).toBeTruthy(); + state.invalidate(); + expect(state.valid).toBeFalsy(); + + expect(listener1).toHaveBeenCalledTimes(1); + expect(listener2).toHaveBeenCalledTimes(1); + }); + + it('can be invalidated only', () => { + expect(state.valid).toBeTruthy(); + state.invalidate(); + expect(state.valid).toBeFalsy(); + state.invalidate(); + expect(state.valid).toBeFalsy(); + state.invalidate(); + expect(state.valid).toBeFalsy(); + + expect(listener1).toHaveBeenCalledTimes(1); + expect(listener2).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/packages/types/lib/IAggregatedStore.ts b/packages/types/lib/IAggregatedStore.ts index 2cceebc3f4..925fc3ca90 100644 --- a/packages/types/lib/IAggregatedStore.ts +++ b/packages/types/lib/IAggregatedStore.ts @@ -2,6 +2,7 @@ import type { EventEmitter } from 'events'; import type * as RDF from '@rdfjs/types'; import type { AsyncIterator } from 'asynciterator'; +import type { MetadataQuads } from './IMetadata'; /** * A StreamingStore allows data lookup and insertion to happen in parallel. @@ -32,6 +33,13 @@ export interface IAggregatedStore void; + /** + * Update the metadata of the base iterator, from which the aggregated store is being populated. + * @param metadata The metadata object. + * @param updateState If the metadata state of derived iterators should be immediately updated. + */ + setBaseMetadata: (metadata: MetadataQuads, updateStates: boolean) => void; + match: ( subject?: RDF.Term | null, predicate?: RDF.Term | null, diff --git a/packages/types/lib/IMetadata.ts b/packages/types/lib/IMetadata.ts index ea7e9a823b..f285c4a391 100644 --- a/packages/types/lib/IMetadata.ts +++ b/packages/types/lib/IMetadata.ts @@ -5,10 +5,15 @@ import type * as RDF from '@rdfjs/types'; * This interface still allows other non-standard metadata entries to be added. */ export interface IMetadata extends Record { + /** + * The validity state of this metadata object. + */ + state: IMetadataValidationState; + /** * An estimate of the number of bindings in the source. */ - cardinality: RDF.QueryResultCardinality; + cardinality: QueryResultCardinality; /** * If any of the bindings could contain an undefined variable binding. * If this is false, then all variables are guaranteed to have a defined bound value in the bindingsStream. @@ -57,3 +62,38 @@ export type MetadataBindings = IMetadata & { variables: RDF.Variable[]; }; export type MetadataQuads = IMetadata; + +export type QueryResultCardinality = RDF.QueryResultCardinality & { + /** + * If this field is set, this means that the cardinality is defined across this whole dataset. + * If this field is not set, then the cardinality is only defined for the current stream. + */ + dataset?: string; +}; + +/** + * Represents the validity of a metadata object. + */ +export interface IMetadataValidationState { + /** + * If the metadata object is valid. + * + * If it is invalid, the metadata values should be considered outdated, and a new version should be requested. + */ + valid: boolean; + /** + * Mark the metadta object as invalid. + * + * This will set the `valid` field to false, and invoke the invalidation listeners. + */ + invalidate: () => void; + /** + * Add an listener that will be invoked when the metadata object becomes invalid. + * + * No expensive operations should be done in these listeners, only other invalidations. + * If other operations should be done, those should be scheduled in next ticks. + * + * @param listener An invalidation listener. + */ + addInvalidateListener: (listener: () => void) => void; +} diff --git a/packages/types/lib/IQueryOperationResult.ts b/packages/types/lib/IQueryOperationResult.ts index ecdc271161..da8684a0fd 100644 --- a/packages/types/lib/IQueryOperationResult.ts +++ b/packages/types/lib/IQueryOperationResult.ts @@ -25,11 +25,18 @@ export interface IQueryOperationResultStream< > extends IQueryOperationResultBase { /** * Callback that returns a promise that resolves to the metadata about the stream. + * * This can contain things like the estimated number of total stream elements, * or the order in which the bindings appear. + * * This callback can be invoked multiple times. - * The actors that return this metadata will make sure that multiple calls properly cache this promise. + * Each invocation can return a different metadata object, + * if the previous one would have become invalidated (see `metadata.state`). + * The actors that return this metadata will make sure that multiple calls properly cache this promise, + * and that the cached object is properly invalidated if needed. * Metadata will not be collected until this callback is invoked. + * + * This callback should resolve quickly, so it is safe to call and immediately `await` it. */ metadata: () => Promise; } diff --git a/yarn.lock b/yarn.lock index f8b255ce9c..b2c85fab79 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11966,10 +11966,10 @@ rdf-store-stream@^1.1.0, rdf-store-stream@^1.3.1: "@rdfjs/types" "*" n3 "^1.11.1" -rdf-streaming-store@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/rdf-streaming-store/-/rdf-streaming-store-1.0.2.tgz#e373c1aa633c047c339ae023233d801c2d1c9b30" - integrity sha512-+clYrm8tSfAhTBRZ6NIE5i/JmYZE0B/2Vu8SOCel+lfarjXZwlejoVeA+a/AHcl+lrF2yGh+iw16SsHoE23uTA== +rdf-streaming-store@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/rdf-streaming-store/-/rdf-streaming-store-1.1.0.tgz#a00be02eeb1616e577a095dad9afe96d9106fcfa" + integrity sha512-C/5NTKGpKrNJ5VUo42DtGFsXYDlP3rx/u4C6gEBuSn+6eVFahjFdUDgNGcPtVyhCQkctRCj3GT1lX9UeyoXWtw== dependencies: "@rdfjs/types" "*" "@types/n3" "^1.10.4"