Skip to content

/ dsp-api

Permalink
Browse files
Loading status checks…

feat(api-v2): Remove ForbiddenResource (#1615)

  • Loading branch information
@benjamingeer
benjamingeer committed Apr 7, 2020
Verified
This commit was created on GitHub.com and signed with GitHub’s verified signature.
GPG key ID: 4AEE18F83AFDEB23 Learn about signing commits
1 parent 23d51ce commit 992596e3e031105675ae66e8440b979b64cd08ad
Unified Split
Showing with 2,816 additions and 3,194 deletions.
  1. +16 −5 docs/src/paradox/03-apis/api-v2/query-language.md
  2. +31 −23 docs/src/paradox/05-internals/design/api-v2/gravsearch.md
  3. +1 −14 knora-ontologies/knora-base.ttl
  4. +2 −1 upgrade/src/main/scala/org.knora.upgrade/Main.scala
  5. +111 −0 webapi/_test_data/all_data/anything-data.ttl
  6. +0 −23 webapi/_test_data/all_data/system-data.ttl
  7. +28 −28 webapi/src/main/scala/org/knora/webapi/OntologyConstants.scala
  8. +1 −1 webapi/src/main/scala/org/knora/webapi/Settings.scala
  9. +10 −0 webapi/src/main/scala/org/knora/webapi/SharedTestDataADM.scala
  10. +21 −0 ...ra/webapi/messages/v2/responder/ontologymessages/KnoraBaseToApiV2ComplexTransformationRules.scala
  11. +21 −0 ...ora/webapi/messages/v2/responder/ontologymessages/KnoraBaseToApiV2SimpleTransformationRules.scala
  12. +63 −21 ...i/src/main/scala/org/knora/webapi/messages/v2/responder/resourcemessages/ResourceMessagesV2.scala
  13. +1 −1 webapi/src/main/scala/org/knora/webapi/package.scala
  14. +2 −2 webapi/src/main/scala/org/knora/webapi/responders/Responder.scala
  15. +57 −79 webapi/src/main/scala/org/knora/webapi/responders/v2/ResourcesResponderV2.scala
  16. +8 −8 webapi/src/main/scala/org/knora/webapi/responders/v2/ResponderWithStandoffV2.scala
  17. +52 −160 webapi/src/main/scala/org/knora/webapi/responders/v2/SearchResponderV2.scala
  18. +7 −9 webapi/src/main/scala/org/knora/webapi/responders/v2/StandoffResponderV2.scala
  19. +6 −7 webapi/src/main/scala/org/knora/webapi/responders/v2/ValuesResponderV2.scala
  20. +2 −1 webapi/src/main/scala/org/knora/webapi/routing/v2/SearchRouteV2.scala
  21. +464 −399 webapi/src/main/scala/org/knora/webapi/util/ConstructResponseUtilV2.scala
  22. +8 −44 webapi/src/test/resources/test-data/ontologyR2RV2/knoraApiOntologySimple.jsonld
  23. +161 −191 webapi/src/test/resources/test-data/ontologyR2RV2/knoraApiOntologySimple.rdf
  24. +5 −23 webapi/src/test/resources/test-data/ontologyR2RV2/knoraApiOntologySimple.ttl
  25. +8 −134 webapi/src/test/resources/test-data/ontologyR2RV2/knoraApiOntologyWithValueObjects.jsonld
  26. +1,422 −1,530 webapi/src/test/resources/test-data/ontologyR2RV2/knoraApiOntologyWithValueObjects.rdf
  27. +5 −77 webapi/src/test/resources/test-data/ontologyR2RV2/knoraApiOntologyWithValueObjects.ttl
  28. +82 −0 webapi/src/test/resources/test-data/resourcesR2RV2/ThingWithOneDeletedResource.jsonld
  29. +56 −0 webapi/src/test/resources/test-data/resourcesR2RV2/ThingWithOneHiddenResource.jsonld
  30. +1 −0 webapi/src/test/resources/test-data/searchR2RV2/NarrFulltextSearch.jsonld
  31. +1 −0 webapi/src/test/resources/test-data/searchR2RV2/PagesOfNarrenschiffOrderedBySeqnum.jsonld
  32. +1 −0 webapi/src/test/resources/test-data/searchR2RV2/PagesOfNarrenschiffOrderedBySeqnumNextOffset.jsonld
  33. +1 −0 webapi/src/test/resources/test-data/searchR2RV2/ThingWithBooleanOptionalOffset0.jsonld
  34. +2 −27 webapi/src/test/resources/test-data/searchR2RV2/ThingWithHiddenThing.jsonld
  35. +1 −96 webapi/src/test/resources/test-data/searchR2RV2/ThingsWithOptionalDecimalGreaterThan1.jsonld
  36. +0 −24 ...earchR2RV2/{searchResponseWithforbiddenResource.jsonld → searchResponseWithHiddenResource.jsonld}
  37. +1 −72 webapi/src/test/resources/test-data/searchR2RV2/thingWithOptionalDateSortedDesc.jsonld
  38. +3 −2 webapi/src/test/scala/org/knora/webapi/CoreSpec.scala
  39. +34 −0 webapi/src/test/scala/org/knora/webapi/e2e/v2/ResourcesRouteV2E2ESpec.scala
  40. +1 −1 webapi/src/test/scala/org/knora/webapi/e2e/v2/SearchRouteV2R2RSpec.scala
  41. +7 −35 webapi/src/test/scala/org/knora/webapi/responders/v2/ResourcesResponderV2Spec.scala
  42. +19 −28 webapi/src/test/scala/org/knora/webapi/responders/v2/ResourcesResponderV2SpecFullData.scala
  43. +1 −1 webapi/src/test/scala/org/knora/webapi/responders/v2/ResourcesResponseCheckerV2.scala
  44. +3 −4 webapi/src/test/scala/org/knora/webapi/responders/v2/ResourcesResponseCheckerV2SpecFullData.scala
  45. +6 −6 webapi/src/test/scala/org/knora/webapi/responders/v2/SearchResponderV2Spec.scala
  46. +5 −8 webapi/src/test/scala/org/knora/webapi/responders/v2/SearchResponderV2SpecFullData.scala
  47. +62 −100 webapi/src/test/scala/org/knora/webapi/responders/v2/ValuesResponderV2Spec.scala
  48. +16 −9 webapi/src/test/scala/org/knora/webapi/util/ConstructResponseUtilV2Spec.scala
21 docs/src/paradox/03-apis/api-v2/query-language.md
View file
@@ -186,9 +186,21 @@ that refer to events that took place within a certain date range.

Each matching resource is returned with the values that the user has
permission to see. If the user does not have permission to see a matching
main resource, it is replaced by a proxy resource called
`knora-api:ForbiddenResource`. If a user does not have permission to see
a matching dependent resource, only its IRI is returned.
main resource, it is hidden in the results. If a user does not have
permission to see a matching dependent resource, the link value is hidden.

## Paging

Gravsearch results are returned in pages. The maximum number of main
resources per page is determined by Knora (and can be configured
in `application.conf` via the setting `app/v2/resources-sequence/results-per-page`).
If some resources have been filtered out because the user does not have
permission to see them, a page could contain fewer results, or no results.
If it is possible that more results are available in subsequent pages,
the Gravsearch response will contain the predicate `knora-api:mayHaveMoreResults`
with the boolean value `true`, otherwise it will not contain this predicate.
Therefore, to retrieve all available results, the client must request each page
one at a time, until the response does not contain `knora-api:mayHaveMoreResults`.

## Inference

@@ -227,8 +239,7 @@ clauses use the following patterns, with the specified restrictions:
- `OFFSET`: the `OFFSET` is needed for paging. It does not actually
refer to the number of triples to be returned, but to the
requested page of results. The default value is 0, which refers
to the first page of results. The number of results per page is
defined in `app/v2` in `application.conf`.
to the first page of results.
- `ORDER BY`: In SPARQL, the result of a `CONSTRUCT` query is an
unordered set of triples. However, a Gravsearch query returns an
ordered list of resources, which can be ordered by the values of
54 docs/src/paradox/05-internals/design/api-v2/gravsearch.md
View file
@@ -228,32 +228,39 @@ the main query can specifically ask for more detailed information on these resou

#### Generating the Main Query

The classes involved in generating prequeries can be found in `org.knora.webapi.responders.v2.search.gravsearch.mainquery`.

The main query is a SPARQL CONSTRUCT query. Its generation is handled by the method `GravsearchMainQueryGenerator.createMainQuery`.
It takes three arguments: `mainResourceIris: Set[IriRef], dependentResourceIris: Set[IriRef], valueObjectIris: Set[IRI]`.
From the given IRIs, statements are generated that ask for complete information on *exactly* these resources and values.
For any given resource IRI, only the values present in `valueObjectIris` are to be queried.
This is achieved by using SPARQL's `VALUES` expression for the main resource and dependent resources as well as for values.
The classes involved in generating the main query can be found in
`org.knora.webapi.responders.v2.search.gravsearch.mainquery`.

The main query is a SPARQL CONSTRUCT query. Its generation is handled by the
method `GravsearchMainQueryGenerator.createMainQuery`.
It takes three arguments: `mainResourceIris: Set[IriRef], dependentResourceIris:
Set[IriRef], valueObjectIris: Set[IRI]`. From the given Iris, statements are
generated that ask for complete information on *exactly* these resources and
values. For any given resource Iri, only the values present in
`valueObjectIris` are to be queried. This is achieved by using SPARQL's
`VALUES` expression for the main resource and dependent resources as well as
for values.

#### Processing the Main Query's results

When processing the main query's results, permissions are checked and resources and values that the user did not explicitly ask for in the input query are filtered out. This is implemented in `MainQueryResultProcessor`.

The method `getMainQueryResultsWithFullGraphPattern` takes the main query's results as an input and makes sure that the client has sufficient permissions on the results.
A main resource and its dependent resources and values are only returned if the user has view permissions on all the resources and value objects present in the main query.
Otherwise the method suppresses the main resource.
To do the permission checking, the results of the main query are passed to `ConstructResponseUtilV2` which transforms a `SparqlConstructResponse` (a set of RDF triples)
into a structure organized by main resource IRIs. In this structure, dependent resources and values are nested can be accessed via their main resource.
`SparqlConstructResponse` suppresses all resources and values the user has insufficient permissions on.
For each main resource, a check is performed for the presence of all resources and values after permission checking.

The method `getRequestedValuesFromResultsWithFullGraphPattern` filters out those resources and values that the user does not want to be returned by the query.
All the resources and values not present in the input query's CONSTRUCT clause are filtered out. This only happens after permission checking.

The main resources that have been filtered out due to insufficient permissions are represented by the placeholder `ForbiddenResource`.
This placeholder stands for a main resource that cannot be returned, nevertheless it informs the client that such a resource exists.
This is necessary for a consistent behaviour when doing paging.
To do the permission checking, the results of the main query are passed to
`ConstructResponseUtilV2.splitMainResourcesAndValueRdfData`,
which transforms a `SparqlConstructResponse` (a set of RDF triples)
into a structure organized by main resource Iris. In this structure, dependent
resources and values are nested and can be accessed via their main resource,
and resources and values that the user does not have permission to see are
filtered out. As a result, a page of results may contain fewer than the maximum
allowed number of results per page, even if more pages of results are available.

`MainQueryResultProcessor.getRequestedValuesFromResultsWithFullGraphPattern`
then filters out values that the user did not explicitly ask for in the input
query.

Finally, `ConstructResponseUtilV2.createApiResponse` transforms the query
results into an API response (a `ReadResourcesSequenceV2`). If the number
of main resources found (even if filtered out because of permissions) is equal
to the maximum allowed page size, the predicate
`knora-api:mayHaveMoreResults: true` is included in the response.

## Inference

@@ -279,5 +286,6 @@ as an optimisation if the triplestore provides it. For example, the virtual prop
`knora-api:standoffTagHasStartAncestor` is equivalent to `knora-base:standoffTagHasStartParent*`, but
with GraphDB it is implemented using a custom inference rule (in `KnoraRules.pie`) and is therefore more
efficient. If Knora is not using the triplestore's inference,

`SparqlTransformer.transformStatementInWhereForNoInference` replaces `knora-api:standoffTagHasStartAncestor`
with `knora-base:standoffTagHasStartParent*`.
15 knora-ontologies/knora-base.ttl
View file
@@ -33,7 +33,7 @@

:attachedToProject knora-admin:SystemProject ;

:ontologyVersion "knora-base v7" .
:ontologyVersion "knora-base v8" .



@@ -2558,19 +2558,6 @@
rdfs:comment "A resource containing a text file"@en .


### http://www.knora.org/ontology/knora-base#ForbiddenResource

:ForbiddenResource rdf:type owl:Class ;

rdfs:subClassOf :Resource ,
[ rdf:type owl:Restriction ;
owl:onProperty :hasComment ;
owl:minCardinality "0"^^xsd:nonNegativeInteger
];

rdfs:label """A ForbiddenResource is a proxy for a resource that the client has insufficient permissions to see."""@en ;

rdfs:comment """A ForbiddenResource is a proxy for a resource that the client has insufficient permissions to see."""@en .

### http://www.knora.org/ontology/knora-base#XSLTransformation

3 upgrade/src/main/scala/org.knora.upgrade/Main.scala
View file
@@ -51,7 +51,8 @@ object Main extends App {
PluginForKnoraBaseVersion(versionNumber = 4, plugin = new UpgradePluginPR1372, prBasedVersionString = Some("PR 1372")),
PluginForKnoraBaseVersion(versionNumber = 5, plugin = new NoopPlugin, prBasedVersionString = Some("PR 1440")),
PluginForKnoraBaseVersion(versionNumber = 6, plugin = new NoopPlugin), // PR 1206
PluginForKnoraBaseVersion(versionNumber = 7, plugin = new NoopPlugin) // PR 1403
PluginForKnoraBaseVersion(versionNumber = 7, plugin = new NoopPlugin), // PR 1403
PluginForKnoraBaseVersion(versionNumber = 8, plugin = new NoopPlugin) // PR 1615
)

/**
111 webapi/_test_data/all_data/anything-data.ttl
View file
@@ -1618,6 +1618,117 @@
knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:UnknownUser";
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> .

<http://rdfh.ch/0001/0JhgKcqoRIeRRG6ownArSw> a anything:Thing;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q>;
knora-base:attachedToProject <http://rdfh.ch/projects/0001>;
knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember";
knora-base:creationDate "2020-04-07T09:12:56.710717Z"^^xsd:dateTime;
anything:hasOtherThingValue <http://rdfh.ch/0001/0JhgKcqoRIeRRG6ownArSw/values/UgSp5mXTTSKdI02ZU1KIAA>;
anything:hasOtherThing <http://rdfh.ch/0001/XTxSMt0ySraVmwXD-bD2wQ>;
anything:hasInteger <http://rdfh.ch/0001/0JhgKcqoRIeRRG6ownArSw/values/U1PwfNaVRQebbOSFWNdMqQ>;
rdfs:label "thing with one hidden thing";
knora-base:isDeleted false .

<http://rdfh.ch/0001/0JhgKcqoRIeRRG6ownArSw/values/UgSp5mXTTSKdI02ZU1KIAA> a knora-base:LinkValue;
knora-base:valueHasUUID "UgSp5mXTTSKdI02ZU1KIAA"^^xsd:string;
rdf:subject <http://rdfh.ch/0001/0JhgKcqoRIeRRG6ownArSw>;
rdf:predicate anything:hasOtherThing;
rdf:object <http://rdfh.ch/0001/XTxSMt0ySraVmwXD-bD2wQ>;
knora-base:isDeleted false;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q>;
knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember";
knora-base:valueCreationDate "2020-04-07T09:12:56.710717Z"^^xsd:dateTime;
knora-base:valueHasString "http://rdfh.ch/0001/XTxSMt0ySraVmwXD-bD2wQ";
knora-base:valueHasComment "link value pointing to hidden resource";
knora-base:valueHasRefCount "1"^^xsd:int .

<http://rdfh.ch/0001/0JhgKcqoRIeRRG6ownArSw/values/U1PwfNaVRQebbOSFWNdMqQ> a knora-base:IntValue;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q>;
knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember";
knora-base:valueHasUUID "U1PwfNaVRQebbOSFWNdMqQ"^^xsd:string;
knora-base:isDeleted false;
knora-base:valueCreationDate "2020-04-07T09:12:56.710717Z"^^xsd:dateTime;
knora-base:valueHasInteger 123454321;
knora-base:valueHasOrder 0;
knora-base:valueHasComment "visible int value in main resource";
knora-base:valueHasString "123454321" .

<http://rdfh.ch/0001/XTxSMt0ySraVmwXD-bD2wQ> a anything:Thing;
knora-base:attachedToUser <http://rdfh.ch/users/BhkfBc3hTeS_IDo-JgXRbQ>;
knora-base:attachedToProject <http://rdfh.ch/projects/0001>;
knora-base:hasPermissions "V knora-admin:Creator";
knora-base:creationDate "2020-04-07T09:12:56.710717Z"^^xsd:dateTime;
anything:hasInteger <http://rdfh.ch/0001/XTxSMt0ySraVmwXD-bD2wQ/values/PVPAa37xR--K_wxQwlvSsg>;
rdfs:label "hidden thing";
knora-base:isDeleted false .

<http://rdfh.ch/0001/XTxSMt0ySraVmwXD-bD2wQ/values/PVPAa37xR--K_wxQwlvSsg> a knora-base:IntValue;
knora-base:attachedToUser <http://rdfh.ch/users/BhkfBc3hTeS_IDo-JgXRbQ>;
knora-base:hasPermissions "V knora-admin:Creator";
knora-base:valueHasUUID "PVPAa37xR--K_wxQwlvSsg"^^xsd:string;
knora-base:isDeleted false;
knora-base:valueCreationDate "2020-04-07T09:12:56.710717Z"^^xsd:dateTime;
knora-base:valueHasInteger 123454321;
knora-base:valueHasOrder 0;
knora-base:valueHasComment "hidden int value in hidden resource";
knora-base:valueHasString "123454321" .

<http://rdfh.ch/0001/l8f8FVEiSCeq9A1p8gBR-A> a anything:Thing;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q>;
knora-base:attachedToProject <http://rdfh.ch/projects/0001>;
knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember";
knora-base:creationDate "2020-04-07T09:12:56.710717Z"^^xsd:dateTime;
anything:hasOtherThingValue <http://rdfh.ch/0001/l8f8FVEiSCeq9A1p8gBR-A/values/Nlcc7XWXQtmEITsIRQ5z4w>;
anything:hasOtherThing <http://rdfh.ch/0001/PHbbrEsVR32q5D_ioKt6pA>;
anything:hasInteger <http://rdfh.ch/0001/l8f8FVEiSCeq9A1p8gBR-A/values/a-40v6WiT4GHa79Kqwojjw>;
rdfs:label "thing with one deleted thing";
knora-base:isDeleted false .

<http://rdfh.ch/0001/l8f8FVEiSCeq9A1p8gBR-A/values/Nlcc7XWXQtmEITsIRQ5z4w> a knora-base:LinkValue;
knora-base:valueHasUUID "Nlcc7XWXQtmEITsIRQ5z4w"^^xsd:string;
rdf:subject <http://rdfh.ch/0001/l8f8FVEiSCeq9A1p8gBR-A>;
rdf:predicate anything:hasOtherThing;
rdf:object <http://rdfh.ch/0001/PHbbrEsVR32q5D_ioKt6pA>;
knora-base:isDeleted false;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q>;
knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember";
knora-base:valueCreationDate "2020-04-07T09:12:56.710717Z"^^xsd:dateTime;
knora-base:valueHasString "http://rdfh.ch/0001/XTxSMt0ySraVmwXD-bD2wQ";
knora-base:valueHasComment "link value pointing to deleted resource";
knora-base:valueHasRefCount "1"^^xsd:int .

<http://rdfh.ch/0001/l8f8FVEiSCeq9A1p8gBR-A/values/a-40v6WiT4GHa79Kqwojjw> a knora-base:IntValue;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q>;
knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember";
knora-base:valueHasUUID "a-40v6WiT4GHa79Kqwojjw"^^xsd:string;
knora-base:isDeleted false;
knora-base:valueCreationDate "2020-04-07T09:12:56.710717Z"^^xsd:dateTime;
knora-base:valueHasInteger 123454321;
knora-base:valueHasOrder 0;
knora-base:valueHasComment "int value in main resource";
knora-base:valueHasString "123454321" .

<http://rdfh.ch/0001/PHbbrEsVR32q5D_ioKt6pA> a anything:Thing;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q>;
knora-base:attachedToProject <http://rdfh.ch/projects/0001>;
knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember";
knora-base:creationDate "2020-04-07T09:12:56.710717Z"^^xsd:dateTime;
anything:hasInteger <http://rdfh.ch/0001/XTxSMt0ySraVmwXD-bD2wQ/values/PVPAa37xR--K_wxQwlvSsg>;
rdfs:label "deleted thing";
knora-base:isDeleted true;
knora-base:deleteDate "2020-04-07T14:59:28.960124Z"^^xsd:dateTime .

<http://rdfh.ch/0001/PHbbrEsVR32q5D_ioKt6pA/values/7BIm9QAiQqKixcgXDWf12Q> a knora-base:IntValue;
knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q>;
knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember";
knora-base:valueHasUUID "7BIm9QAiQqKixcgXDWf12Q"^^xsd:string;
knora-base:isDeleted false;
knora-base:valueCreationDate "2020-04-07T09:12:56.710717Z"^^xsd:dateTime;
knora-base:valueHasInteger 123454321;
knora-base:valueHasOrder 0;
knora-base:valueHasComment "int value in deleted resource";
knora-base:valueHasString "123454321" .

<http://rdfh.ch/lists/0001/otherTreeList> a knora-base:ListNode;
knora-base:isRootNode true;
rdfs:label "Tree list root"@en;
23 webapi/_test_data/all_data/system-data.ttl
View file
@@ -39,26 +39,3 @@
knora-base:listNodePosition 2 ;
rdfs:label "Maybe"@en ;
rdfs:label "Vielleicht"@de .


##########################################################
#
# Sweet and tasty but forbidden resource
#
# but of the tree of the knowledge of good and evil, thou shalt not eat of it
#
##########################################################

<http://rdfh.ch/0000/forbiddenResource> a knora-base:ForbiddenResource ;

rdfs:label "This resource is a proxy for a resource you are not allowed to see (may depend on the context: query path)" ;

knora-base:isDeleted false ;

knora-base:attachedToUser <http://rdfh.ch/users/root> ;

knora-base:attachedToProject knora-admin:SystemProject ;

knora-base:creationDate "2017-10-06T11:05:37Z"^^xsd:dateTime ;

knora-base:hasPermissions "CR knora-admin:Creator|M knora-admin:ProjectMember|V knora-admin:KnownUser|V knora-admin:UnknownUser" .

0 comments on commit 992596e

Please sign in to comment.