Skip to content
Permalink
Browse files

feat(api-v2): Make inference optional in Gravsearch (#1696)

  • Loading branch information
benjamingeer committed Sep 1, 2020
1 parent b66292d commit 166a26061dd434ca1d95fedf6dec75728760c78c
Showing with 452 additions and 256 deletions.
  1. +10 −0 docs/03-apis/api-v2/query-language.md
  2. +3 −5 docs/05-internals/design/api-v2/gravsearch.md
  3. +1 −1 webapi/scripts/fuseki-dump-repository.sh
  4. +1 −1 webapi/scripts/fuseki-upload-repository.sh
  5. +27 −6 webapi/src/main/scala/org/knora/webapi/messages/OntologyConstants.scala
  6. +9 −1 webapi/src/main/scala/org/knora/webapi/messages/util/search/QueryTraverser.scala
  7. +20 −4 webapi/src/main/scala/org/knora/webapi/messages/util/search/SparqlQuery.scala
  8. +109 −84 webapi/src/main/scala/org/knora/webapi/messages/util/search/SparqlTransformer.scala
  9. +55 −25 ...n/scala/org/knora/webapi/messages/util/search/gravsearch/prequery/AbstractPrequeryGenerator.scala
  10. +1 −3 .../util/search/gravsearch/prequery/NonTriplestoreSpecificGravsearchToCountPrequeryTransformer.scala
  11. +2 −3 ...knora/webapi/messages/util/search/gravsearch/types/AnnotationReadingGravsearchTypeInspector.scala
  12. +13 −1 ...n/scala/org/knora/webapi/messages/util/search/gravsearch/types/GravsearchTypeInspectionUtil.scala
  13. +5 −2 ...ala/org/knora/webapi/messages/util/search/gravsearch/types/InferringGravsearchTypeInspector.scala
  14. +26 −14 webapi/src/main/scala/org/knora/webapi/responders/v2/SearchResponderV2.scala
  15. +1 −1 webapi/src/main/scala/org/knora/webapi/routing/v2/OntologiesRouteV2.scala
  16. +23 −0 webapi/src/test/scala/org/knora/webapi/e2e/v2/SearchRouteV2R2RSpec.scala
  17. +146 −105 ...st/scala/org/knora/webapi/messages/util/search/gravsearch/types/GravsearchTypeInspectorSpec.scala
@@ -212,6 +212,16 @@ also match subproperties of that property, and if a statement specifies that
a subject has a particular `rdf:type`, the statement will also match subjects
belonging to subclasses of that type.

If you know that reasoning will not return any additional results for
your query, you can disable it by adding this line to the `WHERE` clause:

```sparql
knora-api:GravsearchOptions knora-api:useInference false .
```

If Knora is implementing reasoning by query expansion, disabling it can
improve the performance of some queries.

## Gravsearch Syntax

Every Gravsearch query is a valid SPARQL 1.1
@@ -315,17 +315,15 @@ When the triplestore-specific version of the query is generated:
for text searches.

- If Knora is not using the triplestore's inference (e.g. with Fuseki),
`SparqlTransformer.expandStatementForNoInference` removes `<http://www.knora.org/explicit>`, and expands unmarked
`SparqlTransformer.transformStatementInWhereForNoInference` removes `<http://www.knora.org/explicit>`, and expands unmarked
statements using `rdfs:subClassOf*` and `rdfs:subPropertyOf*`.

Gravsearch also provides some virtual properties, which take advantage of forward-chaining inference
as an optimisation if the triplestore provides it. For example, the virtual property
`knora-api:standoffTagHasStartAncestor` is equivalent to `knora-base:standoffTagHasStartParent*`, but
with GraphDB it is implemented using a custom inference rule (in `KnoraRules.pie`) and is therefore more
efficient. If Knora is not using the triplestore's inference,

`SparqlTransformer.transformStatementInWhereForNoInference` replaces `knora-api:standoffTagHasStartAncestor`
with `knora-base:standoffTagHasStartParent*`.
efficient. If Knora is not using the triplestore's inference, `SparqlTransformer.transformStatementInWhereForNoInference`
replaces `knora-api:standoffTagHasStartAncestor` with `knora-base:standoffTagHasStartParent*`.

# Optimisation of generated SPARQL

@@ -50,7 +50,7 @@ if [[ -z "${PASSWORD}" ]]; then
fi

if [[ -z "${HOST}" ]]; then
HOST="localhost:8080"
HOST="localhost:3030"
fi

curl -sS -X GET -H "Accept: application/trig" -u "${USERNAME}:${PASSWORD}" "http://${HOST}/${REPOSITORY}" > "${FILE}"
@@ -50,7 +50,7 @@ if [[ -z "${PASSWORD}" ]]; then
fi

if [[ -z "${HOST}" ]]; then
HOST="localhost:8080"
HOST="localhost:3030"
fi

curl -sS -X POST -H "Content-Type: application/trig" --data-binary "@${FILE}" -u "${USERNAME}:${PASSWORD}" "http://${HOST}/${REPOSITORY}" | tee /dev/null
@@ -655,13 +655,28 @@ object OntologyConstants {
val KnoraApiPrefix: String = KnoraApiOntologyLabel + ":"

/**
* Returns `true` if the specified IRI is `knora-api:Resource` in Knora API v2, in the simple
* or complex schema.
* The IRIs representing `knora-api:Resource` in Knora API v2, in the simple and complex schemas.
*/
def isKnoraApiV2Resource(iri: SmartIri): Boolean = {
val iriStr = iri.toString
iriStr == OntologyConstants.KnoraApiV2Simple.Resource || iriStr == OntologyConstants.KnoraApiV2Complex.Resource
}
lazy val KnoraApiV2ResourceIris: Set[IRI] = Set(
OntologyConstants.KnoraApiV2Simple.Resource,
OntologyConstants.KnoraApiV2Complex.Resource
)

/**
* The IRIs representing `knora-api:GravsearchOptions` in Knora API v2, in the simple and complex schemas.
*/
lazy val GravsearchOptionsIris: Set[IRI] = Set(
OntologyConstants.KnoraApiV2Simple.GravsearchOptions,
OntologyConstants.KnoraApiV2Complex.GravsearchOptions
)

/**
* The IRIs representing `knora-api:useInference` in Knora API v2, in the simple and complex schemas.
*/
lazy val UseInferenceIris: Set[IRI] = Set(
OntologyConstants.KnoraApiV2Simple.UseInference,
OntologyConstants.KnoraApiV2Complex.UseInference
)

/**
* Returns the IRI of `knora-api:subjectType` in the specified schema.
@@ -925,6 +940,9 @@ object OntologyConstants {
val MatchTextInStandoffFunction: IRI = KnoraApiV2PrefixExpansion + "matchTextInStandoff"
val MatchLabelFunction: IRI = KnoraApiV2PrefixExpansion + "matchLabel"
val StandoffLinkFunction: IRI = KnoraApiV2PrefixExpansion + "standoffLink"

val GravsearchOptions: IRI = KnoraApiV2PrefixExpansion + "GravsearchOptions"
val UseInference: IRI = KnoraApiV2PrefixExpansion + "useInference"
}

object SalsahGuiApiV2WithValueObjects {
@@ -1026,6 +1044,9 @@ object OntologyConstants {

val ArkUrl: IRI = KnoraApiV2PrefixExpansion + "arkUrl"
val VersionArkUrl: IRI = KnoraApiV2PrefixExpansion + "versionArkUrl"

val GravsearchOptions: IRI = KnoraApiV2PrefixExpansion + "GravsearchOptions"
val UseInference: IRI = KnoraApiV2PrefixExpansion + "useInference"
}

/**
@@ -95,6 +95,13 @@ trait SelectToSelectTransformer extends WhereTransformer {
* @return the result of the transformation.
*/
def transformStatementInSelect(statementPattern: StatementPattern): Seq[StatementPattern]

/**
* Specifies a FROM clause, if needed.
*
* @return the FROM clause to be used, if any.
*/
def getFromClause: Option[FromClause]
}

/**
@@ -189,7 +196,7 @@ object QueryTraverser {
// remove statements that would otherwise be expanded by transformStatementInWhere
val optimisedPatterns = whereTransformer.optimiseQueryPatterns(patterns)

optimisedPatterns.flatMap {
optimisedPatterns.flatMap {
case statementPattern: StatementPattern =>
whereTransformer.transformStatementInWhere(
statementPattern = statementPattern,
@@ -338,6 +345,7 @@ object QueryTraverser {

def transformSelectToSelect(inputQuery: SelectQuery, transformer: SelectToSelectTransformer): SelectQuery = {
inputQuery.copy(
fromClause = transformer.getFromClause,
whereClause = WhereClause(
patterns = transformWherePatterns(
patterns = inputQuery.whereClause.patterns,
@@ -573,19 +573,32 @@ case class OrderCriterion(queryVariable: QueryVariable, isAscending: Boolean) ex
}
}

/**
* Represents a FROM clause.
*
* @param defaultGraph the graph to be used as the default graph in the query.
*/
case class FromClause(defaultGraph: IriRef) extends SparqlGenerator {
override def toSparql: String = s"FROM ${defaultGraph.toSparql}\n"
}

/**
* Represents a SPARQL CONSTRUCT query.
*
* @param constructClause the CONSTRUCT clause.
* @param fromClause the FROM clause, if any.
* @param whereClause the WHERE clause.
* @param orderBy the variables that the results should be ordered by.
* @param offset if this is a Gravsearch query, represents the OFFSET specified in the query.
* @param querySchema if this is a Gravsearch query, represents the Knora API v2 ontology schema used in the query.
*/
case class ConstructQuery(constructClause: ConstructClause, whereClause: WhereClause, orderBy: Seq[OrderCriterion] = Seq.empty[OrderCriterion], offset: Long = 0, querySchema: Option[ApiV2Schema] = None) extends SparqlGenerator {
case class ConstructQuery(constructClause: ConstructClause, fromClause: Option[FromClause] = None, whereClause: WhereClause, orderBy: Seq[OrderCriterion] = Seq.empty[OrderCriterion], offset: Long = 0, querySchema: Option[ApiV2Schema] = None) extends SparqlGenerator {
override def toSparql: String = {
val stringBuilder = new StringBuilder
stringBuilder.append(constructClause.toSparql).append(whereClause.toSparql)

stringBuilder.append(constructClause.toSparql)
.append(fromClause.map(_.toSparql).getOrElse(""))
.append(whereClause.toSparql)

if (orderBy.nonEmpty) {
stringBuilder.append("ORDER BY ").append(orderBy.map(_.toSparql).mkString(" ")).append("\n")
@@ -604,12 +617,13 @@ case class ConstructQuery(constructClause: ConstructClause, whereClause: WhereCl
*
* @param variables the variables to be returned by the query.
* @param useDistinct indicates if DISTINCT should be used.
* @param fromClause the FROM clause, if any.
* @param whereClause the WHERE clause.
* @param orderBy the variables that the results should be ordered by.
* @param limit the maximum number of result rows to be returned.
* @param offset the offset to be used (limit of the previous query + 1 to do paging).
*/
case class SelectQuery(variables: Seq[SelectQueryColumn], useDistinct: Boolean = true, whereClause: WhereClause, groupBy: Seq[QueryVariable] = Seq.empty[QueryVariable], orderBy: Seq[OrderCriterion] = Seq.empty[OrderCriterion], limit: Option[Int] = None, offset: Long = 0) extends SparqlGenerator {
case class SelectQuery(variables: Seq[SelectQueryColumn], useDistinct: Boolean = true, fromClause: Option[FromClause] = None, whereClause: WhereClause, groupBy: Seq[QueryVariable] = Seq.empty[QueryVariable], orderBy: Seq[OrderCriterion] = Seq.empty[OrderCriterion], limit: Option[Int] = None, offset: Long = 0) extends SparqlGenerator {
override def toSparql: String = {
val stringBuilder = new StringBuilder

@@ -620,7 +634,9 @@ case class SelectQuery(variables: Seq[SelectQueryColumn], useDistinct: Boolean =

}

stringBuilder.append(variables.map(_.toSparql).mkString(" ")).append("\n").append(whereClause.toSparql)
stringBuilder.append(variables.map(_.toSparql).mkString(" ")).append("\n")
.append(fromClause.map(_.toSparql).getOrElse(""))
.append(whereClause.toSparql)

if (groupBy.nonEmpty) {
stringBuilder.append("GROUP BY ").append(groupBy.map(_.toSparql).mkString(" ")).append("\n")

0 comments on commit 166a260

Please sign in to comment.