feat(gravsearch): Remove deprecated functions (#1660)

* feat(gravsearch): Remove deprecated functions.

* test(gravsearch): Fix tests.

Co-authored-by: Ivan Subotic <400790+subotic@users.noreply.github.com>

Author: Benjamin Geer

docs/src/paradox/03-apis/api-v2/query-language.md

@@ -382,10 +382,6 @@ For example, to search for titles that contain the words 'Zeitglöcklein' and
 FILTER knora-api:matchText(?title, "Zeitglöcklein Lebens")
 ```
 
-Note: the `knora-api:match` function has been deprecated, and will no longer work in
-a future release of Knora. Please change your Gravsearch queries to use `knora-api:matchText`
-instead. Attention: the first argument is different.
-
 #### Filtering Text by Language
 
 To filter a text value by language in the simple schema, use the SPARQL `lang` function
@@ -465,10 +461,6 @@ CONSTRUCT {
 Here we are looking for letters containing the words "Grund" and "Richtigkeit"
 within a single paragraph.
 
-Note: the `knora-api:matchInStandoff` function has been deprecated, and will no longer
-work in a future release of Knora. Please change your Gravsearch queries to use
-`knora-api:matchTextInStandoff` instead. Attention: the first argument is different.
-
 #### Matching Standoff Links
 
 If you are only interested in specifying that a resource has some text

webapi/src/main/scala/org/knora/webapi/OntologyConstants.scala

@@ -334,10 +334,6 @@ object OntologyConstants {
 
         val IsMainResource: IRI = KnoraBasePrefixExpansion + "isMainResource"
 
-        // A deprecated virtual property, used by the deprecated Gravsearch functions knora-api:match and
-        // knora-api:matchInStandoff. Replaced during processing by a triplestore-specific property.
-        val MatchesTextIndex: IRI = KnoraBasePrefixExpansion + "matchesTextIndex"
-
         /* Resource creator */
         val AttachedToUser: IRI = KnoraBasePrefixExpansion + "attachedToUser"
 
@@ -925,9 +921,7 @@ object OntologyConstants {
 
         val IsMainResource: IRI = KnoraApiV2PrefixExpansion + "isMainResource"
         val ToSimpleDateFunction: IRI = KnoraApiV2PrefixExpansion + "toSimpleDate"
-        val MatchFunction: IRI = KnoraApiV2PrefixExpansion + "match"
         val MatchTextFunction: IRI = KnoraApiV2PrefixExpansion + "matchText"
-        val MatchInStandoffFunction: IRI = KnoraApiV2PrefixExpansion + "matchInStandoff"
         val MatchTextInStandoffFunction: IRI = KnoraApiV2PrefixExpansion + "matchTextInStandoff"
         val MatchLabelFunction: IRI = KnoraApiV2PrefixExpansion + "matchLabel"
         val StandoffLinkFunction: IRI = KnoraApiV2PrefixExpansion + "standoffLink"
@@ -966,8 +960,6 @@ object OntologyConstants {
         val ObjectType: IRI = KnoraApiV2PrefixExpansion + "objectType"
 
         val IsMainResource: IRI = KnoraApiV2PrefixExpansion + "isMainResource"
-        val MatchesTextIndex: IRI = KnoraApiV2PrefixExpansion + "matchesTextIndex" // virtual property to be replaced by a triplestore-specific one
-        val MatchFunction: IRI = KnoraApiV2PrefixExpansion + "match"
         val MatchTextFunction: IRI = KnoraApiV2PrefixExpansion + "matchText"
         val MatchLabelFunction: IRI = KnoraApiV2PrefixExpansion + "matchLabel"
 

webapi/src/main/scala/org/knora/webapi/responders/v2/search/SparqlTransformer.scala

@@ -300,10 +300,7 @@ object SparqlTransformer {
 
         val transformedPattern = statement.copy(
             // Replace the deprecated property knora-base:matchesTextIndex with a GraphDB-specific one.
-            pred = statement.pred match {
-                case iri: IriRef if iri.iri == OntologyConstants.KnoraBase.MatchesTextIndex.toSmartIri => IriRef(OntologyConstants.Ontotext.LuceneFulltext.toSmartIri) // convert to special Lucene property
-                case other => other // no conversion needed
-            },
+            pred = statement.pred,
             namedGraph = statement.namedGraph match {
                 case Some(IriRef(SmartIri(OntologyConstants.NamedGraphs.KnoraExplicitNamedGraph), _)) => Some(IriRef(OntologyConstants.NamedGraphs.GraphDBExplicitNamedGraph.toSmartIri))
                 case Some(IriRef(_, _)) => throw AssertionException(s"Named graphs other than ${OntologyConstants.NamedGraphs.KnoraExplicitNamedGraph} cannot occur in non-triplestore-specific generated search query SPARQL")

webapi/src/main/scala/org/knora/webapi/responders/v2/search/gravsearch/prequery/AbstractPrequeryGenerator.scala

@@ -1119,74 +1119,6 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause,
         }
     }
 
-    /**
-     * Handles the deprecated function `knora-api:match` in the simple schema.
-     *
-     * @param functionCallExpression the function call to be handled.
-     * @param typeInspectionResult   the type inspection results.
-     * @param isTopLevel             if `true`, this is the top-level expression in the `FILTER`.
-     * @return a [[TransformedFilterPattern]].
-     */
-    private def handleMatchFunctionInSimpleSchema(functionCallExpression: FunctionCallExpression, typeInspectionResult: GravsearchTypeInspectionResult, isTopLevel: Boolean): TransformedFilterPattern = {
-        val functionIri: SmartIri = functionCallExpression.functionIri.iri
-
-        if (querySchema == ApiV2Complex) {
-            throw GravsearchException(s"Function ${functionIri.toSparql} cannot be used in a Gravsearch query written in the complex schema; use ${OntologyConstants.KnoraApiV2Complex.MatchFunction.toSmartIri.toSparql} instead")
-        }
-
-        // The match function must be the top-level expression, otherwise boolean logic won't work properly.
-        if (!isTopLevel) {
-            throw GravsearchException(s"Function ${functionIri.toSparql} must be the top-level expression in a FILTER")
-        }
-
-        // two arguments are expected: the first must be a variable representing a string value,
-        // the second must be a string literal
-
-        if (functionCallExpression.args.size != 2) throw GravsearchException(s"Two arguments are expected for ${functionIri.toSparql}")
-
-        // a QueryVariable expected to represent a text value
-        val textValueVar: QueryVariable = functionCallExpression.getArgAsQueryVar(pos = 0)
-
-        typeInspectionResult.getTypeOfEntity(textValueVar) match {
-            case Some(nonPropInfo: NonPropertyTypeInfo) =>
-
-                nonPropInfo.typeIri.toString match {
-
-                    case OntologyConstants.Xsd.String => () // xsd:string is expected, TODO: should also xsd:anyUri be allowed?
-
-                    case _ => throw GravsearchException(s"${textValueVar.toSparql} must be an xsd:string")
-                }
-
-            case _ => throw GravsearchException(s"${textValueVar.toSparql} must be an xsd:string")
-        }
-
-        val textValHasString: QueryVariable = SparqlTransformer.createUniqueVariableNameFromEntityAndProperty(base = textValueVar, propertyIri = OntologyConstants.KnoraBase.ValueHasString)
-
-        // Add a statement to assign the literal to a variable, which we'll use in the transformed FILTER expression,
-        // if that statement hasn't been added already.
-        val valueHasStringStatement = if (addGeneratedVariableForValueLiteral(textValueVar, textValHasString)) {
-            Seq(StatementPattern.makeExplicit(subj = textValueVar, pred = IriRef(OntologyConstants.KnoraBase.ValueHasString.toSmartIri), textValHasString))
-        } else {
-            Seq.empty[StatementPattern]
-        }
-
-        val searchTerm: XsdLiteral = functionCallExpression.getArgAsLiteral(1, xsdDatatype = OntologyConstants.Xsd.String.toSmartIri)
-
-        val searchTerms: LuceneQueryString = LuceneQueryString(searchTerm.value)
-
-        // Replace the filter with a statement using the deprecated knora-base:matchesTextIndex property.
-        TransformedFilterPattern(
-            None, // FILTER has been replaced by statements
-            valueHasStringStatement ++ Seq(
-                StatementPattern.makeExplicit(
-                    subj = textValHasString,
-                    pred = IriRef(OntologyConstants.KnoraBase.MatchesTextIndex.toSmartIri),
-                    obj = XsdLiteral(searchTerms.getQueryString, OntologyConstants.Xsd.String.toSmartIri)
-                )
-            )
-        )
-    }
-
     /**
      * Handles the function `knora-api:matchText` in the simple schema.
      *
@@ -1253,51 +1185,6 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause,
         )
     }
 
-    /**
-     * Handles the deprecated function `knora-api:match` in the complex schema.
-     *
-     * @param functionCallExpression the function call to be handled.
-     * @param typeInspectionResult   the type inspection results.
-     * @param isTopLevel             if `true`, this is the top-level expression in the `FILTER`.
-     * @return a [[TransformedFilterPattern]].
-     */
-    private def handleMatchFunctionInComplexSchema(functionCallExpression: FunctionCallExpression, typeInspectionResult: GravsearchTypeInspectionResult, isTopLevel: Boolean): TransformedFilterPattern = {
-        val functionIri: SmartIri = functionCallExpression.functionIri.iri
-
-        if (querySchema == ApiV2Simple) {
-            throw GravsearchException(s"Function ${functionIri.toSparql} cannot be used in a Gravsearch query written in the simple schema; use ${OntologyConstants.KnoraApiV2Simple.MatchFunction.toSmartIri.toSparql} instead")
-        }
-
-        // The match function must be the top-level expression, otherwise boolean logic won't work properly.
-        if (!isTopLevel) {
-            throw GravsearchException(s"Function ${functionIri.toSparql} must be the top-level expression in a FILTER")
-        }
-
-        // two arguments are expected: the first must be a variable representing a string value,
-        // the second must be a string literal
-
-        if (functionCallExpression.args.size != 2) throw GravsearchException(s"Two arguments are expected for ${functionIri.toSparql}")
-
-        // a QueryVariable expected to represent a string
-        val stringVar: QueryVariable = functionCallExpression.getArgAsQueryVar(pos = 0)
-
-        val searchTermStr: XsdLiteral = functionCallExpression.getArgAsLiteral(1, xsdDatatype = OntologyConstants.Xsd.String.toSmartIri)
-
-        val searchTerms: LuceneQueryString = LuceneQueryString(searchTermStr.value)
-
-        // Replace the filter with a statement using the deprecated knora-base:matchesTextIndex property.
-        TransformedFilterPattern(
-            None, // FILTER has been replaced by statements
-            Seq(
-                StatementPattern.makeExplicit(
-                    subj = stringVar,
-                    pred = IriRef(OntologyConstants.KnoraBase.MatchesTextIndex.toSmartIri),
-                    XsdLiteral(searchTerms.getQueryString, OntologyConstants.Xsd.String.toSmartIri)
-                )
-            )
-        )
-    }
-
     /**
      * Handles the function `knora-api:matchText` in the complex schema.
      *
@@ -1364,105 +1251,6 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause,
         )
     }
 
-    /**
-     * Handles the deprecated function `knora-api:matchInStandoff`.
-     *
-     * @param functionCallExpression the function call to be handled.
-     * @param typeInspectionResult   the type inspection results.
-     * @param isTopLevel             if `true`, this is the top-level expression in the `FILTER`.
-     * @return a [[TransformedFilterPattern]].
-     */
-    private def handleMatchInStandoffFunction(functionCallExpression: FunctionCallExpression, typeInspectionResult: GravsearchTypeInspectionResult, isTopLevel: Boolean): TransformedFilterPattern = {
-        val functionIri: SmartIri = functionCallExpression.functionIri.iri
-
-        if (querySchema == ApiV2Simple) {
-            throw GravsearchException(s"Function ${functionIri.toSparql} cannot be used in a Gravsearch query written in the simple schema")
-        }
-
-        if (!isTopLevel) {
-            throw GravsearchException(s"Function ${functionIri.toSparql} must be the top-level expression in a FILTER")
-        }
-
-        // Three arguments are expected:
-        // 1. a variable representing the string literal value of the text value
-        // 2. a variable representing the standoff tag
-        // 3. a string literal containing space-separated search terms
-
-        if (functionCallExpression.args.size != 3) throw GravsearchException(s"Three arguments are expected for ${functionIri.toSparql}")
-
-        // A variable representing the object of the text value's valueHasString.
-        val textValueStringLiteralVar: QueryVariable = functionCallExpression.getArgAsQueryVar(pos = 0)
-
-        // A variable representing the standoff tag.
-        val standoffTagVar: QueryVariable = functionCallExpression.getArgAsQueryVar(pos = 1)
-
-        // A string literal representing the search terms.
-        val searchTermStr: XsdLiteral = functionCallExpression.getArgAsLiteral(pos = 2, xsdDatatype = OntologyConstants.Xsd.String.toSmartIri)
-
-        val searchTerms: LuceneQueryString = LuceneQueryString(searchTermStr.value)
-
-        // Generate a statement to search the full-text search index, using the deprecated virtual predicate
-        // knora-base:matchesTextIndex, to assert that text value contains the search terms.
-        val fullTextSearchStatement: Seq[StatementPattern] = Seq(
-            StatementPattern.makeInferred(
-                subj = textValueStringLiteralVar,
-                pred = IriRef(OntologyConstants.KnoraBase.MatchesTextIndex.toSmartIri),
-                XsdLiteral(searchTerms.getQueryString, OntologyConstants.Xsd.String.toSmartIri)
-            )
-        )
-
-        // Generate query patterns to assign the text in the standoff tag to a variable, if we
-        // haven't done so already.
-
-        val startVariable = QueryVariable(standoffTagVar.variableName + "__start")
-        val endVariable = QueryVariable(standoffTagVar.variableName + "__end")
-
-        val markedUpPatternsToAdd: Seq[QueryPattern] = if (!standoffMarkedUpVariables.contains(startVariable)) {
-            standoffMarkedUpVariables += startVariable
-
-            Seq(
-                // ?standoffTag knora-base:standoffTagHasStart ?standoffTag__start .
-                StatementPattern.makeExplicit(standoffTagVar, IriRef(OntologyConstants.KnoraBase.StandoffTagHasStart.toSmartIri), startVariable),
-                // ?standoffTag knora-base:standoffTagHasEnd ?standoffTag__end .
-                StatementPattern.makeExplicit(standoffTagVar, IriRef(OntologyConstants.KnoraBase.StandoffTagHasEnd.toSmartIri), endVariable)
-            )
-        } else {
-            Seq.empty[QueryPattern]
-        }
-
-        // Generate a FILTER pattern for each search term, using the regex function to assert that the text in the
-        // standoff tag contains the term:
-        // FILTER REGEX(SUBSTR(?textValueStr, ?standoffTag__start + 1, ?standoffTag__end - ?standoffTag__start), 'term', "i")
-        // TODO: handle the differences between regex syntax and Lucene syntax.
-        val regexFilters: Seq[FilterPattern] = searchTerms.getSingleTerms.map {
-            term: String =>
-                FilterPattern(
-                    expression = RegexFunction(
-                        textExpr = SubStrFunction(
-                            textLiteralVar = textValueStringLiteralVar,
-                            startExpression = ArithmeticExpression(
-                                leftArg = startVariable,
-                                operator = PlusOperator,
-                                rightArg = IntegerLiteral(1)
-                            ),
-                            lengthExpression = ArithmeticExpression(
-                                leftArg = endVariable,
-                                operator = MinusOperator,
-                                rightArg = startVariable
-                            )
-                        ),
-                        pattern = term, // TODO: Ignore Lucene operators
-                        modifier = Some("i")
-                    )
-                )
-        }
-
-        TransformedFilterPattern(
-            expression = None, // The expression has been replaced by additional patterns.
-            additionalPatterns = fullTextSearchStatement ++ markedUpPatternsToAdd ++ regexFilters
-        )
-    }
-
     /**
      * Handles the function `knora-api:matchTextInStandoff`.
      *
@@ -1779,9 +1567,6 @@ abstract class AbstractPrequeryGenerator(constructClause: ConstructClause,
 
         // Get a Scala function that implements the Gravsearch function.
         val functionFunction: (FunctionCallExpression, GravsearchTypeInspectionResult, Boolean) => TransformedFilterPattern = functionIri.toString match {
-            case OntologyConstants.KnoraApiV2Simple.MatchFunction => handleMatchFunctionInSimpleSchema // deprecated
-            case OntologyConstants.KnoraApiV2Complex.MatchFunction => handleMatchFunctionInComplexSchema // deprecated
-            case OntologyConstants.KnoraApiV2Complex.MatchInStandoffFunction => handleMatchInStandoffFunction // deprecated
             case OntologyConstants.KnoraApiV2Simple.MatchTextFunction => handleMatchTextFunctionInSimpleSchema
             case OntologyConstants.KnoraApiV2Complex.MatchTextFunction => handleMatchTextFunctionInComplexSchema
             case OntologyConstants.KnoraApiV2Simple.MatchLabelFunction => handleMatchLabelFunctionInSimpleSchema

webapi/src/main/scala/org/knora/webapi/responders/v2/search/gravsearch/types/InferringGravsearchTypeInspector.scala

@@ -982,9 +982,7 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe
                     // the function.
 
                     functionCallExpression.functionIri.iri.toString match {
-                        case OntologyConstants.KnoraApiV2Simple.MatchFunction |
-                             OntologyConstants.KnoraApiV2Simple.MatchTextFunction |
-                             OntologyConstants.KnoraApiV2Complex.MatchFunction =>
+                        case OntologyConstants.KnoraApiV2Simple.MatchTextFunction =>
                             // The first argument is a variable representing a string.
                             val textVar = TypeableVariable(functionCallExpression.getArgAsQueryVar(0).variableName)
                             val currentTextVarTypesFromFilters: Set[SmartIri] = acc.typedEntitiesInFilters.getOrElse(textVar, Set.empty[SmartIri])
@@ -1024,21 +1022,6 @@ class InferringGravsearchTypeInspector(nextInspector: Option[GravsearchTypeInspe
                                     (resourceVar -> (currentResourceVarTypesFromFilters + OntologyConstants.KnoraApiV2Complex.Resource.toSmartIri))
                             )
 
-                        case OntologyConstants.KnoraApiV2Complex.MatchInStandoffFunction =>
-                            // The first argument is a variable representing a string.
-                            val textVar = TypeableVariable(functionCallExpression.getArgAsQueryVar(0).variableName)
-                            val currentTextVarTypesFromFilters: Set[SmartIri] = acc.typedEntitiesInFilters.getOrElse(textVar, Set.empty[SmartIri])
-
-                            // The second argument is a variable representing a standoff tag.
-                            val standoffTagVar = TypeableVariable(functionCallExpression.getArgAsQueryVar(1).variableName)
-                            val currentStandoffVarTypesFromFilters: Set[SmartIri] = acc.typedEntitiesInFilters.getOrElse(standoffTagVar, Set.empty[SmartIri])
-
-                            acc.copy(
-                                typedEntitiesInFilters = acc.typedEntitiesInFilters +
-                                    (textVar -> (currentTextVarTypesFromFilters + OntologyConstants.Xsd.String.toSmartIri)) +
-                                    (standoffTagVar -> (currentStandoffVarTypesFromFilters + OntologyConstants.KnoraApiV2Complex.StandoffTag.toSmartIri))
-                            )
-
                         case OntologyConstants.KnoraApiV2Complex.MatchTextInStandoffFunction =>
                             // The first argument is a variable representing a text value.
                             val textVar = TypeableVariable(functionCallExpression.getArgAsQueryVar(0).variableName)

webapi/src/test/scala/org/knora/webapi/responders/v2/search/gravsearch/types/GravsearchTypeInspectorSpec.scala

@@ -302,7 +302,7 @@ class GravsearchTypeInspectorSpec extends CoreSpec() with ImplicitSender {
           |
           |        ?mainRes ?titleProp ?propVal0 .
           |
-          |        FILTER knora-api:match(?propVal0, "Zeitglöcklein")
+          |        FILTER knora-api:matchText(?propVal0, "Zeitglöcklein")
           |
           |     }
         """.stripMargin