feat: Add metrics to TriplestoreService SparqlQuery execution DEV-2627 (

#2823)

integration/src/test/scala/org/knora/webapi/store/triplestore/TriplestoreServiceLiveSpec.scala

@@ -221,18 +221,5 @@ class TriplestoreServiceLiveSpec extends CoreSpec with ImplicitSender {
         actual should ===(1)
       }
     }
-
-    "put the graph data as turtle" in {
-      UnsafeZioRun.runOrThrow(
-        TriplestoreService
-          .insertDataGraphRequest(graphContent = graphDataContent, "http://jedi.org/graph")
-          .timeout(java.time.Duration.ofSeconds(10))
-      )
-    }
-
-    "read the graph data as turtle" in {
-      val response = UnsafeZioRun.runOrThrow(TriplestoreService.sparqlHttpGraphData("http://jedi.org/graph"))
-      response.turtle.length should be > 0
-    }
   }
 }

webapi/src/main/resources/application.conf

@@ -470,6 +470,8 @@ app {
             username = ${?KNORA_WEBAPI_TRIPLESTORE_FUSEKI_USERNAME}
             password = "test"
             password = ${?KNORA_WEBAPI_TRIPLESTORE_FUSEKI_PASSWORD}
+            query-logging-threshold = 1s
+            query-logging-threshold = ${?KNORA_WEBAPI_TRIPLESTORE_FUSEKI_QUERY_LOGGING_THRESHOLD}
         }
 
         // If true, the time taken by each SPARQL query is logged at DEBUG level. To see these messages,

webapi/src/main/scala/org/knora/webapi/config/AppConfig.scala

@@ -185,7 +185,8 @@ final case class Fuseki(
   port: Int,
   repositoryName: String,
   username: String,
-  password: String
+  password: String,
+  queryLoggingThreshold: Duration = Duration.ofMillis(1000)
 )
 
 final case class CacheService(

webapi/src/main/scala/org/knora/webapi/core/AppServer.scala

@@ -43,7 +43,7 @@ final case class AppServer(
   private val checkTriplestoreService: Task[Unit] =
     for {
       _      <- state.set(AppState.WaitingForTriplestore)
-      status <- ts.checkTriplestore().map(_.triplestoreStatus)
+      status <- ts.checkTriplestore()
       _ <- status match {
              case TriplestoreStatus.Available           => ZIO.unit
              case TriplestoreStatus.NotInitialized(msg) => ZIO.die(new Exception(msg))

webapi/src/main/scala/org/knora/webapi/messages/admin/responder/listsmessages/ListsMessagesADM.scala

@@ -312,7 +312,7 @@ case class ListItemDeleteRequestADM(
 case class CanDeleteListRequestADM(iri: IRI, requestingUser: UserADM) extends ListsResponderRequestADM
 
 /**
- * Requests deletion of all list node comments. A successful response will be a [[ListNodeCommentsDeleteADM]]
+ * Requests deletion of all list node comments. A successful response will be a [[ListNodeCommentsDeleteResponseADM]]
  *
  * @param iri                  the IRI of the list node (root or child).
  * @param requestingUser       the user making the request.
@@ -358,14 +358,14 @@ case class ListsGetResponseADM(lists: Seq[ListNodeInfoADM]) extends KnoraRespons
   def toJsValue: JsValue = listsGetResponseADMFormat.write(this)
 }
 
-abstract class ListItemGetResponseADM(listItem: ListItemADM) extends KnoraResponseADM with ListADMJsonProtocol
+abstract class ListItemGetResponseADM() extends KnoraResponseADM with ListADMJsonProtocol
 
 /**
  * Provides completes information about the list. The basic information (rood node) and all the child nodes.
  *
  * @param list the complete list.
  */
-case class ListGetResponseADM(list: ListADM) extends ListItemGetResponseADM(list) {
+case class ListGetResponseADM(list: ListADM) extends ListItemGetResponseADM() {
 
   def toJsValue: JsValue = listGetResponseADMFormat.write(this)
 }
@@ -375,7 +375,7 @@ case class ListGetResponseADM(list: ListADM) extends ListItemGetResponseADM(list
  *
  * @param node the node.
  */
-case class ListNodeGetResponseADM(node: NodeADM) extends ListItemGetResponseADM(node) {
+case class ListNodeGetResponseADM(node: NodeADM) extends ListItemGetResponseADM() {
 
   def toJsValue: JsValue = listNodeGetResponseADMFormat.write(this)
 }
@@ -452,36 +452,26 @@ case class NodePositionChangeResponseADM(node: ListNodeADM) extends KnoraRespons
 
 //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 // Components of messages
-abstract class ListItemADM(info: ListNodeInfoADM, children: Seq[ListChildNodeADM])
+abstract class ListItemADM()
 
-case class ListADM(listinfo: ListRootNodeInfoADM, children: Seq[ListChildNodeADM])
-    extends ListItemADM(listinfo, children) {
+case class ListADM(listinfo: ListRootNodeInfoADM, children: Seq[ListChildNodeADM]) extends ListItemADM() {
 
   /**
    * Sorts the whole hierarchy.
    *
    * @return a sorted [[List]].
    */
-  def sorted: ListADM =
-    ListADM(
-      listinfo = listinfo,
-      children = children.sortBy(_.position).map(_.sorted)
-    )
+  def sorted: ListADM = this.copy(children = children.sortBy(_.position).map(_.sorted))
 }
 
-case class NodeADM(nodeinfo: ListChildNodeInfoADM, children: Seq[ListChildNodeADM])
-    extends ListItemADM(nodeinfo, children) {
+case class NodeADM(nodeinfo: ListChildNodeInfoADM, children: Seq[ListChildNodeADM]) extends ListItemADM() {
 
   /**
    * Sorts the whole hierarchy.
    *
    * @return a sorted [[List]].
    */
-  def sorted: NodeADM =
-    NodeADM(
-      nodeinfo = nodeinfo,
-      children = children.sortBy(_.position).map(_.sorted)
-    )
+  def sorted: NodeADM = this.copy(children = children.sortBy(_.position).map(_.sorted))
 }
 
 /**
@@ -508,8 +498,6 @@ abstract class ListNodeInfoADM(
 
   def getName: Option[String] = name
 
-  def getLabels: StringLiteralSequenceV2 = labels
-
   def getComments: StringLiteralSequenceV2 = comments
 
   /**
@@ -683,8 +671,6 @@ abstract class ListNodeADM(
 
   def getName: Option[String] = name
 
-  def getLabels: StringLiteralSequenceV2 = labels
-
   def getComments: StringLiteralSequenceV2 = comments
 
   def getChildren: Seq[ListChildNodeADM] = children

webapi/src/main/scala/org/knora/webapi/messages/store/triplestoremessages/TriplestoreMessages.scala

@@ -19,8 +19,6 @@ import org.knora.webapi.messages.IriConversions._
 import org.knora.webapi.messages._
 import org.knora.webapi.messages.util.ErrorHandlingMap
 import org.knora.webapi.messages.util.rdf._
-import org.knora.webapi.store.triplestore.domain
-import org.knora.webapi.store.triplestore.domain.TriplestoreStatus
 
 /**
  * A response to a [[org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Construct]] query.
@@ -142,21 +140,6 @@ object SparqlExtendedConstructResponse {
     )
 }
 
-/**
- * A graph of triples in Turtle format.
- */
-case class NamedGraphDataResponse(turtle: String)
-
-/**
- * Response indicating whether the triplestore has finished initialization and is ready for processing messages
- *
- * @param triplestoreStatus the state of the triplestore.
- */
-case class CheckTriplestoreResponse(triplestoreStatus: domain.TriplestoreStatus)
-object CheckTriplestoreResponse {
-  val Available: CheckTriplestoreResponse = CheckTriplestoreResponse(TriplestoreStatus.Available)
-}
-
 /**
  * Indicates whether the repository is up to date.
  *

webapi/src/main/scala/org/knora/webapi/slice/admin/domain/service/ProjectExportService.scala

@@ -24,7 +24,7 @@ import zio.nio.file.Path
 import java.io.OutputStream
 import scala.collection.mutable
 
-import org.knora.webapi.messages.twirl.queries.sparql.admin.txt._
+import org.knora.webapi.messages.twirl.queries._
 import org.knora.webapi.messages.util.rdf.TriG
 import org.knora.webapi.slice.admin.AdminConstants.adminDataNamedGraph
 import org.knora.webapi.slice.admin.AdminConstants.permissionsDataNamedGraph
@@ -148,9 +148,8 @@ final case class ProjectExportServiceLive(
   private def downloadOntologyAndData(project: KnoraProject, tempDir: Path): Task[List[NamedGraphTrigFile]] = for {
     allGraphsTrigFile <-
       projectService.getNamedGraphsForProject(project).map(_.map(NamedGraphTrigFile(_, tempDir)))
-    files <- ZIO.foreach(allGraphsTrigFile)(file =>
-               triplestore.sparqlHttpGraphFile(file.graphIri, file.dataFile, TriG).as(file)
-             )
+    files <-
+      ZIO.foreach(allGraphsTrigFile)(file => triplestore.downloadGraph(file.graphIri, file.dataFile, TriG).as(file))
   } yield files
 
   /**
@@ -167,14 +166,14 @@ final case class ProjectExportServiceLive(
   private def downloadProjectAdminData(project: KnoraProject, targetDir: Path): Task[NamedGraphTrigFile] = {
     val graphIri = adminDataNamedGraph
     val file     = NamedGraphTrigFile(graphIri, targetDir)
-    val query    = Construct(getProjectAdminData(project.id.value))
+    val query    = Construct(sparql.admin.txt.getProjectAdminData(project.id.value))
     triplestore.queryToFile(query, graphIri, file.dataFile, TriG).as(file)
   }
 
   private def downloadPermissionData(project: KnoraProject, tempDir: Path) = {
     val graphIri = permissionsDataNamedGraph
     val file     = NamedGraphTrigFile(graphIri, tempDir)
-    val query    = Construct(getProjectPermissions(project.id.value))
+    val query    = Construct(sparql.admin.txt.getProjectPermissions(project.id.value))
     triplestore.queryToFile(query, graphIri, file.dataFile, TriG).as(file)
   }
 

webapi/src/main/scala/org/knora/webapi/store/triplestore/api/TriplestoreService.scala

@@ -11,7 +11,6 @@ import zio.macros.accessible
 
 import java.nio.file.Path
 
-import org.knora.webapi._
 import org.knora.webapi.messages.store.triplestoremessages._
 import org.knora.webapi.messages.util.rdf.QuadFormat
 import org.knora.webapi.messages.util.rdf.SparqlSelectResult
@@ -20,6 +19,7 @@ import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Ask
 import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Construct
 import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Select
 import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Update
+import org.knora.webapi.store.triplestore.domain.TriplestoreStatus
 
 @accessible
 trait TriplestoreService {
@@ -79,15 +79,7 @@ trait TriplestoreService {
    * @param outputFile           the file to be written.
    * @param outputFormat         the output file format.
    */
-  def sparqlHttpGraphFile(graphIri: InternalIri, outputFile: zio.nio.file.Path, outputFormat: QuadFormat): Task[Unit]
-
-  /**
-   * Requests the contents of a named graph, returning the response as Turtle.
-   *
-   * @param graphIri the IRI of the named graph.
-   * @return a string containing the contents of the graph in Turtle format.
-   */
-  def sparqlHttpGraphData(graphIri: IRI): Task[NamedGraphDataResponse]
+  def downloadGraph(graphIri: InternalIri, outputFile: zio.nio.file.Path, outputFormat: QuadFormat): Task[Unit]
 
   /**
    * Resets the content of the triplestore with the data supplied with the request.
@@ -101,11 +93,6 @@ trait TriplestoreService {
     prependDefaults: Boolean = true
   ): Task[Unit]
 
-  /**
-   * Drops (deletes) all data from the triplestore using "DROP ALL" SPARQL query.
-   */
-  def dropAllTriplestoreContent(): Task[Unit]
-
   /**
    * Wipes all triplestore data out using HTTP requests.
    */
@@ -128,7 +115,7 @@ trait TriplestoreService {
    * Checks the Fuseki triplestore if it is available and configured correctly. If it is not
    * configured, tries to automatically configure () the required dataset.
    */
-  def checkTriplestore(): Task[CheckTriplestoreResponse]
+  def checkTriplestore(): Task[TriplestoreStatus]
 
   /**
    * Dumps the whole repository in N-Quads format, saving the response in a file.
@@ -143,23 +130,19 @@ trait TriplestoreService {
    * @param inputFile an N-Quads file containing the content to be uploaded to the repository.
    */
   def uploadRepository(inputFile: Path): Task[Unit]
-
-  /**
-   * Puts a data graph into the repository.
-   *
-   * @param graphContent a data graph in Turtle format to be inserted into the repository.
-   * @param graphName    the name of the graph.
-   */
-  def insertDataGraphRequest(graphContent: String, graphName: String): Task[Unit]
-
 }
 
 object TriplestoreService {
   object Queries {
 
-    sealed trait SparqlQuery { val sparql: String }
+    sealed trait SparqlQuery {
+      val sparql: String
+      val isGravsearch: Boolean
+    }
 
-    case class Ask(sparql: String) extends SparqlQuery
+    case class Ask(sparql: String) extends SparqlQuery {
+      override val isGravsearch: Boolean = false
+    }
     object Ask {
       def apply(sparql: TxtFormat.Appendable): Ask = Ask(sparql.toString)
     }
@@ -180,7 +163,9 @@ object TriplestoreService {
       def apply(sparql: String): Construct = Construct(sparql, isGravsearch = false)
     }
 
-    case class Update(sparql: String) extends SparqlQuery
+    case class Update(sparql: String) extends SparqlQuery {
+      override val isGravsearch: Boolean = false
+    }
     object Update {
       def apply(sparql: TxtFormat.Appendable): Update = Update(sparql.toString())
     }

webapi/src/main/scala/org/knora/webapi/store/triplestore/api/TriplestoreServiceInMemory.scala

@@ -23,7 +23,6 @@ import zio.URIO
 import zio.ZIO
 import zio.ZLayer
 
-import java.io.StringReader
 import java.nio.charset.StandardCharsets
 import java.nio.file.Path
 import java.nio.file.Paths
@@ -32,8 +31,6 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala
 
 import org.knora.webapi.IRI
 import org.knora.webapi.messages.StringFormatter
-import org.knora.webapi.messages.store.triplestoremessages.CheckTriplestoreResponse
-import org.knora.webapi.messages.store.triplestoremessages.NamedGraphDataResponse
 import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject
 import org.knora.webapi.messages.store.triplestoremessages.SparqlConstructResponse
 import org.knora.webapi.messages.util.rdf.QuadFormat
@@ -53,6 +50,8 @@ import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Select
 import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Update
 import org.knora.webapi.store.triplestore.api.TriplestoreServiceInMemory.createEmptyDataset
 import org.knora.webapi.store.triplestore.defaults.DefaultRdfData
+import org.knora.webapi.store.triplestore.domain.TriplestoreStatus
+import org.knora.webapi.store.triplestore.domain.TriplestoreStatus.Available
 import org.knora.webapi.store.triplestore.errors.TriplestoreResponseException
 import org.knora.webapi.store.triplestore.errors.TriplestoreTimeoutException
 import org.knora.webapi.store.triplestore.errors.TriplestoreUnsupportedFeatureException
@@ -165,7 +164,7 @@ final case class TriplestoreServiceInMemory(datasetRef: Ref[Dataset], implicit v
     ZIO.scoped(getDataSetWithTransaction(ReadWrite.WRITE).flatMap(doUpdate)).unit
   }
 
-  override def sparqlHttpGraphFile(
+  override def downloadGraph(
     graphIri: InternalIri,
     outputFile: zio.nio.file.Path,
     outputFormat: QuadFormat
@@ -182,27 +181,15 @@ final case class TriplestoreServiceInMemory(datasetRef: Ref[Dataset], implicit v
     } yield ()
   }
 
-  override def sparqlHttpGraphData(graphIri: IRI): Task[NamedGraphDataResponse] = ZIO.scoped {
-    for {
-      ds <- getDataSetWithTransaction(ReadWrite.READ)
-      model <- ZIO
-                 .fromOption(Option(ds.getNamedModel(graphIri)))
-                 .orElseFail(TriplestoreResponseException(s"Triplestore returned no content for graph $graphIri."))
-      turtle <- modelToTurtle(model)
-    } yield NamedGraphDataResponse(turtle)
-  }
-
   override def resetTripleStoreContent(
     rdfDataObjects: List[RdfDataObject],
     prependDefaults: Boolean
   ): Task[Unit] = for {
-    _ <- dropAllTriplestoreContent()
+    _ <- dropDataGraphByGraph()
     _ <- insertDataIntoTriplestore(rdfDataObjects, prependDefaults)
   } yield ()
 
-  override def dropAllTriplestoreContent(): Task[Unit] = createEmptyDataset.flatMap(datasetRef.set(_)).unit
-
-  override def dropDataGraphByGraph(): Task[Unit] = dropAllTriplestoreContent()
+  override def dropDataGraphByGraph(): Task[Unit] = createEmptyDataset.flatMap(datasetRef.set(_)).unit
 
   override def insertDataIntoTriplestore(
     rdfDataObjects: List[RdfDataObject],
@@ -246,22 +233,13 @@ final case class TriplestoreServiceInMemory(datasetRef: Ref[Dataset], implicit v
       ZIO.succeed(graphName)
     }
 
-  override def checkTriplestore(): Task[CheckTriplestoreResponse] = ZIO.succeed(CheckTriplestoreResponse.Available)
+  override def checkTriplestore(): Task[TriplestoreStatus] = ZIO.succeed(Available)
 
   override def downloadRepository(outputFile: Path): Task[Unit] =
     ZIO.fail(new UnsupportedOperationException("Not implemented in TriplestoreServiceInMemory."))
 
   override def uploadRepository(inputFile: Path): Task[Unit] =
     ZIO.fail(new UnsupportedOperationException("Not implemented in TriplestoreServiceInMemory."))
-
-  override def insertDataGraphRequest(turtle: String, graphName: String): Task[Unit] =
-    ZIO.scoped {
-      for {
-        name <- checkGraphName(graphName)
-        ds   <- getDataSetWithTransaction(ReadWrite.WRITE)
-        _     = ds.getNamedModel(name).read(new StringReader(turtle), null, turtle)
-      } yield ()
-    }
 }
 
 object TriplestoreServiceInMemory {