Merge branch 'main' into refactor/migrate-update-lists-to-tapir

.dockerignore

@@ -1,4 +1,3 @@
-!knora-ontologies
 !webapi
 .git
 **/.gitignore

build.sbt

@@ -183,11 +183,6 @@ lazy val webapi: Project = Project(id = "webapi", base = file("webapi"))
   .settings(
     // add needed files to production jar
     Compile / packageBin / mappings ++= Seq(
-      (rootBaseDir.value / "knora-ontologies" / "knora-admin.ttl")                         -> "knora-ontologies/knora-admin.ttl",
-      (rootBaseDir.value / "knora-ontologies" / "knora-base.ttl")                          -> "knora-ontologies/knora-base.ttl",
-      (rootBaseDir.value / "knora-ontologies" / "salsah-gui.ttl")                          -> "knora-ontologies/salsah-gui.ttl",
-      (rootBaseDir.value / "knora-ontologies" / "standoff-data.ttl")                       -> "knora-ontologies/standoff-data.ttl",
-      (rootBaseDir.value / "knora-ontologies" / "standoff-onto.ttl")                       -> "knora-ontologies/standoff-onto.ttl",
       (rootBaseDir.value / "webapi" / "scripts" / "fuseki-repository-config.ttl.template") -> "webapi/scripts/fuseki-repository-config.ttl.template" // needed for initialization of triplestore
     ),
     // use packaged jars (through packageBin) on classpaths instead of class directories for production
@@ -208,8 +203,6 @@ lazy val webapi: Project = Project(id = "webapi", base = file("webapi"))
     Universal / mappings ++= {
       // copy the scripts folder
       directory("webapi/scripts") ++
-        // add knora-ontologies
-        directory("knora-ontologies") ++
         // copy configuration files to config directory
         contentOf("webapi/src/main/resources").toMap.mapValues("config/" + _)
     },

docker-compose.yml

@@ -93,6 +93,7 @@ services:
       - "3339:3339"
     volumes:
       - /tmp:/tmp
+      # Needed for local development when resetting the triplestore
       - ./knora-ontologies:/opt/knora-ontologies
       - ./test_data:/opt/test_data
     networks:

knora-ontologies

@@ -0,0 +1 @@
+./webapi/src/main/resources/knora-ontologies

project/Dependencies.scala

@@ -13,7 +13,7 @@ object Dependencies {
 
   val fusekiImage =
     "daschswiss/apache-jena-fuseki:2.1.4" // should be the same version as in docker-compose.yml, also make sure to use the same version when deploying it (i.e. version in ops-deploy)!
-  val sipiImage = "daschswiss/sipi:3.8.10" // base image the knora-sipi image is created from
+  val sipiImage = "daschswiss/sipi:3.8.11" // base image the knora-sipi image is created from
 
   val ScalaVersion = "2.13.12"
 

knora-ontologies/knora-base.ttl → webapi/src/main/resources/knora-ontologies/knora-base.ttl

@@ -19,7 +19,7 @@
     rdf:type           owl:Ontology ;
     rdfs:label         "The Knora base ontology"@en ;
     :attachedToProject knora-admin:SystemProject ;
-    :ontologyVersion   "knora-base v25" .
+    :ontologyVersion   "knora-base v26" .
 
 
 #################################################################

webapi/src/main/scala/org/knora/webapi/messages/OntologyConstants.scala

@@ -7,6 +7,8 @@ package org.knora.webapi
 package messages
 
 import dsp.constants.SalsahGui
+import org.knora.webapi.messages.OntologyConstants.KnoraAdmin.KnoraAdminOntologyIri
+import org.knora.webapi.slice.resourceinfo.domain.InternalIri
 
 /**
  * Contains string constants for IRIs from ontologies used by the application.
@@ -1106,6 +1108,8 @@ object OntologyConstants {
    */
   object NamedGraphs {
     val DataNamedGraphStart: IRI = "http://www.knora.org/data"
+
+    val KnoraAdminOntology: InternalIri = InternalIri(KnoraAdminOntologyIri)
   }
 
   object Fuseki {

webapi/src/main/scala/org/knora/webapi/package.scala

@@ -11,7 +11,7 @@ package object webapi {
    * The version of `knora-base` and of the other built-in ontologies that this version of Knora requires.
    * Must be the same as the object of `knora-base:ontologyVersion` in the `knora-base` ontology being used.
    */
-  val KnoraBaseVersion: String = "knora-base v25"
+  val KnoraBaseVersion: String = "knora-base v26"
 
   /**
    * `IRI` is a synonym for `String`, used to improve code readability.

webapi/src/main/scala/org/knora/webapi/store/triplestore/api/TriplestoreService.scala

@@ -20,6 +20,7 @@ import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Constru
 import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Select
 import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Update
 import org.knora.webapi.store.triplestore.domain.TriplestoreStatus
+import org.knora.webapi.store.triplestore.upgrade.GraphsForMigration
 
 @accessible
 trait TriplestoreService {
@@ -33,7 +34,7 @@ trait TriplestoreService {
   def query(sparql: Ask): Task[Boolean]
 
   /**
-   *  Performs a SPARQL CONSTRUCT query.
+   * Performs a SPARQL CONSTRUCT query.
    *
    * @param sparql The SPARQL [[Construct]] query.
    * @return a [[SparqlConstructResponse]]
@@ -43,7 +44,7 @@ trait TriplestoreService {
   /**
    * Performs a SPARQL SELECT query.
    *
-   * @param sparql          The SPARQL [[Select]] query.
+   * @param sparql The SPARQL [[Select]] query.
    * @return A [[SparqlSelectResult]].
    */
   def query(sparql: Select): Task[SparqlSelectResult]
@@ -63,7 +64,7 @@ trait TriplestoreService {
    * @param graphIri     the named graph IRI to be used in the output file.
    * @param outputFile   the output file.
    * @param outputFormat the output file format.
-   * @return  [[Unit]].
+   * @return [[Unit]].
    */
   def queryToFile(
     sparql: Construct,
@@ -75,17 +76,17 @@ trait TriplestoreService {
   /**
    * Requests the contents of a named graph, saving the response in a file.
    *
-   * @param graphIri             the IRI of the named graph.
-   * @param outputFile           the file to be written.
-   * @param outputFormat         the output file format.
+   * @param graphIri     the IRI of the named graph.
+   * @param outputFile   the file to be written.
+   * @param outputFormat the output file format.
    */
   def downloadGraph(graphIri: InternalIri, outputFile: zio.nio.file.Path, outputFormat: QuadFormat): Task[Unit]
 
   /**
    * Resets the content of the triplestore with the data supplied with the request.
    * First performs `dropAllTriplestoreContent` and afterwards `insertDataIntoTriplestore`.
    *
-   * @param rdfDataObjects a sequence of paths and graph names referencing data that needs to be inserted.
+   * @param rdfDataObjects  a sequence of paths and graph names referencing data that needs to be inserted.
    * @param prependDefaults denotes if the rdfDataObjects list should be prepended with a default set. Default is `true`.
    */
   def resetTripleStoreContent(
@@ -120,16 +121,18 @@ trait TriplestoreService {
   /**
    * Dumps the whole repository in N-Quads format, saving the response in a file.
    *
-   * @param outputFile           the output file.
+   * @param outputFile the output file.
    */
-  def downloadRepository(outputFile: Path): Task[Unit]
+  def downloadRepository(outputFile: Path, graphs: GraphsForMigration): Task[Unit]
 
   /**
    * Uploads repository content from an N-Quads file.
    *
    * @param inputFile an N-Quads file containing the content to be uploaded to the repository.
    */
   def uploadRepository(inputFile: Path): Task[Unit]
+
+  def dropGraph(graphName: String): Task[Unit]
 }
 
 object TriplestoreService {

webapi/src/main/scala/org/knora/webapi/store/triplestore/api/TriplestoreServiceInMemory.scala

@@ -43,6 +43,7 @@ import org.knora.webapi.store.triplestore.domain.TriplestoreStatus.Available
 import org.knora.webapi.store.triplestore.errors.TriplestoreResponseException
 import org.knora.webapi.store.triplestore.errors.TriplestoreTimeoutException
 import org.knora.webapi.store.triplestore.errors.TriplestoreUnsupportedFeatureException
+import org.knora.webapi.store.triplestore.upgrade.GraphsForMigration
 import org.knora.webapi.util.ZScopedJavaIoStreams.byteArrayOutputStream
 import org.knora.webapi.util.ZScopedJavaIoStreams.fileInputStream
 import org.knora.webapi.util.ZScopedJavaIoStreams.fileOutputStream
@@ -227,14 +228,18 @@ final case class TriplestoreServiceInMemory(datasetRef: Ref[Dataset], implicit v
 
   override def checkTriplestore(): Task[TriplestoreStatus] = ZIO.succeed(Available)
 
-  override def downloadRepository(outputFile: Path): Task[Unit] =
-    ZIO.fail(new UnsupportedOperationException("Not implemented in TriplestoreServiceInMemory."))
+  override def setDataset(ds: Dataset): UIO[Unit] = datasetRef.set(ds)
+
+  private val notImplemented = ZIO.die(new UnsupportedOperationException("Not implemented yet."))
+
+  override def downloadRepository(outputFile: Path, graphs: GraphsForMigration): Task[Unit] =
+    notImplemented
 
   override def uploadRepository(inputFile: Path): Task[Unit] =
-    ZIO.fail(new UnsupportedOperationException("Not implemented in TriplestoreServiceInMemory."))
+    notImplemented
 
-  override def setDataset(ds: Dataset): UIO[Unit] =
-    datasetRef.set(ds)
+  override def dropGraph(graphName: IRI): Task[Unit] =
+    notImplemented
 }
 
 object TriplestoreServiceInMemory {

webapi/src/main/scala/org/knora/webapi/store/triplestore/impl/TriplestoreServiceLive.scala

@@ -52,17 +52,16 @@ import org.knora.webapi.messages.store.triplestoremessages.*
 import org.knora.webapi.messages.util.rdf.*
 import org.knora.webapi.slice.resourceinfo.domain.InternalIri
 import org.knora.webapi.store.triplestore.api.TriplestoreService
-import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Ask
-import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Construct
-import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Select
-import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.SparqlQuery
-import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Update
+import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.*
 import org.knora.webapi.store.triplestore.defaults.DefaultRdfData
 import org.knora.webapi.store.triplestore.domain.TriplestoreStatus
 import org.knora.webapi.store.triplestore.domain.TriplestoreStatus.Available
 import org.knora.webapi.store.triplestore.domain.TriplestoreStatus.NotInitialized
 import org.knora.webapi.store.triplestore.domain.TriplestoreStatus.Unavailable
 import org.knora.webapi.store.triplestore.errors.*
+import org.knora.webapi.store.triplestore.upgrade.GraphsForMigration
+import org.knora.webapi.store.triplestore.upgrade.MigrateAllGraphs
+import org.knora.webapi.store.triplestore.upgrade.MigrateSpecificGraphs
 import org.knora.webapi.util.FileUtil
 
 case class TriplestoreServiceLive(
@@ -194,7 +193,7 @@ case class TriplestoreServiceLive(
       _      <- ZIO.logInfo("==>> Drop All Data End")
     } yield ()
 
-  private def dropGraph(graphName: String): Task[Unit] =
+  override def dropGraph(graphName: String): Task[Unit] =
     ZIO.logInfo(s"==>> Dropping graph: $graphName") *>
       query(Update(s"DROP GRAPH <$graphName>")) *>
       ZIO.logDebug("Graph dropped")
@@ -426,15 +425,24 @@ case class TriplestoreServiceLive(
   }
 
   /**
-   * Dumps the whole repository in N-Quads format, saving the response in a file.
+   * Dumps the whole repository or only specific graphs in N-Quads format, saving the response in a file.
    *
-   * @param outputFile           the output file.
-   * @return a string containing the contents of the graph in N-Quads format.
+   * @param outputFile  The path to the output file.
+   * @param graphs      Specify which graphs are to be dumped.
+   * @return [[Unit]]   Or fails if the export was not successful.
    */
-  override def downloadRepository(outputFile: Path): Task[Unit] = {
-    val request = new HttpGet(paths.repository)
-    request.addHeader("Accept", mimeTypeApplicationNQuads)
-    doHttpRequest(request, writeResponseFileAsPlainContent(outputFile)).unit
+  override def downloadRepository(outputFile: Path, graphs: GraphsForMigration): Task[Unit] = {
+    def downloadWholeRepository = {
+      val request = new HttpGet(paths.repository)
+      request.addHeader("Accept", mimeTypeApplicationNQuads)
+      doHttpRequest(request, writeResponseFileAsPlainContent(outputFile)).unit
+    }
+    graphs match {
+      case MigrateAllGraphs =>
+        downloadWholeRepository
+      case MigrateSpecificGraphs(graphIris) =>
+        ZIO.foreach(graphIris)(graphIri => downloadGraph(graphIri, zio.nio.file.Path.fromJava(outputFile), NQuads)).unit
+    }
   }
 
   /**

webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala

@@ -7,6 +7,7 @@ package org.knora.webapi.store.triplestore.upgrade
 
 import com.typesafe.scalalogging.Logger
 
+import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject
 import org.knora.webapi.store.triplestore.upgrade.plugins.*
 
 /**
@@ -41,51 +42,52 @@ object RepositoryUpdatePlan {
       ),
       PluginForKnoraBaseVersion(
         versionNumber = 5,
-        plugin = new NoopPlugin,
+        plugin = new MigrateOnlyBuiltInGraphs,
         prBasedVersionString = Some("PR 1440")
       ),
-      PluginForKnoraBaseVersion(versionNumber = 6, plugin = new NoopPlugin), // PR 1206
-      PluginForKnoraBaseVersion(versionNumber = 7, plugin = new NoopPlugin), // PR 1403
+      PluginForKnoraBaseVersion(versionNumber = 6, plugin = new MigrateOnlyBuiltInGraphs), // PR 1206
+      PluginForKnoraBaseVersion(versionNumber = 7, plugin = new MigrateOnlyBuiltInGraphs), // PR 1403
       PluginForKnoraBaseVersion(versionNumber = 8, plugin = new UpgradePluginPR1615()),
       PluginForKnoraBaseVersion(versionNumber = 9, plugin = new UpgradePluginPR1746(log)),
-      PluginForKnoraBaseVersion(versionNumber = 10, plugin = new NoopPlugin), // PR 1808
-      PluginForKnoraBaseVersion(versionNumber = 11, plugin = new NoopPlugin), // PR 1813
-      PluginForKnoraBaseVersion(versionNumber = 12, plugin = new NoopPlugin), // PR 1891
+      PluginForKnoraBaseVersion(versionNumber = 10, plugin = new MigrateOnlyBuiltInGraphs), // PR 1808
+      PluginForKnoraBaseVersion(versionNumber = 11, plugin = new MigrateOnlyBuiltInGraphs), // PR 1813
+      PluginForKnoraBaseVersion(versionNumber = 12, plugin = new MigrateOnlyBuiltInGraphs), // PR 1891
       PluginForKnoraBaseVersion(versionNumber = 13, plugin = new UpgradePluginPR1921(log)),
-      PluginForKnoraBaseVersion(versionNumber = 14, plugin = new NoopPlugin), // PR 1992
+      PluginForKnoraBaseVersion(versionNumber = 14, plugin = new MigrateOnlyBuiltInGraphs), // PR 1992
       PluginForKnoraBaseVersion(versionNumber = 20, plugin = new UpgradePluginPR2018(log)),
       PluginForKnoraBaseVersion(versionNumber = 21, plugin = new UpgradePluginPR2079(log)),
       PluginForKnoraBaseVersion(versionNumber = 22, plugin = new UpgradePluginPR2081(log)),
       PluginForKnoraBaseVersion(versionNumber = 23, plugin = new UpgradePluginPR2094(log)),
-      PluginForKnoraBaseVersion(versionNumber = 24, plugin = new NoopPlugin), // PR 2076
-      PluginForKnoraBaseVersion(versionNumber = 25, plugin = new NoopPlugin)  // PR 2268
+      PluginForKnoraBaseVersion(versionNumber = 24, plugin = new MigrateOnlyBuiltInGraphs), // PR 2076
+      PluginForKnoraBaseVersion(versionNumber = 25, plugin = new MigrateOnlyBuiltInGraphs), // PR 2268
+      PluginForKnoraBaseVersion(versionNumber = 26, plugin = new MigrateOnlyBuiltInGraphs)  // PR 3003
       // KEEP IT ON THE BOTTOM
       // From "versionNumber = 6" don't use prBasedVersionString!
     )
 
   /**
    * The built-in named graphs that are always updated when there is a new version of knora-base.
    */
-  val builtInNamedGraphs: Set[BuiltInNamedGraph] = Set(
-    BuiltInNamedGraph(
-      filename = "knora-ontologies/knora-admin.ttl",
-      iri = "http://www.knora.org/ontology/knora-admin"
+  val builtInNamedGraphs: Set[RdfDataObject] = Set(
+    RdfDataObject(
+      path = "knora-ontologies/knora-admin.ttl",
+      name = "http://www.knora.org/ontology/knora-admin"
     ),
-    BuiltInNamedGraph(
-      filename = "knora-ontologies/knora-base.ttl",
-      iri = "http://www.knora.org/ontology/knora-base"
+    RdfDataObject(
+      path = "knora-ontologies/knora-base.ttl",
+      name = "http://www.knora.org/ontology/knora-base"
     ),
-    BuiltInNamedGraph(
-      filename = "knora-ontologies/salsah-gui.ttl",
-      iri = "http://www.knora.org/ontology/salsah-gui"
+    RdfDataObject(
+      path = "knora-ontologies/salsah-gui.ttl",
+      name = "http://www.knora.org/ontology/salsah-gui"
     ),
-    BuiltInNamedGraph(
-      filename = "knora-ontologies/standoff-onto.ttl",
-      iri = "http://www.knora.org/ontology/standoff"
+    RdfDataObject(
+      path = "knora-ontologies/standoff-onto.ttl",
+      name = "http://www.knora.org/ontology/standoff"
     ),
-    BuiltInNamedGraph(
-      filename = "knora-ontologies/standoff-data.ttl",
-      iri = "http://www.knora.org/data/standoff"
+    RdfDataObject(
+      path = "knora-ontologies/standoff-data.ttl",
+      name = "http://www.knora.org/data/standoff"
     )
   )
 
@@ -107,13 +109,4 @@ object RepositoryUpdatePlan {
         case None      => s"knora-base v$versionNumber"
       }
   }
-
-  /**
-   * Represents a Knora built-in named graph.
-   *
-   * @param filename the filename containing the named graph.
-   * @param iri      the IRI of the named graph.
-   */
-  case class BuiltInNamedGraph(filename: String, iri: String)
-
 }