From df28afc23f90ec067ef3e9e20ff1603e8bb4d501 Mon Sep 17 00:00:00 2001 From: Raitis Veinbahs Date: Tue, 13 Feb 2024 18:35:58 +0100 Subject: [PATCH] chore: Remove `knora-ontologies` symlink (DEV-3236) (#3035) --- build.sbt | 6 ++- docker-compose.yml | 1 - knora-ontologies | 1 - test_data/test_route/texts/README.md | 4 +- .../scripts/fuseki-init-knora-test-minimal.sh | 10 ++-- webapi/scripts/fuseki-init-knora-test.sh | 10 ++-- .../triplestore/defaults/DefaultRdfData.scala | 26 ++-------- .../impl/TriplestoreServiceLive.scala | 51 ++++++++++--------- .../api/TriplestoreServiceInMemory.scala | 14 +++-- .../api/TriplestoreServiceInMemorySpec.scala | 4 +- 10 files changed, 59 insertions(+), 68 deletions(-) delete mode 120000 knora-ontologies diff --git a/build.sbt b/build.sbt index ec02996ca1..eb1089e6ae 100644 --- a/build.sbt +++ b/build.sbt @@ -207,7 +207,11 @@ lazy val webapi: Project = Project(id = "webapi", base = file("webapi")) contentOf("webapi/src/main/resources").toMap.mapValues("config/" + _) }, // add 'config' directory to the classpath of the start script, - Universal / scriptClasspath := Seq("webapi/scripts", "knora-ontologies", "../config/") ++ scriptClasspath.value, + Universal / scriptClasspath := Seq( + "webapi/scripts", + "webapi/src/main/resources/knora-ontologies", + "../config/" + ) ++ scriptClasspath.value, // need this here, so that the Manifest inside the jars has the correct main class set. Compile / mainClass := Some("org.knora.webapi.Main"), // add dockerCommands used to create the image diff --git a/docker-compose.yml b/docker-compose.yml index 50fd7e7f22..70785b34cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -94,7 +94,6 @@ services: volumes: - /tmp:/tmp # Needed for local development when resetting the triplestore - - ./knora-ontologies:/opt/knora-ontologies - ./test_data:/opt/test_data networks: - knora-net diff --git a/knora-ontologies b/knora-ontologies deleted file mode 120000 index 1ae87a5693..0000000000 --- a/knora-ontologies +++ /dev/null @@ -1 +0,0 @@ -./webapi/src/main/resources/knora-ontologies \ No newline at end of file diff --git a/test_data/test_route/texts/README.md b/test_data/test_route/texts/README.md index 95428ab4e9..b50804c7cf 100644 --- a/test_data/test_route/texts/README.md +++ b/test_data/test_route/texts/README.md @@ -21,7 +21,7 @@ Is the mapping used to translate html markup using the default mapping `Ontology - example text file: `StandardHTML` Note: that you can use this mapping to generate the default mapping that is distributed -as `knora-ontologies/standoff-data.ttl`. +as `webapi/src/main/resources/knora-ontologies/standoff-data.ttl`. Please read the official documentation about Standoff Standard Mapping. Step to update standard mapping: @@ -76,7 +76,7 @@ WHERE { } ``` -5. Find differences to update by hand (to avoid massive IRI update) the `knora-ontologies/standoff-data.ttl`, pay attention to namespaces (e.g. replace all `http://rdfh.ch/projects/0001/mappings/update-standard-mapping` by `http://rdfh.ch/standoff/mappings/StandardMapping`)! +5. Find differences to update by hand (to avoid massive IRI update) the `webapi/src/main/resources/knora-ontologies/standoff-data.ttl`, pay attention to namespaces (e.g. replace all `http://rdfh.ch/projects/0001/mappings/update-standard-mapping` by `http://rdfh.ch/standoff/mappings/StandardMapping`)! 6. Prepare an upgrade script of an existing Knora base ## HTML diff --git a/webapi/scripts/fuseki-init-knora-test-minimal.sh b/webapi/scripts/fuseki-init-knora-test-minimal.sh index 5486c19557..4e0baa0077 100755 --- a/webapi/scripts/fuseki-init-knora-test-minimal.sh +++ b/webapi/scripts/fuseki-init-knora-test-minimal.sh @@ -8,10 +8,10 @@ REPOSITORY="knora-test" # delete-repository // delete dos not work correctly. need to delete database manually. create-repository -upload-graph ../../knora-ontologies/knora-admin.ttl http://www.knora.org/ontology/knora-admin -upload-graph ../../knora-ontologies/knora-base.ttl http://www.knora.org/ontology/knora-base -upload-graph ../../knora-ontologies/standoff-onto.ttl http://www.knora.org/ontology/standoff -upload-graph ../../knora-ontologies/standoff-data.ttl http://www.knora.org/data/standoff -upload-graph ../../knora-ontologies/salsah-gui.ttl http://www.knora.org/ontology/salsah-gui +upload-graph ../../webapi/src/main/resources/knora-ontologies/knora-admin.ttl http://www.knora.org/ontology/knora-admin +upload-graph ../../webapi/src/main/resources/knora-ontologies/knora-base.ttl http://www.knora.org/ontology/knora-base +upload-graph ../../webapi/src/main/resources/knora-ontologies/standoff-onto.ttl http://www.knora.org/ontology/standoff +upload-graph ../../webapi/src/main/resources/knora-ontologies/standoff-data.ttl http://www.knora.org/data/standoff +upload-graph ../../webapi/src/main/resources/knora-ontologies/salsah-gui.ttl http://www.knora.org/ontology/salsah-gui upload-graph ../../test_data/project_data/admin-data-minimal.ttl http://www.knora.org/data/admin upload-graph ../../test_data/project_data/permissions-data-minimal.ttl http://www.knora.org/data/permissions diff --git a/webapi/scripts/fuseki-init-knora-test.sh b/webapi/scripts/fuseki-init-knora-test.sh index fcd63afd0e..14cf863da4 100755 --- a/webapi/scripts/fuseki-init-knora-test.sh +++ b/webapi/scripts/fuseki-init-knora-test.sh @@ -8,11 +8,11 @@ REPOSITORY="knora-test" # delete-repository // delete dos not work correctly. need to delete database manually. create-repository -upload-graph ../../knora-ontologies/knora-admin.ttl http://www.knora.org/ontology/knora-admin -upload-graph ../../knora-ontologies/knora-base.ttl http://www.knora.org/ontology/knora-base -upload-graph ../../knora-ontologies/standoff-onto.ttl http://www.knora.org/ontology/standoff -upload-graph ../../knora-ontologies/standoff-data.ttl http://www.knora.org/data/standoff -upload-graph ../../knora-ontologies/salsah-gui.ttl http://www.knora.org/ontology/salsah-gui +upload-graph ../../webapi/src/main/resources/knora-ontologies/knora-admin.ttl http://www.knora.org/ontology/knora-admin +upload-graph ../../webapi/src/main/resources/knora-ontologies/knora-base.ttl http://www.knora.org/ontology/knora-base +upload-graph ../../webapi/src/main/resources/knora-ontologies/standoff-onto.ttl http://www.knora.org/ontology/standoff +upload-graph ../../webapi/src/main/resources/knora-ontologies/standoff-data.ttl http://www.knora.org/data/standoff +upload-graph ../../webapi/src/main/resources/knora-ontologies/salsah-gui.ttl http://www.knora.org/ontology/salsah-gui upload-graph ../../test_data/project_data/admin-data.ttl http://www.knora.org/data/admin upload-graph ../../test_data/project_data/permissions-data.ttl http://www.knora.org/data/permissions upload-graph ../../test_data/project_data/system-data.ttl http://www.knora.org/data/0000/SystemProject diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/defaults/DefaultRdfData.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/defaults/DefaultRdfData.scala index 5a1dfc1ab3..b789cb00ee 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/defaults/DefaultRdfData.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/defaults/DefaultRdfData.scala @@ -5,9 +5,11 @@ package org.knora.webapi.store.triplestore.defaults +import zio.Chunk import zio.NonEmptyChunk import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject +import org.knora.webapi.store.triplestore.upgrade.RepositoryUpdatePlan object DefaultRdfData { @@ -18,27 +20,7 @@ object DefaultRdfData { * then a list of `RdfDataObject` instances containing the path and the name of the named graph * can be supplied to the `ResetTriplestoreContent` message. */ - val data = NonEmptyChunk( - RdfDataObject( - path = "knora-ontologies/knora-admin.ttl", - name = "http://www.knora.org/ontology/knora-admin" - ), - RdfDataObject( - path = "knora-ontologies/knora-base.ttl", - name = "http://www.knora.org/ontology/knora-base" - ), - RdfDataObject( - path = "knora-ontologies/standoff-onto.ttl", - name = "http://www.knora.org/ontology/standoff" - ), - RdfDataObject( - path = "knora-ontologies/standoff-data.ttl", - name = "http://www.knora.org/data/standoff" - ), - RdfDataObject( - path = "knora-ontologies/salsah-gui.ttl", - name = "http://www.knora.org/ontology/salsah-gui" - ), + val data: NonEmptyChunk[RdfDataObject] = NonEmptyChunk( RdfDataObject( path = "test_data/project_data/admin-data.ttl", name = "http://www.knora.org/data/admin" @@ -83,5 +65,5 @@ object DefaultRdfData { path = "test_data/project_ontologies/webern-onto.ttl", name = "http://www.knora.org/ontology/0806/webern" ) - ) + ) ++ Chunk(RepositoryUpdatePlan.builtInNamedGraphs.toSeq: _*) } diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/impl/TriplestoreServiceLive.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/impl/TriplestoreServiceLive.scala index 206027c16d..b4dc8ed7da 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/impl/TriplestoreServiceLive.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/impl/TriplestoreServiceLive.scala @@ -42,6 +42,7 @@ import java.nio.file.Paths import java.nio.file.StandardCopyOption import java.time.temporal.ChronoUnit import java.util +import scala.io.Source import dsp.errors.* import org.knora.webapi.* @@ -224,21 +225,17 @@ case class TriplestoreServiceLive( rdfDataObjects: List[RdfDataObject], prependDefaults: Boolean ): Task[Unit] = { - val calculateCompleteRdfDataObjectList: Task[NonEmptyChunk[RdfDataObject]] = - if (prependDefaults) { // prepend - if (rdfDataObjects.isEmpty) { - ZIO.succeed(DefaultRdfData.data) - } else { - // prepend default data objects like those of knora-base, knora-admin, etc. - ZIO.succeed(DefaultRdfData.data ++ NonEmptyChunk.fromIterable(rdfDataObjects.head, rdfDataObjects.tail)) - } - } else { // don't prepend - if (rdfDataObjects.isEmpty) { - ZIO.fail(BadRequestException("Cannot insert list with empty data into triplestore.")) - } else { - ZIO.succeed(NonEmptyChunk.fromIterable(rdfDataObjects.head, rdfDataObjects.tail)) - } + if (prependDefaults) { + ZIO.succeed(DefaultRdfData.data.append(Chunk(rdfDataObjects: _*))) + } else { + NonEmptyChunk + .fromIterableOption(rdfDataObjects) + .fold[Task[NonEmptyChunk[RdfDataObject]]]( + ZIO.fail(BadRequestException("Cannot insert list with empty data into triplestore.")) + )( + ZIO.succeed(_) + ) } for { @@ -261,19 +258,13 @@ case class TriplestoreServiceLive( uriBuilder } + rdfContents <- loadRdfObject(elem.path) + httpPost <- ZIO.attemptBlocking { - val httpPost = new HttpPost(uriBuilder.build()) - // Add the input file to the body of the request. - // here we need to tweak the base directory path from "webapi" - // to the parent folder where the files can be found - val inputFile = Paths.get("..", elem.path) - if (!Files.exists(inputFile)) { - throw BadRequestException(s"File ${inputFile.toAbsolutePath} does not exist") - } - val fileEntity = - new FileEntity(inputFile.toFile, ContentType.create(mimeTypeTextTurtle, "UTF-8")) - httpPost.setEntity(fileEntity) + val httpPost = new HttpPost(uriBuilder.build()) + val turtleContentType = ContentType.create(mimeTypeTextTurtle, "UTF-8") + httpPost.setEntity(new StringEntity(rdfContents, turtleContentType)) httpPost } responseHandler <- ZIO.attempt(returnInsertGraphDataResponse(graphName)(_)) @@ -284,6 +275,16 @@ case class TriplestoreServiceLive( } yield () } + /** + * Load the RdfDataObject into some AbstractHttpEntity from either a relative ../$path or through a resource. + */ + private def loadRdfObject(path: String): Task[String] = { + val relativeFileEntity = ZIO.readFile(Paths.get("..", path)) + val resourceStringEntity = ZIO.attemptBlocking(Source.fromResource(path).mkString) + + relativeFileEntity.orElse(resourceStringEntity) + } + /** * Checks the Fuseki triplestore if it is available and configured correctly. If it is not * configured, tries to automatically configure (initialize) the required dataset. diff --git a/webapi/src/test/scala/org/knora/webapi/store/triplestore/api/TriplestoreServiceInMemory.scala b/webapi/src/test/scala/org/knora/webapi/store/triplestore/api/TriplestoreServiceInMemory.scala index b1c00d0e9d..2676e7221f 100644 --- a/webapi/src/test/scala/org/knora/webapi/store/triplestore/api/TriplestoreServiceInMemory.scala +++ b/webapi/src/test/scala/org/knora/webapi/store/triplestore/api/TriplestoreServiceInMemory.scala @@ -22,6 +22,7 @@ import zio.ZIO import zio.ZLayer import zio.macros.accessible +import java.io.InputStream import java.nio.charset.StandardCharsets import java.nio.file.Path import java.nio.file.Paths @@ -210,18 +211,23 @@ final case class TriplestoreServiceInMemory(datasetRef: Ref[Dataset], implicit v } } - private def insertRdfDataObject(elem: RdfDataObject): ZIO[Any, Throwable, Unit] = { - val inputFile = Paths.get("..", elem.path) + private def insertRdfDataObject(elem: RdfDataObject): ZIO[Any, Throwable, Unit] = ZIO.scoped { for { graphName <- checkGraphName(elem) - in <- fileInputStream(inputFile) + in <- loadRdfUrl(elem.path) ds <- getDataSetWithTransaction(ReadWrite.WRITE) model = ds.getNamedModel(graphName) _ = model.read(in, null, "TURTLE") } yield () } - } + + private def loadRdfUrl(path: String): ZIO[Any & Scope, Throwable, InputStream] = + ZIO + .attemptBlocking( + Option(getClass.getClassLoader.getResourceAsStream(path)).getOrElse(throw new Exception("can't find resource")) + ) + .orElse(fileInputStream(Paths.get("..", path))) private def checkGraphName(elem: RdfDataObject): Task[String] = checkGraphName(elem.name) diff --git a/webapi/src/test/scala/org/knora/webapi/store/triplestore/api/TriplestoreServiceInMemorySpec.scala b/webapi/src/test/scala/org/knora/webapi/store/triplestore/api/TriplestoreServiceInMemorySpec.scala index 1cd97cd05a..cf40998306 100644 --- a/webapi/src/test/scala/org/knora/webapi/store/triplestore/api/TriplestoreServiceInMemorySpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/store/triplestore/api/TriplestoreServiceInMemorySpec.scala @@ -262,7 +262,7 @@ object TriplestoreServiceInMemorySpec extends ZIOSpecDefault { _ <- TriplestoreService.insertDataIntoTriplestore( List( RdfDataObject( - path = "knora-ontologies/knora-base.ttl", + path = "webapi/src/main/resources/knora-ontologies/knora-base.ttl", name = "http://www.knora.org/ontology/knora-admin" ) ), @@ -283,7 +283,7 @@ object TriplestoreServiceInMemorySpec extends ZIOSpecDefault { _ <- TriplestoreService.insertDataIntoTriplestore( List( RdfDataObject( - path = "knora-ontologies/knora-base.ttl", + path = "webapi/src/main/resources/knora-ontologies/knora-base.ttl", name = "http://www.knora.org/ontology/knora-base" ) ),