diff --git a/integration/src/test/resources/sipi/testfiles/B1D0OkEgfFp-Cew2Seur7Wi.jp2 b/integration/src/test/resources/sipi/testfiles/B1D0OkEgfFp-Cew2Seur7Wi.jp2 new file mode 100644 index 0000000000..4818db7d07 Binary files /dev/null and b/integration/src/test/resources/sipi/testfiles/B1D0OkEgfFp-Cew2Seur7Wi.jp2 differ diff --git a/integration/src/test/resources/sipi/testfiles/De6XyNL4H71-D9QxghOuOPJ.info b/integration/src/test/resources/sipi/testfiles/De6XyNL4H71-D9QxghOuOPJ.info index 9b1745b97d..063e51d847 100644 --- a/integration/src/test/resources/sipi/testfiles/De6XyNL4H71-D9QxghOuOPJ.info +++ b/integration/src/test/resources/sipi/testfiles/De6XyNL4H71-D9QxghOuOPJ.info @@ -3,5 +3,9 @@ "internalFilename": "De6XyNL4H71-D9QxghOuOPJ.jp2", "checksumOriginal": "710aa7c06f7a7293f4cab70855ab98e166762a6b3ab7b6bdb4f6750bfc7028ed", "originalFilename": "dog.png", - "checksumDerivative": "a89a2522adefe4510016ebdf337d82d3af0eef2a5c008c8c76553db4adb64155" + "checksumDerivative": "a89a2522adefe4510016ebdf337d82d3af0eef2a5c008c8c76553db4adb64155", + "width": 72, + "height": 72, + "internalMimeType": "image/jpx", + "originalMimeType": "image/png" } \ No newline at end of file diff --git a/integration/src/test/resources/sipi/testfiles/FGiLaT4zzuV-CqwbEDFAFeS.info b/integration/src/test/resources/sipi/testfiles/FGiLaT4zzuV-CqwbEDFAFeS.info index 286314d79c..127a95aa97 100644 --- a/integration/src/test/resources/sipi/testfiles/FGiLaT4zzuV-CqwbEDFAFeS.info +++ b/integration/src/test/resources/sipi/testfiles/FGiLaT4zzuV-CqwbEDFAFeS.info @@ -3,5 +3,9 @@ "checksumDerivative": "0ce405c9b183fb0d0a9998e9a49e39c93b699e0f8e2a9ac3496c349e5cea09cc", "internalFilename": "FGiLaT4zzuV-CqwbEDFAFeS.jp2", "checksumOriginal": "fb252a4fb3d90ce4ebc7e123d54a4112398a7994541b11aab5e4230eac01a61c", - "originalFilename": "250x250.jp2" + "originalFilename": "250x250.jp2", + "width": 250, + "height": 250, + "internalMimeType": "image/jpx", + "originalMimeType": "image/jp2" } \ No newline at end of file diff --git a/integration/src/test/scala/org/knora/sipi/SipiIT.scala b/integration/src/test/scala/org/knora/sipi/SipiIT.scala index fc2853c001..d1a733e070 100644 --- a/integration/src/test/scala/org/knora/sipi/SipiIT.scala +++ b/integration/src/test/scala/org/knora/sipi/SipiIT.scala @@ -28,6 +28,7 @@ import org.knora.webapi.messages.admin.responder.sipimessages._ import org.knora.webapi.messages.util.KnoraSystemInstances.Users.SystemUser import org.knora.webapi.routing.JwtService import org.knora.webapi.routing.JwtServiceLive +import org.knora.webapi.testcontainers.SharedVolumes import org.knora.webapi.testcontainers.SipiTestContainer object SipiIT extends ZIOSpecDefault { @@ -36,9 +37,6 @@ object SipiIT extends ZIOSpecDefault { private val infoTestfile = "FGiLaT4zzuV-CqwbEDFAFeS.info" private val origTestfile = "FGiLaT4zzuV-CqwbEDFAFeS.jp2.orig" private val prefix = "0001" - private def copyTestFilesToSipi = ZIO.foreach(List(imageTestfile, infoTestfile, origTestfile))( - SipiTestContainer.copyFileToImageFolderInContainer(prefix, _) - ) private def getWithoutAuthorization(path: Path) = SipiTestContainer @@ -62,7 +60,6 @@ object SipiIT extends ZIOSpecDefault { ) { for { jwt <- getToken - _ <- copyTestFilesToSipi _ <- MockDspApiServer.resetAndAllowWithPermissionCode(prefix, imageTestfile, 2) response <- SipiTestContainer @@ -96,7 +93,6 @@ object SipiIT extends ZIOSpecDefault { ) { for { jwt <- getToken - _ <- copyTestFilesToSipi _ <- MockDspApiServer.resetAndAllowWithPermissionCode(prefix, imageTestfile, 2) response <- SipiTestContainer @@ -139,7 +135,6 @@ object SipiIT extends ZIOSpecDefault { |}""".stripMargin.fromJson[Json] for { _ <- MockDspApiServer.resetAndAllowWithPermissionCode(prefix, imageTestfile, permissionCode = 2) - _ <- copyTestFilesToSipi response <- getWithoutAuthorization(Root / prefix / imageTestfile / "knora.json") json <- response.body.asString.map(_.fromJson[Json]) expected <- SipiTestContainer.portAndHost.map { case (port, host) => expectedJson(port, host) } @@ -173,7 +168,6 @@ object SipiIT extends ZIOSpecDefault { val dspApiPermissionPath = s"/admin/files/$prefix/$imageTestfile" for { server <- MockDspApiServer.resetAndStubGetResponse(dspApiPermissionPath, 200, dspApiResponse) - _ <- copyTestFilesToSipi response <- getWithoutAuthorization(Root / prefix / imageTestfile / "file") } yield assertTrue( response.status == Status.Ok, @@ -189,7 +183,6 @@ object SipiIT extends ZIOSpecDefault { val dspApiPermissionPath = s"/admin/files/$prefix/$imageTestfile" for { server <- MockDspApiServer.resetAndStubGetResponse(dspApiPermissionPath, 200, dspApiResponse) - _ <- copyTestFilesToSipi response <- getWithoutAuthorization(Root / prefix / imageTestfile / "file") } yield assertTrue( response.status == Status.Unauthorized, @@ -204,7 +197,6 @@ object SipiIT extends ZIOSpecDefault { val dspApiPermissionPath = s"/admin/files/$prefix/$imageTestfile" for { server <- MockDspApiServer.resetAndStubGetResponse(dspApiPermissionPath, 404) - _ <- copyTestFilesToSipi response <- getWithoutAuthorization(Root / prefix / imageTestfile / "file") } yield assertTrue( response.status == Status.NotFound, @@ -239,7 +231,6 @@ object SipiIT extends ZIOSpecDefault { val dspApiPermissionPath = s"/admin/files/$prefix/$imageTestfile" for { server <- MockDspApiServer.resetAndStubGetResponse(dspApiPermissionPath, 200, dspApiResponse) - _ <- copyTestFilesToSipi response <- getWithoutAuthorization(Root / prefix / imageTestfile / "full/max/0/default.jp2") } yield assertTrue( response.status == Status.Ok, @@ -255,7 +246,6 @@ object SipiIT extends ZIOSpecDefault { val dspApiPermissionPath = s"/admin/files/$prefix/$imageTestfile" for { server <- MockDspApiServer.resetAndStubGetResponse(dspApiPermissionPath, 200, dspApiResponse) - _ <- copyTestFilesToSipi response <- getWithoutAuthorization(Root / prefix / imageTestfile / "full/max/0/default.jp2") } yield assertTrue( response.status == Status.Unauthorized, @@ -270,7 +260,6 @@ object SipiIT extends ZIOSpecDefault { val dspApiPermissionPath = s"/admin/files/$prefix/$imageTestfile" for { server <- MockDspApiServer.resetAndStubGetResponse(dspApiPermissionPath, 404) - _ <- copyTestFilesToSipi response <- getWithoutAuthorization(Root / prefix / imageTestfile / "full/max/0/default.jp2") } yield assertTrue( response.status == Status.NotFound, @@ -293,7 +282,9 @@ object SipiIT extends ZIOSpecDefault { } yield assertTrue(response.status.isSuccess, verifyNoInteractionWith(server)) } ) - .provideSomeLayerShared[Scope with Client with WireMockServer](SipiTestContainer.layer) + .provideSomeLayerShared[Scope with Client with WireMockServer]( + SharedVolumes.Images.layer >+> SipiTestContainer.layer + ) .provideSomeLayerShared[Scope with Client](MockDspApiServer.layer) .provideSomeLayer[Scope](Client.default) @@ TestAspect.sequential } diff --git a/integration/src/test/scala/org/knora/webapi/ITKnoraLiveSpec.scala b/integration/src/test/scala/org/knora/webapi/ITKnoraLiveSpec.scala index 179ad23799..e8bfe26f5f 100644 --- a/integration/src/test/scala/org/knora/webapi/ITKnoraLiveSpec.scala +++ b/integration/src/test/scala/org/knora/webapi/ITKnoraLiveSpec.scala @@ -8,6 +8,9 @@ package org.knora.webapi import com.typesafe.scalalogging.LazyLogging import com.typesafe.scalalogging.Logger import org.apache.pekko +import org.apache.pekko.http.scaladsl.client.RequestBuilding +import org.apache.pekko.http.scaladsl.model._ +import org.apache.pekko.testkit.TestKitBase import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec @@ -31,15 +34,10 @@ import org.knora.webapi.messages.store.triplestoremessages.TriplestoreJsonProtoc import org.knora.webapi.messages.util.rdf.JsonLDDocument import org.knora.webapi.messages.util.rdf.JsonLDUtil import org.knora.webapi.routing.UnsafeZioRun -import org.knora.webapi.testcontainers.SipiTestContainer import org.knora.webapi.testservices.FileToUpload import org.knora.webapi.testservices.TestClientService import org.knora.webapi.util.LogAspect -import pekko.http.scaladsl.client.RequestBuilding -import pekko.http.scaladsl.model._ -import pekko.testkit.TestKitBase - /** * This class can be used in End-to-End testing. It starts the DSP stack and * provides access to configuration and logging. @@ -83,9 +81,6 @@ abstract class ITKnoraLiveSpec config <- ZIO.service[AppConfig] } yield (router, config) - def copyFileToImageFolderInContainer(prefix: String, filename: String) = - UnsafeZioRun.runOrThrow(SipiTestContainer.copyFileToImageFolderInContainer(prefix, filename)) - /** * Create router and config by unsafe running them. */ diff --git a/integration/src/test/scala/org/knora/webapi/config/AppConfigForTestContainers.scala b/integration/src/test/scala/org/knora/webapi/config/AppConfigForTestContainers.scala index c4228e1701..eb74f6a244 100644 --- a/integration/src/test/scala/org/knora/webapi/config/AppConfigForTestContainers.scala +++ b/integration/src/test/scala/org/knora/webapi/config/AppConfigForTestContainers.scala @@ -12,6 +12,7 @@ import zio.config.magnolia._ import zio.config.typesafe.TypesafeConfigSource import org.knora.webapi.config.AppConfig.AppConfigurations +import org.knora.webapi.testcontainers.DspIngestTestContainer import org.knora.webapi.testcontainers.FusekiTestContainer import org.knora.webapi.testcontainers.SipiTestContainer @@ -23,19 +24,27 @@ object AppConfigForTestContainers { private def alterFusekiAndSipiPort( oldConfig: AppConfig, fusekiContainer: FusekiTestContainer, - sipiContainer: SipiTestContainer + sipiContainer: SipiTestContainer, + dspIngestContainer: DspIngestTestContainer ): UIO[AppConfig] = { - val newFusekiPort = fusekiContainer.getFirstMappedPort - val newSipiPort = sipiContainer.container.getFirstMappedPort() + val newFusekiPort = fusekiContainer.getFirstMappedPort + val newSipiPort = sipiContainer.getFirstMappedPort + val newDspIngestPort = dspIngestContainer.getFirstMappedPort val alteredFuseki = oldConfig.triplestore.fuseki.copy(port = newFusekiPort) val alteredTriplestore = oldConfig.triplestore.copy(fuseki = alteredFuseki) val alteredSipi = oldConfig.sipi.copy(internalPort = newSipiPort) + val alteredDspIngest = oldConfig.dspIngest.copy(baseUrl = s"http://localhost:$newDspIngestPort") val newConfig: AppConfig = - oldConfig.copy(allowReloadOverHttp = true, triplestore = alteredTriplestore, sipi = alteredSipi) + oldConfig.copy( + allowReloadOverHttp = true, + triplestore = alteredTriplestore, + sipi = alteredSipi, + dspIngest = alteredDspIngest + ) ZIO.succeed(newConfig) } @@ -69,15 +78,17 @@ object AppConfigForTestContainers { private val config: UIO[AppConfig] = (read(descriptor[AppConfig].mapKey(toKebabCase) from source)).orDie /** - * Altered AppConfig with ports from TestContainers for Fuseki and Sipi. + * Altered AppConfig with ports from TestContainers for DSP-Ingest, Fuseki and Sipi. */ - val testcontainers: ZLayer[FusekiTestContainer & SipiTestContainer, Nothing, AppConfigurations] = { + val testcontainers + : ZLayer[DspIngestTestContainer & FusekiTestContainer & SipiTestContainer, Nothing, AppConfigurations] = { val appConfigLayer = ZLayer { for { - appConfig <- config - fusekiContainer <- ZIO.service[FusekiTestContainer] - sipiContainer <- ZIO.service[SipiTestContainer] - alteredConfig <- alterFusekiAndSipiPort(appConfig, fusekiContainer, sipiContainer) + appConfig <- config + fusekiContainer <- ZIO.service[FusekiTestContainer] + sipiContainer <- ZIO.service[SipiTestContainer] + dspIngestContainer <- ZIO.service[DspIngestTestContainer] + alteredConfig <- alterFusekiAndSipiPort(appConfig, fusekiContainer, sipiContainer, dspIngestContainer) } yield alteredConfig } AppConfig diff --git a/integration/src/test/scala/org/knora/webapi/config/AppConfigForTestContainersZSpec.scala b/integration/src/test/scala/org/knora/webapi/config/AppConfigForTestContainersZSpec.scala index 646c71529e..d36f0ee463 100644 --- a/integration/src/test/scala/org/knora/webapi/config/AppConfigForTestContainersZSpec.scala +++ b/integration/src/test/scala/org/knora/webapi/config/AppConfigForTestContainersZSpec.scala @@ -8,23 +8,31 @@ package org.knora.webapi.config import zio._ import zio.test._ +import org.knora.webapi.testcontainers.DspIngestTestContainer import org.knora.webapi.testcontainers.FusekiTestContainer +import org.knora.webapi.testcontainers.SharedVolumes import org.knora.webapi.testcontainers.SipiTestContainer object AppConfigForTestContainersZSpec extends ZIOSpecDefault { - def spec = suite("AppConfigForTestContainersSpec")( + def spec: Spec[Any, Nothing] = suite("AppConfigForTestContainersSpec")( test("successfully provide the adapted application configuration for using with test containers") { for { - appConfig <- ZIO.service[AppConfig] - sipiContainer <- ZIO.service[SipiTestContainer] + appConfig <- ZIO.service[AppConfig] + sipiContainer <- ZIO.service[SipiTestContainer] + dspIngestContainer <- ZIO.service[DspIngestTestContainer] } yield { - assertTrue(appConfig.sipi.internalPort == sipiContainer.port) + assertTrue( + appConfig.sipi.internalPort == sipiContainer.getFirstMappedPort, + appConfig.dspIngest.baseUrl.endsWith(dspIngestContainer.getFirstMappedPort.toString) + ) } } ).provide( AppConfigForTestContainers.testcontainers, - SipiTestContainer.layer, - FusekiTestContainer.layer + DspIngestTestContainer.layer, + FusekiTestContainer.layer, + SharedVolumes.Images.layer, + SipiTestContainer.layer ) } diff --git a/integration/src/test/scala/org/knora/webapi/core/LayersTest.scala b/integration/src/test/scala/org/knora/webapi/core/LayersTest.scala index b4102a1456..63087b6093 100644 --- a/integration/src/test/scala/org/knora/webapi/core/LayersTest.scala +++ b/integration/src/test/scala/org/knora/webapi/core/LayersTest.scala @@ -60,7 +60,9 @@ import org.knora.webapi.store.iiif.impl.SipiServiceMock import org.knora.webapi.store.triplestore.api.TriplestoreService import org.knora.webapi.store.triplestore.impl.TriplestoreServiceLive import org.knora.webapi.store.triplestore.upgrade.RepositoryUpdater +import org.knora.webapi.testcontainers.DspIngestTestContainer import org.knora.webapi.testcontainers.FusekiTestContainer +import org.knora.webapi.testcontainers.SharedVolumes import org.knora.webapi.testcontainers.SipiTestContainer import org.knora.webapi.testservices.TestClientService @@ -70,7 +72,10 @@ object LayersTest { * The `Environment`s that we require for the tests to run - with or without Sipi */ type DefaultTestEnvironmentWithoutSipi = LayersLive.DspEnvironmentLive with FusekiTestContainer with TestClientService - type DefaultTestEnvironmentWithSipi = DefaultTestEnvironmentWithoutSipi with SipiTestContainer + type DefaultTestEnvironmentWithSipi = DefaultTestEnvironmentWithoutSipi + with SipiTestContainer + with DspIngestTestContainer + with SharedVolumes.Images type CommonR0 = ActorSystem with AppConfigurations with SipiService with JwtService with StringFormatter type CommonR = @@ -202,18 +207,23 @@ object LayersTest { private val fusekiAndSipiTestcontainers = ZLayer.make[ - FusekiTestContainer - with SipiTestContainer - with AppConfigurations + AppConfigurations + with DspIngestTestContainer + with FusekiTestContainer with JwtService + with SharedVolumes.Images with SipiService + with SipiTestContainer with StringFormatter ]( AppConfigForTestContainers.testcontainers, + DspIngestClientLive.layer, + DspIngestTestContainer.layer, FusekiTestContainer.layer, SipiTestContainer.layer, SipiServiceLive.layer, JwtServiceLive.layer, + SharedVolumes.Images.layer, StringFormatter.test ) diff --git a/integration/src/test/scala/org/knora/webapi/it/v2/KnoraSipiAuthenticationITSpec.scala b/integration/src/test/scala/org/knora/webapi/it/v2/KnoraSipiAuthenticationITSpec.scala index 8d5cc82f6e..83735fec74 100644 --- a/integration/src/test/scala/org/knora/webapi/it/v2/KnoraSipiAuthenticationITSpec.scala +++ b/integration/src/test/scala/org/knora/webapi/it/v2/KnoraSipiAuthenticationITSpec.scala @@ -5,7 +5,9 @@ package org.knora.webapi.it.v2 -import org.apache.pekko +import org.apache.pekko.http.scaladsl.model._ +import org.apache.pekko.http.scaladsl.model.headers.BasicHttpCredentials +import org.apache.pekko.http.scaladsl.unmarshalling.Unmarshal import java.nio.file.Files import java.nio.file.Paths @@ -20,10 +22,6 @@ import org.knora.webapi.routing.Authenticator import org.knora.webapi.routing.UnsafeZioRun import org.knora.webapi.sharedtestdata.SharedTestDataADM -import pekko.http.scaladsl.model._ -import pekko.http.scaladsl.model.headers.BasicHttpCredentials -import pekko.http.scaladsl.unmarshalling.Unmarshal - /** * Tests interaction between Knora and Sipi using Knora API v2. */ @@ -65,7 +63,6 @@ class KnoraSipiAuthenticationITSpec } "successfully get an image with provided credentials inside cookie" in { - // using cookie to authenticate when accessing sipi (test for cookie parsing in sipi) val KnoraAuthenticationCookieName = UnsafeZioRun.runOrThrow(Authenticator.calculateCookieName()) val cookieHeader = headers.Cookie(KnoraAuthenticationCookieName, loginToken) diff --git a/integration/src/test/scala/org/knora/webapi/it/v2/KnoraSipiIntegrationV2ITSpec.scala b/integration/src/test/scala/org/knora/webapi/it/v2/KnoraSipiIntegrationV2ITSpec.scala index 1cd59a8f75..e9e52bcb2a 100644 --- a/integration/src/test/scala/org/knora/webapi/it/v2/KnoraSipiIntegrationV2ITSpec.scala +++ b/integration/src/test/scala/org/knora/webapi/it/v2/KnoraSipiIntegrationV2ITSpec.scala @@ -504,9 +504,6 @@ class KnoraSipiIntegrationV2ITSpec } "create a resource with a still image file that has already been ingested" in { - copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.jp2") - copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.info") - copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.png.orig") // Create the resource in the API. val jsonLdEntity = UploadFileRequest .make(fileType = FileType.StillImageFile(), internalFilename = "De6XyNL4H71-D9QxghOuOPJ.jp2") @@ -522,9 +519,6 @@ class KnoraSipiIntegrationV2ITSpec } "not create a resource with a still image file that has already been ingested if the header is not provided" in { - copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.jp2") - copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.info") - copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.png.orig") // Create the resource in the API. val jsonLdEntity = UploadFileRequest .make(fileType = FileType.StillImageFile(), internalFilename = "De6XyNL4H71-D9QxghOuOPJ.jp2") diff --git a/integration/src/test/scala/org/knora/webapi/messages/util/standoff/XMLUtilSpec.scala b/integration/src/test/scala/org/knora/webapi/messages/util/standoff/XMLUtilSpec.scala index 669afb3dd7..69b1071dd0 100644 --- a/integration/src/test/scala/org/knora/webapi/messages/util/standoff/XMLUtilSpec.scala +++ b/integration/src/test/scala/org/knora/webapi/messages/util/standoff/XMLUtilSpec.scala @@ -3,8 +3,10 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.knora.webapi.util.standoff +package org.knora.webapi.messages.util.standoff +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec import org.xmlunit.builder.DiffBuilder import org.xmlunit.builder.Input import org.xmlunit.diff.Diff @@ -12,14 +14,9 @@ import org.xmlunit.diff.Diff import java.nio.file.Paths import dsp.errors.StandoffConversionException -import org.knora.webapi.CoreSpec -import org.knora.webapi.messages.util.standoff.XMLUtil import org.knora.webapi.util.FileUtil -/** - * Tests [[org.knora.webapi.messages.util.standoff.XMLToStandoffUtil]]. - */ -class XMLUtilSpec extends CoreSpec { +class XMLUtilSpec extends AnyWordSpec with Matchers { "The XML to standoff utility" should { diff --git a/integration/src/test/scala/org/knora/webapi/slice/admin/domain/service/ProjectImportServiceIT.scala b/integration/src/test/scala/org/knora/webapi/slice/admin/domain/service/ProjectImportServiceIT.scala index 538e6e297b..b629766e89 100644 --- a/integration/src/test/scala/org/knora/webapi/slice/admin/domain/service/ProjectImportServiceIT.scala +++ b/integration/src/test/scala/org/knora/webapi/slice/admin/domain/service/ProjectImportServiceIT.scala @@ -13,7 +13,8 @@ import java.io.IOException import org.knora.webapi.config.Fuseki import org.knora.webapi.config.Triplestore -import org.knora.webapi.slice.admin.domain.model.KnoraProject.Shortcode +import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.AssetId +import org.knora.webapi.slice.admin.domain.model.KnoraProject import org.knora.webapi.testcontainers.FusekiTestContainer object ProjectImportServiceIT extends ZIOSpecDefault { @@ -26,15 +27,6 @@ object ProjectImportServiceIT extends ZIOSpecDefault { } yield ProjectExportStorageServiceLive(exportDirectory) } - private val dspIngestClientLayer: ULayer[DspIngestClient] = ZLayer.succeed { - new DspIngestClient { - override def exportProject(shortcode: Shortcode): ZIO[Scope, Throwable, Path] = - ZIO.succeed(Path("unused")) - override def importProject(shortcode: Shortcode, fileToImport: Path): Task[Path] = - ZIO.succeed(Path("unused")) - } - } - private val importServiceTestLayer : URLayer[FusekiTestContainer with ProjectExportStorageService, ProjectImportServiceLive] = ZLayer.fromZIO { (for { @@ -58,7 +50,7 @@ object ProjectImportServiceIT extends ZIOSpecDefault { profileQueries = false ) } yield ProjectImportServiceLive(config, exportStorageService, dspIngestClient)) - .provideSomeLayer[FusekiTestContainer with ProjectExportStorageService](dspIngestClientLayer) + .provideSomeLayer[FusekiTestContainer with ProjectExportStorageService](DspIngestClientITMock.layer) } private val trigContent = @@ -117,3 +109,26 @@ object FileTestUtil { _ <- Files.writeBytes(filePath, Chunk.fromIterable(content.getBytes)) } yield filePath } + +final case class DspIngestClientITMock() extends DspIngestClient { + override def exportProject(shortcode: KnoraProject.Shortcode): ZIO[Scope, Throwable, Path] = + ZIO.succeed(Path("/tmp/test.zip")) + override def importProject(shortcode: KnoraProject.Shortcode, fileToImport: Path): Task[Path] = + ZIO.succeed(Path("/tmp/test.zip")) + + override def getAssetInfo(shortcode: KnoraProject.Shortcode, assetId: AssetId): Task[AssetInfoResponse] = + ZIO.succeed( + AssetInfoResponse( + s"$assetId.txt", + s"$assetId.txt.orig", + "test.txt", + "bfd3192ea04d5f42d79836cf3b8fbf17007bab71", + "17bab70071fbf8b3fc63897d24f5d40ae2913dfb", + internalMimeType = Some("text/plain"), + originalMimeType = Some("text/plain") + ) + ) +} +object DspIngestClientITMock { + val layer = ZLayer.derive[DspIngestClientITMock] +} diff --git a/integration/src/test/scala/org/knora/webapi/store/iiif/impl/SipiServiceMock.scala b/integration/src/test/scala/org/knora/webapi/store/iiif/impl/SipiServiceMock.scala index d105796e0b..19595caf4e 100644 --- a/integration/src/test/scala/org/knora/webapi/store/iiif/impl/SipiServiceMock.scala +++ b/integration/src/test/scala/org/knora/webapi/store/iiif/impl/SipiServiceMock.scala @@ -11,6 +11,7 @@ import zio.nio.file.Path import org.knora.webapi.messages.admin.responder.usersmessages.UserADM import org.knora.webapi.messages.store.sipimessages._ import org.knora.webapi.messages.v2.responder.SuccessResponseV2 +import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.AssetId import org.knora.webapi.slice.admin.domain.model.KnoraProject import org.knora.webapi.slice.admin.domain.service.Asset import org.knora.webapi.store.iiif.api.FileMetadataSipiResponse @@ -28,7 +29,7 @@ case class SipiServiceMock() extends SipiService { */ private val FAILURE_FILENAME: String = "failure.jp2" - override def getFileMetadataFromTemp(filename: String): Task[FileMetadataSipiResponse] = + override def getFileMetadataFromSipiTemp(filename: String): Task[FileMetadataSipiResponse] = ZIO.succeed( FileMetadataSipiResponse( originalFilename = Some("test2.tiff"), @@ -64,8 +65,10 @@ case class SipiServiceMock() extends SipiService { override def downloadAsset(asset: Asset, targetDir: Path, user: UserADM): Task[Option[Path]] = ??? - override def getFileMetadata(filename: String, shortcode: KnoraProject.Shortcode): Task[FileMetadataSipiResponse] = - ??? + override def getFileMetadataFromDspIngest( + shortcode: KnoraProject.Shortcode, + assetId: AssetId + ): Task[FileMetadataSipiResponse] = ??? } object SipiServiceMock { diff --git a/integration/src/test/scala/org/knora/webapi/testcontainers/DspIngestTestContainer.scala b/integration/src/test/scala/org/knora/webapi/testcontainers/DspIngestTestContainer.scala new file mode 100644 index 0000000000..a814163e92 --- /dev/null +++ b/integration/src/test/scala/org/knora/webapi/testcontainers/DspIngestTestContainer.scala @@ -0,0 +1,47 @@ +/* + * Copyright © 2021 - 2023 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.knora.webapi.testcontainers + +import org.testcontainers.containers.BindMode +import org.testcontainers.containers.GenericContainer +import zio.URLayer +import zio.ZIO +import zio.ZLayer + +import org.knora.webapi.testcontainers.TestContainerOps.StartableOps + +final class DspIngestTestContainer extends GenericContainer[DspIngestTestContainer](s"daschswiss/dsp-ingest:latest") + +object DspIngestTestContainer { + + private val assetDir = "/opt/images" + private val tempDir = "/opt/temp" + + def make(imagesVolume: SharedVolumes.Images): DspIngestTestContainer = { + val port = 3340 + new DspIngestTestContainer() + .withExposedPorts(port) + .withEnv("SERVICE_PORT", s"$port") + .withEnv("SERVICE_LOG_FORMAT", "text") + .withEnv("JWT_AUDIENCE", s"http://localhost:$port") + .withEnv("JWT_ISSUER", "0.0.0.0:3333") + .withEnv("STORAGE_ASSET_DIR", assetDir) + .withEnv("STORAGE_TEMP_DIR", tempDir) + .withEnv("JWT_SECRET", "UP 4888, nice 4-8-4 steam engine") + .withEnv("SIPI_USE_LOCAL_DEV", "false") + .withEnv("JWT_DISABLE_AUTH", "true") + .withFileSystemBind(imagesVolume.hostPath, assetDir, BindMode.READ_WRITE) + } + + private val initDspIngest = ZLayer.fromZIO( + ZIO.serviceWithZIO[DspIngestTestContainer] { it => + ZIO.attemptBlocking(it.execInContainer("mkdir", s"$tempDir")).orDie + } + ) + + val layer: URLayer[SharedVolumes.Images, DspIngestTestContainer] = + ZLayer.scoped(ZIO.service[SharedVolumes.Images].flatMap(make(_).toZio)) >+> initDspIngest +} diff --git a/integration/src/test/scala/org/knora/webapi/testcontainers/FusekiTestContainer.scala b/integration/src/test/scala/org/knora/webapi/testcontainers/FusekiTestContainer.scala index 8050ac5881..03e03bd566 100644 --- a/integration/src/test/scala/org/knora/webapi/testcontainers/FusekiTestContainer.scala +++ b/integration/src/test/scala/org/knora/webapi/testcontainers/FusekiTestContainer.scala @@ -6,11 +6,10 @@ package org.knora.webapi.testcontainers import org.testcontainers.containers.GenericContainer -import org.testcontainers.utility.DockerImageName +import zio.Task +import zio.ULayer import zio.ZIO -import zio._ import zio.http.URL -import zio.macros.accessible import zio.nio.file.Files import zio.nio.file.Path @@ -22,9 +21,9 @@ import java.net.http.HttpRequest.BodyPublishers import java.net.http.HttpResponse.BodyHandlers import org.knora.webapi.http.version.BuildInfo +import org.knora.webapi.testcontainers.TestContainerOps.StartableOps -@accessible -trait FusekiTestContainer extends GenericContainer[FusekiTestContainer] { +final class FusekiTestContainer extends GenericContainer[FusekiTestContainer](BuildInfo.fuseki) { def baseUrl: URL = { val urlString = s"http://$getHost:$getFirstMappedPort" @@ -59,24 +58,16 @@ trait FusekiTestContainer extends GenericContainer[FusekiTestContainer] { } object FusekiTestContainer { - def apply(dockerImageName: DockerImageName): FusekiTestContainer = - new GenericContainer[FusekiTestContainer](dockerImageName) with FusekiTestContainer - - def apply(): FusekiTestContainer = - new GenericContainer[FusekiTestContainer](DockerImageName.parse(BuildInfo.fuseki)) with FusekiTestContainer val adminPassword = "test" - private val acquire: Task[FusekiTestContainer] = { - val container = FusekiTestContainer() - .withExposedPorts(3030) - .withEnv("ADMIN_PASSWORD", adminPassword) - .withEnv("JVM_ARGS", "-Xmx3G") - ZIO.attemptBlocking(container.start()).as(container).orDie <* ZIO.logInfo(">>> Acquire Fuseki TestContainer <<<") - } + def initializeWithDataset(repositoryName: String): ZIO[FusekiTestContainer, Throwable, Unit] = + ZIO.serviceWithZIO[FusekiTestContainer](_.initializeWithDataset(repositoryName)) - private def release(container: FusekiTestContainer): UIO[Unit] = - ZIO.attemptBlocking(container.stop()).logError.ignore <* ZIO.logInfo(">>> Release Fuseki TestContainer <<<") + def make: FusekiTestContainer = new FusekiTestContainer() + .withExposedPorts(3030) + .withEnv("ADMIN_PASSWORD", adminPassword) + .withEnv("JVM_ARGS", "-Xmx3G") - val layer: ULayer[FusekiTestContainer] = ZLayer.scoped(ZIO.acquireRelease(acquire)(release)).orDie + val layer: ULayer[FusekiTestContainer] = make.toLayer } diff --git a/integration/src/test/scala/org/knora/webapi/testcontainers/SharedVolumes.scala b/integration/src/test/scala/org/knora/webapi/testcontainers/SharedVolumes.scala new file mode 100644 index 0000000000..366c38560d --- /dev/null +++ b/integration/src/test/scala/org/knora/webapi/testcontainers/SharedVolumes.scala @@ -0,0 +1,63 @@ +/* + * Copyright © 2021 - 2023 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.knora.webapi.testcontainers + +import zio.ULayer +import zio.ZIO +import zio.ZLayer +import zio.nio.file.Files +import zio.nio.file.Path + +import java.io.FileNotFoundException +import java.nio.file.StandardCopyOption +import java.nio.file.attribute.PosixFilePermissions +import java.nio.file.attribute.PosixFilePermissions.asFileAttribute + +import org.knora.webapi.slice.admin.domain.model.KnoraProject.Shortcode + +object SharedVolumes { + + final case class Images private (hostPath: String) extends AnyVal + + object Images { + + private val rwPermissions = asFileAttribute(PosixFilePermissions.fromString("rwxrwxrwx")) + + val layer: ULayer[Images] = + ZLayer scoped { + val tmp = Path(Option(System.getenv("RUNNER_TEMP")).getOrElse(System.getProperty("java.io.tmpdir"))) + for { + tmpPath <- Files.createTempDirectoryScoped(tmp, None, Seq(rwPermissions)) + _ <- createAssets(tmpPath).tap(n => ZIO.logInfo(s"Created $n assets")).logError + absDir <- tmpPath.toAbsolutePath.map(_.toString()) + } yield Images(absDir) + }.orDie + + private def createAssets(assetDir: Path) = { + val testfilesDir = Path(getClass.getResource("/sipi/testfiles").toURI) + val shortcode = Shortcode.unsafeFrom("0001") + Files + .walk(testfilesDir, 1) + .filterZIO(p => Files.isRegularFile(p) && Files.isHidden(p).negate) + .mapZIO(p => copyFileToAssetFolder(assetDir, p, shortcode).as(1)) + .runSum + } + + private def copyFileToAssetFolder( + assetDir: Path, + source: Path, + shortcode: Shortcode + ) = + ZIO.fail(new FileNotFoundException(s"File not found $source")).whenZIO(Files.notExists(source)).logError *> { + val filename = source.filename.toString() + val seg01 = filename.substring(0, 2).toLowerCase() + val seg02 = filename.substring(2, 4).toLowerCase() + val targetDir = assetDir / shortcode.value / seg01 / seg02 + Files.createDirectories(targetDir, rwPermissions).logError *> + Files.copy(source, targetDir / filename, StandardCopyOption.REPLACE_EXISTING) + } + } +} diff --git a/integration/src/test/scala/org/knora/webapi/testcontainers/SipiTestContainer.scala b/integration/src/test/scala/org/knora/webapi/testcontainers/SipiTestContainer.scala index 292053fecf..a316c8e086 100644 --- a/integration/src/test/scala/org/knora/webapi/testcontainers/SipiTestContainer.scala +++ b/integration/src/test/scala/org/knora/webapi/testcontainers/SipiTestContainer.scala @@ -7,45 +7,24 @@ package org.knora.webapi.testcontainers import org.testcontainers.containers.BindMode import org.testcontainers.containers.GenericContainer -import org.testcontainers.utility.DockerImageName -import org.testcontainers.utility.MountableFile -import zio.Task -import zio.UIO import zio.URIO +import zio.URLayer import zio.ZIO import zio.ZLayer import zio.http import zio.http.URL -import zio.nio.file.Path import java.net.Inet6Address import java.net.InetAddress -import java.nio.file.Paths import org.knora.webapi.http.version.BuildInfo +import org.knora.webapi.testcontainers.TestContainerOps.StartableOps -final case class SipiTestContainer(container: GenericContainer[Nothing]) { - def copyFileToImageFolderInContainer(prefix: String, filename: String): Task[Unit] = { - val seg01 = filename.substring(0, 2).toLowerCase() - val seg02 = filename.substring(2, 4).toLowerCase() - val target = Path(s"/sipi/images/$prefix/$seg01/$seg02/$filename") - copyTestFileToContainer(filename, target) - } - - def copyTestFileToContainer(file: String, target: Path): Task[Unit] = { - val resourceName = s"sipi/testfiles/$file" - val mountableFile = MountableFile.forClasspathResource(resourceName, 777) - ZIO.attemptBlockingIO(container.copyFileToContainer(mountableFile, target.toFile.toString)) <* ZIO.logInfo( - s"copied $resourceName to $target" - ) - } - - val port: Int = container.getFirstMappedPort - - val host: String = SipiTestContainer.localHostAddress +final class SipiTestContainer + extends GenericContainer[SipiTestContainer](s"daschswiss/knora-sipi:${BuildInfo.version}") { - val sipiBaseUrl: URL = { - val urlString = s"http://$host:$port" + def sipiBaseUrl: URL = { + val urlString = s"http://${SipiTestContainer.localHostAddress}:$getFirstMappedPort" println(s"SIPI URL String: $urlString") val url = URL.decode(urlString).getOrElse(throw new IllegalStateException(s"Invalid URL $urlString")) println(s"SIPI URL: $url") @@ -55,6 +34,8 @@ final case class SipiTestContainer(container: GenericContainer[Nothing]) { object SipiTestContainer { + private val imagesDir = "/sipi/images" + val localHostAddress: String = { val localhost = InetAddress.getLocalHost if (localhost.isInstanceOf[Inet6Address]) { @@ -64,68 +45,46 @@ object SipiTestContainer { } } - def port: ZIO[SipiTestContainer, Nothing, Int] = ZIO.serviceWith[SipiTestContainer](_.port) - def host: ZIO[SipiTestContainer, Nothing, String] = ZIO.serviceWith[SipiTestContainer](_.host) - def portAndHost: ZIO[SipiTestContainer, Nothing, (Int, String)] = port <*> host + def portAndHost: ZIO[SipiTestContainer, Nothing, (Int, String)] = + ZIO.serviceWith[SipiTestContainer](c => (c.getFirstMappedPort, localHostAddress)) def resolveUrl(path: http.Path): URIO[SipiTestContainer, URL] = ZIO.serviceWith[SipiTestContainer](_.sipiBaseUrl.path(path)) - def copyFileToImageFolderInContainer(prefix: String, filename: String): ZIO[SipiTestContainer, Throwable, Unit] = - ZIO.serviceWithZIO[SipiTestContainer](_.copyFileToImageFolderInContainer(prefix, filename)) - - def copyTestFileToContainer(file: String, target: Path): ZIO[SipiTestContainer, Throwable, Unit] = - ZIO.serviceWithZIO[SipiTestContainer](_.copyTestFileToContainer(file, target)) - - /** - * A functional effect that initiates a Sipi Testcontainer - */ - val acquire: UIO[GenericContainer[Nothing]] = ZIO.attemptBlocking { - // Uncomment the following line to use the latest version of Sipi for local development: - // val sipiImageName: DockerImageName = DockerImageName.parse(s"daschswiss/knora-sipi:latest") - val sipiImageName: DockerImageName = DockerImageName.parse(s"daschswiss/knora-sipi:${BuildInfo.version}") - val sipiContainer = new GenericContainer(sipiImageName) - sipiContainer.withExposedPorts(1024) - sipiContainer.withEnv("KNORA_WEBAPI_KNORA_API_EXTERNAL_HOST", "0.0.0.0") - sipiContainer.withEnv("KNORA_WEBAPI_KNORA_API_EXTERNAL_PORT", "3333") - sipiContainer.withEnv("SIPI_EXTERNAL_PROTOCOL", "http") - sipiContainer.withEnv("SIPI_EXTERNAL_HOSTNAME", "0.0.0.0") - sipiContainer.withEnv("SIPI_EXTERNAL_PORT", "1024") - sipiContainer.withEnv("SIPI_WEBAPI_HOSTNAME", SipiTestContainer.localHostAddress) - sipiContainer.withEnv("SIPI_WEBAPI_PORT", "3333") - sipiContainer.withEnv("CLEAN_TMP_DIR_USER", "clean_tmp_dir_user") - sipiContainer.withEnv("CLEAN_TMP_DIR_PW", "clean_tmp_dir_pw") - - sipiContainer.withCommand("--config=/sipi/config/sipi.docker-config.lua") - - sipiContainer.withClasspathResourceMapping( - "/sipi.docker-config.lua", - "/sipi/config/sipi.docker-config.lua", - BindMode.READ_ONLY - ) - - val incunabulaImageDirPath = - Paths.get("..", "sipi/images/0001/b1/d0/B1D0OkEgfFp-Cew2Seur7Wi.jp2") - sipiContainer.withFileSystemBind( - incunabulaImageDirPath.toString, - "/sipi/images/0001/b1/d0/B1D0OkEgfFp-Cew2Seur7Wi.jp2", - BindMode.READ_ONLY - ) - sipiContainer.withLogConsumer(frame => print("SIPI:" + frame.getUtf8String)) - - sipiContainer.start() - - // Create '/sipi/images/tmp' folder inside running container - sipiContainer.execInContainer("mkdir", "/sipi/images/tmp") - sipiContainer.execInContainer("chmod", "777", "/sipi/images/tmp") - - sipiContainer - }.orDie.zipLeft(ZIO.logInfo(">>> Acquire Sipi TestContainer <<<")) - - def release(container: GenericContainer[Nothing]): UIO[Unit] = ZIO.attemptBlocking { - container.stop() - }.orDie.zipLeft(ZIO.logInfo(">>> Release Sipi TestContainer <<<")) - - val layer: ZLayer[Any, Nothing, SipiTestContainer] = - ZLayer.scoped(ZIO.acquireRelease(acquire)(release).map(SipiTestContainer(_))) + def make(imagesVolume: SharedVolumes.Images): SipiTestContainer = + new SipiTestContainer() + .withExposedPorts(1024) + .withEnv("KNORA_WEBAPI_KNORA_API_EXTERNAL_HOST", "0.0.0.0") + .withEnv("KNORA_WEBAPI_KNORA_API_EXTERNAL_PORT", "3333") + .withEnv("SIPI_EXTERNAL_PROTOCOL", "http") + .withEnv("SIPI_EXTERNAL_HOSTNAME", "0.0.0.0") + .withEnv("SIPI_EXTERNAL_PORT", "1024") + .withEnv("SIPI_WEBAPI_HOSTNAME", SipiTestContainer.localHostAddress) + .withEnv("SIPI_WEBAPI_PORT", "3333") + .withEnv("CLEAN_TMP_DIR_USER", "clean_tmp_dir_user") + .withEnv("CLEAN_TMP_DIR_PW", "clean_tmp_dir_pw") + .withCommand("--config=/sipi/config/sipi.docker-config.lua") + .withClasspathResourceMapping( + "/sipi.docker-config.lua", + "/sipi/config/sipi.docker-config.lua", + BindMode.READ_ONLY + ) + .withFileSystemBind(imagesVolume.hostPath, imagesDir, BindMode.READ_WRITE) + .withLogConsumer(frame => print("SIPI:" + frame.getUtf8String)) + + private val initSipi = ZLayer.fromZIO( + for { + container <- ZIO.service[SipiTestContainer] + _ <- ZIO.attemptBlocking { + container.execInContainer("mkdir", s"$imagesDir/tmp") + container.execInContainer("chmod", "777", s"$imagesDir/tmp") + } + + } yield container + ) + + val layer: URLayer[SharedVolumes.Images, SipiTestContainer] = { + val container = ZLayer.scoped(ZIO.service[SharedVolumes.Images].flatMap(make(_).toZio)) + (container >>> initSipi).orDie + } } diff --git a/integration/src/test/scala/org/knora/webapi/testcontainers/TestContainerOps.scala b/integration/src/test/scala/org/knora/webapi/testcontainers/TestContainerOps.scala new file mode 100644 index 0000000000..08e48e7cd6 --- /dev/null +++ b/integration/src/test/scala/org/knora/webapi/testcontainers/TestContainerOps.scala @@ -0,0 +1,28 @@ +/* + * Copyright © 2021 - 2023 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.knora.webapi.testcontainers + +import org.testcontainers.lifecycle.Startable +import zio._ + +object ZioTestContainers { + + def toZio[T <: Startable](self: T): URIO[Scope, T] = { // using `logError.ignore` because there's no point to try to recover if starting/stopping the container fails + val acquire = ZIO.attemptBlocking(self.start()).logError.ignore.as(self) + val release = (container: T) => ZIO.attemptBlocking(container.stop()).logError.ignore + ZIO.acquireRelease(acquire)(release) + } + + def toLayer[T <: Startable: Tag](container: T): ULayer[T] = + ZLayer.scoped(toZio(container)) +} + +object TestContainerOps { + implicit final class StartableOps[T <: Startable](private val self: T) extends AnyVal { + def toZio: URIO[Scope, T] = ZioTestContainers.toZio(self) + def toLayer(implicit ev: Tag[T]): ULayer[T] = ZioTestContainers.toLayer(self) + } +} diff --git a/integration/src/test/scala/org/knora/webapi/util/ZioScalaTestUtil.scala b/integration/src/test/scala/org/knora/webapi/util/ZioScalaTestUtil.scala index 8aa4d2b5cb..8b45b3c189 100644 --- a/integration/src/test/scala/org/knora/webapi/util/ZioScalaTestUtil.scala +++ b/integration/src/test/scala/org/knora/webapi/util/ZioScalaTestUtil.scala @@ -1,3 +1,8 @@ +/* + * Copyright © 2021 - 2023 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + package org.knora.webapi.util import org.scalatest.Assertions diff --git a/webapi/src/main/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValueMessagesV2.scala b/webapi/src/main/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValueMessagesV2.scala index 6da0fdd731..22a8cbf68f 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValueMessagesV2.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValueMessagesV2.scala @@ -44,6 +44,7 @@ import org.knora.webapi.messages.v2.responder.resourcemessages.ReadResourceV2 import org.knora.webapi.messages.v2.responder.standoffmessages.* import org.knora.webapi.routing.RouteUtilV2 import org.knora.webapi.routing.RouteUtilZ +import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.AssetId import org.knora.webapi.slice.admin.domain.model.KnoraProject.Shortcode import org.knora.webapi.slice.resourceinfo.domain.IriConverter import org.knora.webapi.store.iiif.api.FileMetadataSipiResponse @@ -1055,7 +1056,6 @@ object ValueContentV2 { /** * Converts a JSON-LD object to a [[ValueContentV2]]. * - * @param ingestState indicates the state of the file, either ingested or in temp folder * @param jsonLdObject the JSON-LD object. * @param requestingUser the user making the request. * @return a [[ValueContentV2]]. @@ -1134,17 +1134,18 @@ object ValueContentV2 { jsonLdObject: JsonLDObject ): ZIO[SipiService, Throwable, FileInfo] = for { - internalFilename <- ZIO.attempt { - val validationFun: (IRI, => Nothing) => IRI = - (s, errorFun) => Iri.toSparqlEncodedString(s).getOrElse(errorFun) - jsonLdObject.requireStringWithValidation(FileValueHasFilename, validationFun) - } + internalFilename <- { + val fileNameEncoded = jsonLdObject + .getRequiredString(FileValueHasFilename) + .flatMap(it => Iri.toSparqlEncodedString(it).toRight(s"$FileValueHasFilename is invalid.")) + ZIO.fromEither(fileNameEncoded).mapError(BadRequestException(_)) + } metadata <- ingestState match { case AssetIngestState.AssetIngested => - SipiService.getFileMetadata(internalFilename, Shortcode.unsafeFrom(shortcode)) - case AssetIngestState.AssetInTemp => SipiService.getFileMetadataFromTemp(internalFilename) + val assetId = AssetId.unsafeFrom(internalFilename.substring(0, internalFilename.indexOf('.'))) + SipiService.getFileMetadataFromDspIngest(Shortcode.unsafeFrom(shortcode), assetId).logError + case AssetIngestState.AssetInTemp => SipiService.getFileMetadataFromSipiTemp(internalFilename) } - } yield FileInfo(internalFilename, metadata) } diff --git a/webapi/src/main/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClient.scala b/webapi/src/main/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClient.scala index 1d0a115cee..ce2c328dc0 100644 --- a/webapi/src/main/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClient.scala +++ b/webapi/src/main/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClient.scala @@ -25,17 +25,22 @@ import zio.http.Headers import zio.http.MediaType import zio.http.Request import zio.http.URL +import zio.json.DecoderOps +import zio.json.DeriveJsonDecoder +import zio.json.JsonDecoder import zio.macros.accessible import zio.nio.file.Files import zio.nio.file.Path import zio.stream.ZSink +import java.io.IOException import java.util.concurrent.TimeUnit import scala.concurrent.duration.DurationInt import org.knora.webapi.config.DspIngestConfig import org.knora.webapi.routing.Jwt import org.knora.webapi.routing.JwtService +import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.AssetId import org.knora.webapi.slice.admin.domain.model.KnoraProject.Shortcode @accessible @@ -43,6 +48,25 @@ trait DspIngestClient { def exportProject(shortcode: Shortcode): ZIO[Scope, Throwable, Path] def importProject(shortcode: Shortcode, fileToImport: Path): Task[Path] + + def getAssetInfo(shortcode: Shortcode, assetId: AssetId): Task[AssetInfoResponse] +} + +final case class AssetInfoResponse( + internalFilename: String, + originalInternalFilename: String, + originalFilename: String, + checksumOriginal: String, + checksumDerivative: String, + width: Option[Int] = None, + height: Option[Int] = None, + duration: Option[Double] = None, + fps: Option[Double] = None, + internalMimeType: Option[String] = None, + originalMimeType: Option[String] = None +) +object AssetInfoResponse { + implicit val decoder: JsonDecoder[AssetInfoResponse] = DeriveJsonDecoder.gen[AssetInfoResponse] } final case class DspIngestClientLive( @@ -67,7 +91,16 @@ final case class DspIngestClientLive( private val authenticatedRequest = getJwtString.map(basicRequest.auth.bearer(_)) - def exportProject(shortcode: Shortcode): ZIO[Scope, Throwable, Path] = + override def getAssetInfo(shortcode: Shortcode, assetId: AssetId): Task[AssetInfoResponse] = + for { + request <- authenticatedRequest.map(_.get(uri"${projectsPath(shortcode)}/assets/$assetId")) + response <- ZIO.blocking(request.send(backend = sttpBackend)).logError + result <- ZIO + .fromEither(response.body.flatMap(str => str.fromJson[AssetInfoResponse])) + .mapError(err => new IOException(s"Error parsing response: $err")) + } yield result + + override def exportProject(shortcode: Shortcode): ZIO[Scope, Throwable, Path] = for { tempDir <- Files.createTempDirectoryScoped(Some("export"), List.empty) exportFile = tempDir / "export.zip" @@ -76,11 +109,11 @@ final case class DspIngestClientLive( .readTimeout(30.minutes) .response(asStreamAlways(ZioStreams)(_.run(ZSink.fromFile(exportFile.toFile)))) } - response <- request.send(backend = sttpBackend) + response <- ZIO.blocking(request.send(backend = sttpBackend)) _ <- ZIO.logInfo(s"Response from ingest :${response.code}") } yield exportFile - def importProject(shortcode: Shortcode, fileToImport: Path): Task[Path] = ZIO.scoped { + override def importProject(shortcode: Shortcode, fileToImport: Path): Task[Path] = ZIO.scoped { for { importUrl <- ZIO.fromEither(URL.decode(s"${projectsPath(shortcode)}/import")) token <- jwtService.createJwtForDspIngest() diff --git a/webapi/src/main/scala/org/knora/webapi/store/iiif/api/SipiService.scala b/webapi/src/main/scala/org/knora/webapi/store/iiif/api/SipiService.scala index 4be4349138..5bfdd1cce3 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/iiif/api/SipiService.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/iiif/api/SipiService.scala @@ -14,6 +14,7 @@ import zio.nio.file.Path import org.knora.webapi.messages.admin.responder.usersmessages.UserADM import org.knora.webapi.messages.store.sipimessages.* import org.knora.webapi.messages.v2.responder.SuccessResponseV2 +import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.AssetId import org.knora.webapi.slice.admin.domain.model.KnoraProject.Shortcode import org.knora.webapi.slice.admin.domain.service.Asset import org.knora.webapi.store.iiif.errors.SipiException @@ -68,16 +69,16 @@ trait SipiService { * @param filename the path to the file. * @return a [[FileMetadataSipiResponse]] containing the requested metadata. */ - def getFileMetadataFromTemp(filename: String): Task[FileMetadataSipiResponse] + def getFileMetadataFromSipiTemp(filename: String): Task[FileMetadataSipiResponse] /** - * Asks Sipi for metadata about a file in permanent location, served from the 'knora.json' route. + * Asks DSP-Ingest for metadata about a file in permanent location, served from the 'knora.json' route. * - * @param filename the path to the file. * @param shortcode the shortcode of the project. + * @param assetId for the file. * @return a [[FileMetadataSipiResponse]] containing the requested metadata. */ - def getFileMetadata(filename: String, shortcode: Shortcode): Task[FileMetadataSipiResponse] + def getFileMetadataFromDspIngest(shortcode: Shortcode, assetId: AssetId): Task[FileMetadataSipiResponse] /** * Asks Sipi to move a file from temporary storage to permanent storage. @@ -111,10 +112,12 @@ trait SipiService { /** * Downloads an asset from Sipi. - * @param asset The asset to download. + * + * @param asset The asset to download. * @param targetDir The target directory in which the asset should be stored. - * @param user The user who is downloading the asset. + * @param user The user who is downloading the asset. * @return The path to the downloaded asset. If the asset could not be downloaded, [[None]] is returned. */ def downloadAsset(asset: Asset, targetDir: Path, user: UserADM): Task[Option[Path]] + } diff --git a/webapi/src/main/scala/org/knora/webapi/store/iiif/impl/SipiServiceLive.scala b/webapi/src/main/scala/org/knora/webapi/store/iiif/impl/SipiServiceLive.scala index 85f50d1c30..e1ece6833d 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/iiif/impl/SipiServiceLive.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/iiif/impl/SipiServiceLive.scala @@ -40,8 +40,10 @@ import org.knora.webapi.messages.util.KnoraSystemInstances import org.knora.webapi.messages.v2.responder.SuccessResponseV2 import org.knora.webapi.routing.Jwt import org.knora.webapi.routing.JwtService -import org.knora.webapi.slice.admin.domain.model.KnoraProject +import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.AssetId +import org.knora.webapi.slice.admin.domain.model.KnoraProject.Shortcode import org.knora.webapi.slice.admin.domain.service.Asset +import org.knora.webapi.slice.admin.domain.service.DspIngestClient import org.knora.webapi.store.iiif.api.FileMetadataSipiResponse import org.knora.webapi.store.iiif.api.SipiService import org.knora.webapi.store.iiif.errors.SipiException @@ -58,7 +60,8 @@ import org.knora.webapi.util.ZScopedJavaIoStreams final case class SipiServiceLive( private val sipiConfig: Sipi, private val jwtService: JwtService, - private val httpClient: CloseableHttpClient + private val httpClient: CloseableHttpClient, + private val dspIngestClient: DspIngestClient ) extends SipiService { private object SipiRoutes { @@ -75,23 +78,10 @@ final case class SipiServiceLive( * @param filename the file name * @return a [[FileMetadataSipiResponse]] containing the requested metadata. */ - override def getFileMetadataFromTemp(filename: String): Task[FileMetadataSipiResponse] = - getFileMetadataFromUrl(s"${sipiConfig.internalBaseUrl}/tmp/$filename/knora.json") - - /** - * Asks Sipi for metadata about a file in permanent location, served from the 'knora.json' route. - * - * @param filename the path to the file. - * @param shortcode the shortcode of the project. - * @return a [[FileMetadataSipiResponse]] containing the requested metadata. - */ - override def getFileMetadata(filename: String, shortcode: KnoraProject.Shortcode): Task[FileMetadataSipiResponse] = - getFileMetadataFromUrl(s"${sipiConfig.internalBaseUrl}/${shortcode.value}/$filename/knora.json") - - private def getFileMetadataFromUrl(url: String): Task[FileMetadataSipiResponse] = + override def getFileMetadataFromSipiTemp(filename: String): Task[FileMetadataSipiResponse] = for { jwt <- jwtService.createJwt(KnoraSystemInstances.Users.SystemUser) - request = new HttpGet(url) + request = new HttpGet(s"${sipiConfig.internalBaseUrl}/tmp/$filename/knora.json") _ = request.addHeader(new BasicHeader("Authorization", s"Bearer ${jwt.jwtString}")) bodyStr <- doSipiRequest(request) res <- ZIO @@ -99,6 +89,20 @@ final case class SipiServiceLive( .mapError(e => SipiException(s"Invalid response from Sipi: $e, $bodyStr")) } yield res + override def getFileMetadataFromDspIngest(shortcode: Shortcode, assetId: AssetId): Task[FileMetadataSipiResponse] = + for { + response <- dspIngestClient.getAssetInfo(shortcode, assetId) + } yield FileMetadataSipiResponse( + Some(response.originalFilename), + response.originalMimeType, + response.internalMimeType.getOrElse("application/octet-stream"), + response.width, + response.height, + None, + response.duration.map(BigDecimal(_)), + response.fps.map(BigDecimal(_)) + ) + /** * Asks Sipi to move a file from temporary storage to permanent storage. * @@ -408,12 +412,13 @@ object SipiServiceLive { private def release(httpClient: CloseableHttpClient): UIO[Unit] = ZIO.attemptBlocking(httpClient.close()).logError.ignore <* ZIO.logInfo(">>> Release Sipi IIIF Service <<<") - val layer: URLayer[AppConfig & JwtService, SipiService] = + val layer: URLayer[AppConfig & DspIngestClient & JwtService, SipiService] = ZLayer.scoped { for { - config <- ZIO.serviceWith[AppConfig](_.sipi) - jwtService <- ZIO.service[JwtService] - httpClient <- ZIO.acquireRelease(acquire(config))(release) - } yield SipiServiceLive(config, jwtService, httpClient) + config <- ZIO.serviceWith[AppConfig](_.sipi) + jwtService <- ZIO.service[JwtService] + httpClient <- ZIO.acquireRelease(acquire(config))(release) + dspIngestClient <- ZIO.service[DspIngestClient] + } yield SipiServiceLive(config, jwtService, httpClient, dspIngestClient) } } diff --git a/webapi/src/test/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValueContentV2Spec.scala b/webapi/src/test/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValueContentV2Spec.scala new file mode 100644 index 0000000000..4e5a0ed9dc --- /dev/null +++ b/webapi/src/test/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValueContentV2Spec.scala @@ -0,0 +1,103 @@ +/* + * Copyright © 2021 - 2023 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.knora.webapi.messages.v2.responder.valuemessages + +import zio.Task +import zio.ZIO +import zio.ZLayer +import zio.nio.file.Path +import zio.test.Assertion.failsWithA +import zio.test.Spec +import zio.test.ZIOSpecDefault +import zio.test.assert +import zio.test.assertTrue + +import dsp.errors.AssertionException +import org.knora.webapi.messages.admin.responder.usersmessages.UserADM +import org.knora.webapi.messages.store.sipimessages.DeleteTemporaryFileRequest +import org.knora.webapi.messages.store.sipimessages.IIIFServiceStatusResponse +import org.knora.webapi.messages.store.sipimessages.MoveTemporaryFileToPermanentStorageRequest +import org.knora.webapi.messages.store.sipimessages.SipiGetTextFileRequest +import org.knora.webapi.messages.store.sipimessages.SipiGetTextFileResponse +import org.knora.webapi.messages.util.rdf.JsonLDUtil +import org.knora.webapi.messages.v2.responder.SuccessResponseV2 +import org.knora.webapi.messages.v2.responder.resourcemessages.CreateResourceRequestV2.AssetIngestState +import org.knora.webapi.messages.v2.responder.resourcemessages.CreateResourceRequestV2.AssetIngestState.AssetInTemp +import org.knora.webapi.messages.v2.responder.resourcemessages.CreateResourceRequestV2.AssetIngestState.AssetIngested +import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.AssetId +import org.knora.webapi.slice.admin.domain.model.KnoraProject +import org.knora.webapi.slice.admin.domain.service.Asset +import org.knora.webapi.store.iiif.api.FileMetadataSipiResponse +import org.knora.webapi.store.iiif.api.SipiService + +object ValueContentV2Spec extends ZIOSpecDefault { + + private val assetId = AssetId.unsafeFrom("4sAf4AmPeeg-ZjDn3Tot1Zt") + + private val jsonLdObj = JsonLDUtil + .parseJsonLD(s"{\"http://api.knora.org/ontology/knora-api/v2#fileValueHasFilename\" : \"$assetId.txt\"}") + .body + + private val expected = FileMetadataSipiResponse(Some("origName"), None, "text/plain", None, None, None, None, None) + + override def spec: Spec[Any, Throwable] = + suite("ValueContentV2.getFileInfo")( + suite("Given the asset is present in the tmp folder of Sipi")( + test("When getting file metadata with AssetInTemp from Sipi, then it should succeed") { + for { + temp <- ValueContentV2.getFileInfo("0001", AssetInTemp, jsonLdObj) + } yield assertTrue(temp.metadata == expected) + }, + test("When getting file metadata with AssetIngested from dsp-ingest, then it should fail") { + for { + exit <- ValueContentV2.getFileInfo("0001", AssetIngested, jsonLdObj).exit + } yield assert(exit)(failsWithA[AssertionException]) + } + ).provide(mockSipi(AssetInTemp)), + suite("Given the asset is ingested")( + test("When getting file metadata with AssetInTemp from Sipi, then it should fail") { + for { + exit <- ValueContentV2.getFileInfo("0001", AssetInTemp, jsonLdObj).exit + } yield assert(exit)(failsWithA[AssertionException]) + }, + test("When getting file metadata with AssetIngested from dsp-ingest, then it should succeed") { + for { + ingested <- ValueContentV2.getFileInfo("0001", AssetIngested, jsonLdObj) + } yield assertTrue(ingested.metadata == expected) + } + ).provide(mockSipi(AssetIngested)) + ) + + private def mockSipi(flag: AssetIngestState) = ZLayer.succeed(new SipiService { + + override def getFileMetadataFromSipiTemp(filename: String): Task[FileMetadataSipiResponse] = + if (flag == AssetInTemp) { ZIO.succeed(expected) } + else { ZIO.fail(AssertionException("fail")) } + + override def getFileMetadataFromDspIngest( + shortcode: KnoraProject.Shortcode, + assetId: AssetId + ): Task[FileMetadataSipiResponse] = + if (flag == AssetIngested) { ZIO.succeed(expected) } + else { ZIO.fail(AssertionException("fail")) } + + // The following are unsupported operations because they are not used in the test + def moveTemporaryFileToPermanentStorage( + moveTemporaryFileToPermanentStorageRequestV2: MoveTemporaryFileToPermanentStorageRequest + ): Task[SuccessResponseV2] = + ZIO.dieMessage("unsupported operation") + def deleteTemporaryFile( + deleteTemporaryFileRequestV2: DeleteTemporaryFileRequest + ): Task[SuccessResponseV2] = + ZIO.dieMessage("unsupported operation") + def getTextFileRequest(textFileRequest: SipiGetTextFileRequest): Task[SipiGetTextFileResponse] = + ZIO.dieMessage("unsupported operation") + def getStatus(): Task[IIIFServiceStatusResponse] = + ZIO.dieMessage("unsupported operation") + def downloadAsset(asset: Asset, targetDir: Path, user: UserADM): Task[Option[Path]] = + ZIO.dieMessage("unsupported operation") + }) +} diff --git a/webapi/src/test/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValuesV2Spec.scala b/webapi/src/test/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValuesV2Spec.scala deleted file mode 100644 index ba04e8071c..0000000000 --- a/webapi/src/test/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValuesV2Spec.scala +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright © 2021 - 2023 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.knora.webapi.messages.v2.responder.valuemessages - -import zio.* -import zio.nio.file.Path -import zio.test.Assertion.* -import zio.test.* - -import dsp.errors.AssertionException -import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.store.sipimessages.* -import org.knora.webapi.messages.util.rdf.JsonLDUtil -import org.knora.webapi.messages.v2.responder.SuccessResponseV2 -import org.knora.webapi.messages.v2.responder.resourcemessages.CreateResourceRequestV2.AssetIngestState -import org.knora.webapi.slice.admin.domain.model.KnoraProject -import org.knora.webapi.slice.admin.domain.service.Asset -import org.knora.webapi.store.iiif.api.FileMetadataSipiResponse -import org.knora.webapi.store.iiif.api.SipiService - -object ValuesV2Spec extends ZIOSpecDefault { - private val json = - """{ - | "http://api.knora.org/ontology/knora-api/v2#fileValueHasFilename" : "filename" - | }""".stripMargin - - override def spec: Spec[TestEnvironment with Scope, Any] = - suite("ValuesV2")( - test("Expect file to be not ingested and in `tmp` folder") { - for { - ingested <- ValueContentV2 - .getFileInfo("0001", AssetIngestState.AssetIngested, JsonLDUtil.parseJsonLD(json).body) - .exit - temp <- ValueContentV2 - .getFileInfo("0001", AssetIngestState.AssetInTemp, JsonLDUtil.parseJsonLD(json).body) - .exit - } yield assert(ingested)(fails(isSubtype[AssertionException](anything))) && assert(temp)(succeeds(anything)) - }.provide(sipiServiceAllowMetadataInTemp), - test("Expect file to be already ingested and in project folder") { - for { - ingested <- ValueContentV2 - .getFileInfo("0001", AssetIngestState.AssetIngested, JsonLDUtil.parseJsonLD(json).body) - .exit - temp <- ValueContentV2 - .getFileInfo("0001", AssetIngestState.AssetInTemp, JsonLDUtil.parseJsonLD(json).body) - .exit - } yield assert(ingested)(succeeds(anything)) && assert(temp)(fails(isSubtype[AssertionException](anything))) - }.provide(sipiServiceAllowMetadataInProjectFolder) - ) - - private val response: FileMetadataSipiResponse = - FileMetadataSipiResponse(None, None, "", None, None, None, None, None) - - private def sipiServiceAllowMetadataInTemp: ULayer[SipiService] = ZLayer.succeed(new SipiService { - override def getFileMetadataFromTemp(filename: String): Task[FileMetadataSipiResponse] = - ZIO.succeed(response) - override def getFileMetadata(filename: String, shortcode: KnoraProject.Shortcode): Task[FileMetadataSipiResponse] = - ZIO.fail(AssertionException(filename)) - def moveTemporaryFileToPermanentStorage( - moveTemporaryFileToPermanentStorageRequestV2: MoveTemporaryFileToPermanentStorageRequest - ): Task[SuccessResponseV2] = ??? - def deleteTemporaryFile( - deleteTemporaryFileRequestV2: DeleteTemporaryFileRequest - ): Task[SuccessResponseV2] = ??? - def getTextFileRequest(textFileRequest: SipiGetTextFileRequest): Task[SipiGetTextFileResponse] = ??? - def getStatus(): Task[IIIFServiceStatusResponse] = ??? - def downloadAsset(asset: Asset, targetDir: Path, user: UserADM): Task[Option[Path]] = ??? - }) - - private def sipiServiceAllowMetadataInProjectFolder: ULayer[SipiService] = ZLayer.succeed(new SipiService { - override def getFileMetadata(filename: String, shortcode: KnoraProject.Shortcode): Task[FileMetadataSipiResponse] = - ZIO.succeed(response) - override def getFileMetadataFromTemp(filename: String): Task[FileMetadataSipiResponse] = - ZIO.fail(AssertionException(filename)) - def moveTemporaryFileToPermanentStorage( - moveTemporaryFileToPermanentStorageRequestV2: MoveTemporaryFileToPermanentStorageRequest - ): Task[SuccessResponseV2] = ??? - def deleteTemporaryFile( - deleteTemporaryFileRequestV2: DeleteTemporaryFileRequest - ): Task[SuccessResponseV2] = ??? - def getTextFileRequest(textFileRequest: SipiGetTextFileRequest): Task[SipiGetTextFileResponse] = ??? - def getStatus(): Task[IIIFServiceStatusResponse] = ??? - def downloadAsset(asset: Asset, targetDir: Path, user: UserADM): Task[Option[Path]] = ??? - }) - -} diff --git a/webapi/src/test/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClientLiveSpec.scala b/webapi/src/test/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClientLiveSpec.scala index ac8ea668fe..57d514937d 100644 --- a/webapi/src/test/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClientLiveSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClientLiveSpec.scala @@ -6,22 +6,36 @@ package org.knora.webapi.slice.admin.domain.service import com.github.tomakehurst.wiremock.WireMockServer +import com.github.tomakehurst.wiremock.client.CountMatchingStrategy +import com.github.tomakehurst.wiremock.client.MappingBuilder +import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder import com.github.tomakehurst.wiremock.client.WireMock import com.github.tomakehurst.wiremock.client.WireMock.aResponse import com.github.tomakehurst.wiremock.client.WireMock.equalTo +import com.github.tomakehurst.wiremock.client.WireMock.get +import com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor +import com.github.tomakehurst.wiremock.client.WireMock.post import com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor +import com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo import com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo import com.github.tomakehurst.wiremock.core.WireMockConfiguration.options +import com.github.tomakehurst.wiremock.matching.RequestPatternBuilder import spray.json.JsValue +import zio.Console import zio.Random import zio.Scope import zio.Task import zio.UIO import zio.ULayer +import zio.URIO import zio.ZIO import zio.ZLayer +import zio.json.DeriveJsonEncoder +import zio.json.EncoderOps +import zio.json.JsonEncoder import zio.nio.file.Files import zio.test.Spec +import zio.test.TestAspect import zio.test.TestEnvironment import zio.test.ZIOSpecDefault import zio.test.assertTrue @@ -31,94 +45,155 @@ import org.knora.webapi.config.DspIngestConfig import org.knora.webapi.messages.admin.responder.usersmessages.UserADM import org.knora.webapi.routing.Jwt import org.knora.webapi.routing.JwtService +import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.AssetId import org.knora.webapi.slice.admin.domain.model.KnoraProject.Shortcode import org.knora.webapi.slice.admin.domain.service.DspIngestClientLiveSpecLayers.dspIngestConfigLayer -import org.knora.webapi.slice.admin.domain.service.DspIngestClientLiveSpecLayers.mockJwtServiceLayer -import org.knora.webapi.slice.admin.domain.service.DspIngestClientLiveSpecLayers.testPortLayer -import org.knora.webapi.slice.admin.domain.service.DspIngestClientLiveSpecLayers.wireMockServerLayer +import org.knora.webapi.slice.admin.domain.service.DspIngestClientLiveSpecLayers.jwtServiceMockLayer +import org.knora.webapi.slice.admin.domain.service.HttpMockServer.TestPort object DspIngestClientLiveSpec extends ZIOSpecDefault { - private val testShortCodeStr = "0001" - private val testProject = Shortcode.unsafeFrom(testShortCodeStr) + private val testShortcodeStr = "0001" + private val testShortcode = Shortcode.unsafeFrom(testShortcodeStr) private val testContent = "testContent".getBytes() - private val expectedPath = s"/projects/$testShortCodeStr/export" private val exportProjectSuite = suite("exportProject")(test("should download a project export") { - ZIO.scoped { - for { - // given - wiremock <- ZIO.service[WireMockServer] - _ = wiremock.stubFor( - WireMock - .post(urlPathEqualTo(expectedPath)) - .willReturn( - aResponse() - .withHeader("Content-Type", "application/zip") - .withHeader("Content-Disposition", s"export-$testShortCodeStr.zip") - .withBody(testContent) - .withStatus(200) - ) - ) - mockJwt <- JwtService.createJwtForDspIngest() - - // when - path <- DspIngestClient.exportProject(testProject) - - // then - _ = wiremock.verify( - postRequestedFor(urlPathEqualTo(expectedPath)) - .withHeader("Authorization", equalTo(s"Bearer ${mockJwt.jwtString}")) - ) - contentIsDownloaded <- Files.readAllBytes(path).map(_.toArray).map(_ sameElements testContent) - } yield assertTrue(contentIsDownloaded) - } + val expectedUrl = s"/projects/$testShortcodeStr/export" + for { + // given + _ <- HttpMockServer.stub.postResponse( + expectedUrl, + aResponse() + .withHeader("Content-Type", "application/zip") + .withHeader("Content-Disposition", s"export-$testShortcodeStr.zip") + .withBody(testContent) + .withStatus(200) + ) + + // when + path <- DspIngestClient.exportProject(testShortcode) + + // then + mockJwt <- JwtService.createJwtForDspIngest().map(_.jwtString) + _ <- HttpMockServer.verify.request( + postRequestedFor(urlPathEqualTo(expectedUrl)) + .withHeader("Authorization", equalTo(s"Bearer $mockJwt")) + ) + contentIsDownloaded <- Files.readAllBytes(path).map(_.toArray).map(_ sameElements testContent) + } yield assertTrue(contentIsDownloaded) + }) + + private val getAssetInfoSuite = suite("getAssetInfo")(test("should return the assetInfo") { + implicit val encoder: JsonEncoder[AssetInfoResponse] = DeriveJsonEncoder.gen[AssetInfoResponse] + val assetId = AssetId.unsafeFrom("4sAf4AmPeeg-ZjDn3Tot1Zt") + val expectedUrl = s"/projects/$testShortcodeStr/assets/$assetId" + val expected = AssetInfoResponse( + internalFilename = s"$assetId.txt", + originalInternalFilename = s"$assetId.txt.orig", + originalFilename = "test.txt", + checksumOriginal = "bfd3192ea04d5f42d79836cf3b8fbf17007bab71", + checksumDerivative = "17bab70071fbf8b3fc63897d24f5d40ae2913dfb", + internalMimeType = Some("text/plain"), + originalMimeType = Some("text/plain") + ) + for { + // given + _ <- HttpMockServer.stub.getResponseJsonBody(expectedUrl, 200, expected) + + // when + assetInfo <- DspIngestClient.getAssetInfo(testShortcode, assetId) + + // then + mockJwt <- JwtService.createJwtForDspIngest().map(_.jwtString) + _ <- HttpMockServer.verify.request( + getRequestedFor(urlPathEqualTo(expectedUrl)) + .withHeader("Authorization", equalTo(s"Bearer $mockJwt")) + ) + } yield assertTrue(assetInfo == expected) }) override def spec: Spec[TestEnvironment & Scope, Any] = - suite("DspIngestClientLive")(exportProjectSuite).provide( + suite("DspIngestClientLive")( + exportProjectSuite, + getAssetInfoSuite + ).provideSome[Scope]( DspIngestClientLive.layer, + HttpMockServer.layer, + TestPort.random, dspIngestConfigLayer, - mockJwtServiceLayer, - testPortLayer, - wireMockServerLayer - ) + jwtServiceMockLayer + ) @@ TestAspect.sequential } object DspIngestClientLiveSpecLayers { - val mockJwtServiceLayer: ULayer[JwtService] = ZLayer.succeed(new JwtService { - override def createJwt(user: UserADM, content: Map[String, JsValue]): UIO[Jwt] = - throw new UnsupportedOperationException("not implemented") - override def createJwtForDspIngest(): UIO[Jwt] = ZIO.succeed(Jwt("mock-jwt-string-value", Long.MaxValue)) - override def validateToken(token: String): Task[Boolean] = - throw new UnsupportedOperationException("not implemented") - override def extractUserIriFromToken(token: String): Task[Option[IRI]] = - throw new UnsupportedOperationException("not implemented") - }) - case class Testport(port: Int) extends AnyVal - - val testPortLayer: ULayer[Testport] = ZLayer.fromZIO(for { - port <- Random.nextIntBetween(1000, 10_000) - } yield Testport(port)) + val jwtServiceMockLayer: ULayer[JwtService] = ZLayer.succeed { + val unsupported = ZIO.die(new UnsupportedOperationException("not implemented")) + new JwtService { + override def createJwtForDspIngest(): UIO[Jwt] = ZIO.succeed(Jwt("mock-jwt-string-value", Long.MaxValue)) + override def createJwt(user: UserADM, content: Map[String, JsValue]): UIO[Jwt] = unsupported + override def validateToken(token: String): Task[Boolean] = unsupported + override def extractUserIriFromToken(token: String): Task[Option[IRI]] = unsupported + } + } - val dspIngestConfigLayer: ZLayer[Testport, Nothing, DspIngestConfig] = ZLayer.fromZIO( + val dspIngestConfigLayer: ZLayer[TestPort, Nothing, DspIngestConfig] = ZLayer.fromZIO( ZIO - .serviceWith[Testport](_.port) + .serviceWith[TestPort](_.value) .map(port => DspIngestConfig(baseUrl = s"http://localhost:$port", audience = "audience")) ) +} - private def acquireWireMockServer(port: Int): Task[WireMockServer] = ZIO.attempt { - val server = new WireMockServer(options().port(port)) - server.start() - server +object HttpMockServer { + object verify { + def request( + requestPattern: RequestPatternBuilder, + amount: CountMatchingStrategy = WireMock.exactly(1) + ): ZIO[WireMockServer, Throwable, Unit] = ZIO.serviceWithZIO[WireMockServer](server => + ZIO.attempt(server.verify(amount, requestPattern)).tapError { e => + Console.printLine(s"\nMockDspApiServer: ${e.getMessage}") + } + ) } - private def releaseWireMockServer(server: WireMockServer) = ZIO.attempt(server.stop()).logError.ignore - val wireMockServerLayer: ZLayer[Testport, Throwable, WireMockServer] = - ZLayer.scoped { - for { - port <- ZIO.serviceWith[Testport](_.port) - server <- ZIO.acquireRelease(acquireWireMockServer(port))(releaseWireMockServer) - } yield server + object stub { + def getResponseJsonBody[A](url: String, status: Int, body: A)(implicit + encoder: JsonEncoder[A] + ): URIO[WireMockServer, WireMockServer] = + getResponse( + url, + aResponse().withStatus(status).withBody(body.toJson).withHeader("Content-Type", "application/json") + ) + + def getResponse(url: String, response: ResponseDefinitionBuilder): URIO[WireMockServer, WireMockServer] = + resetAndStubServer(get(urlEqualTo(url)).willReturn(response)) + + def postResponse(url: String, response: ResponseDefinitionBuilder): URIO[WireMockServer, WireMockServer] = + resetAndStubServer(post(urlEqualTo(url)).willReturn(response)) + + private def resetAndStubServer(mappingBuilder: MappingBuilder) = + resetAndGetWireMockServer.tap(server => ZIO.succeed(server.stubFor(mappingBuilder))) + private def resetAndGetWireMockServer: URIO[WireMockServer, WireMockServer] = + ZIO.serviceWith[WireMockServer] { it => + it.resetAll(); + it + } + } + case class TestPort(value: Int) extends AnyVal + object TestPort { + val random: ULayer[TestPort] = ZLayer.fromZIO(Random.nextIntBetween(1000, 10_000).map(TestPort.apply)) + } + + private def acquireWireMockServer: ZIO[TestPort, Throwable, WireMockServer] = + ZIO.serviceWith[TestPort](_.value).flatMap { port => + ZIO.attempt { + val server = new WireMockServer(options().port(port)); // No-args constructor will start on port 8080, no HTTPS + server.start() + server + } } + + private def releaseWireMockServer(server: WireMockServer) = ZIO.attempt(server.stop()).logError.ignore + + val layer: ZLayer[Scope & TestPort, Throwable, WireMockServer] = + ZLayer.fromZIO(ZIO.acquireRelease(acquireWireMockServer)(releaseWireMockServer)) } diff --git a/webapi/src/test/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClientMock.scala b/webapi/src/test/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClientMock.scala index 5820375a08..98c06f3ae6 100644 --- a/webapi/src/test/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClientMock.scala +++ b/webapi/src/test/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClientMock.scala @@ -11,14 +11,29 @@ import zio.ZIO import zio.ZLayer import zio.nio.file.Path +import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.AssetId import org.knora.webapi.slice.admin.domain.model.KnoraProject +final case class DspIngestClientMock() extends DspIngestClient { + override def exportProject(shortcode: KnoraProject.Shortcode): ZIO[Scope, Throwable, Path] = + ZIO.succeed(Path("/tmp/test.zip")) + override def importProject(shortcode: KnoraProject.Shortcode, fileToImport: Path): Task[Path] = + ZIO.succeed(Path("/tmp/test.zip")) + + override def getAssetInfo(shortcode: KnoraProject.Shortcode, assetId: AssetId): Task[AssetInfoResponse] = + ZIO.succeed( + AssetInfoResponse( + s"$assetId.txt", + s"$assetId.txt.orig", + "test.txt", + "bfd3192ea04d5f42d79836cf3b8fbf17007bab71", + "17bab70071fbf8b3fc63897d24f5d40ae2913dfb", + internalMimeType = Some("text/plain"), + originalMimeType = Some("text/plain") + ) + ) +} + object DspIngestClientMock { - final case class MockDspIngestClient() extends DspIngestClient { - override def exportProject(shortcode: KnoraProject.Shortcode): ZIO[Scope, Throwable, Path] = - ZIO.succeed(Path("/tmp/test.zip")) - override def importProject(shortcode: KnoraProject.Shortcode, fileToImport: Path): Task[Path] = - ZIO.succeed(Path("/tmp/test.zip")) - } - val layer = ZLayer.fromFunction(MockDspIngestClient.apply _) + val layer = ZLayer.derive[DspIngestClientMock] }