feat: Add dsp ingest asset info resolution (DEV-3147) (#2973)

Co-authored-by: Balduin Landolt <33053745+BalduinLandolt@users.noreply.github.com> Co-authored-by: Marcin Procyk <marcin.procyk@dasch.swiss>
dasch-swiss · Jan 8, 2024 · c081ba8 · c081ba8
1 parent 779fddb
commit c081ba8
Show file tree

Hide file tree

Showing 27 changed files with 653 additions and 385 deletions.
diff --git a/integration/src/test/resources/sipi/testfiles/B1D0OkEgfFp-Cew2Seur7Wi.jp2 b/integration/src/test/resources/sipi/testfiles/B1D0OkEgfFp-Cew2Seur7Wi.jp2
diff --git a/integration/src/test/resources/sipi/testfiles/De6XyNL4H71-D9QxghOuOPJ.info b/integration/src/test/resources/sipi/testfiles/De6XyNL4H71-D9QxghOuOPJ.info
@@ -3,5 +3,9 @@
    "internalFilename": "De6XyNL4H71-D9QxghOuOPJ.jp2",
    "checksumOriginal": "710aa7c06f7a7293f4cab70855ab98e166762a6b3ab7b6bdb4f6750bfc7028ed",
    "originalFilename": "dog.png",
-   "checksumDerivative": "a89a2522adefe4510016ebdf337d82d3af0eef2a5c008c8c76553db4adb64155"
+   "checksumDerivative": "a89a2522adefe4510016ebdf337d82d3af0eef2a5c008c8c76553db4adb64155",
+   "width": 72,
+   "height": 72,
+   "internalMimeType": "image/jpx",
+   "originalMimeType": "image/png"
 }
diff --git a/integration/src/test/resources/sipi/testfiles/FGiLaT4zzuV-CqwbEDFAFeS.info b/integration/src/test/resources/sipi/testfiles/FGiLaT4zzuV-CqwbEDFAFeS.info
@@ -3,5 +3,9 @@
    "checksumDerivative": "0ce405c9b183fb0d0a9998e9a49e39c93b699e0f8e2a9ac3496c349e5cea09cc",
    "internalFilename": "FGiLaT4zzuV-CqwbEDFAFeS.jp2",
    "checksumOriginal": "fb252a4fb3d90ce4ebc7e123d54a4112398a7994541b11aab5e4230eac01a61c",
-   "originalFilename": "250x250.jp2"
+   "originalFilename": "250x250.jp2",
+   "width": 250,
+   "height": 250,
+   "internalMimeType": "image/jpx",
+   "originalMimeType": "image/jp2"
 }
diff --git a/integration/src/test/scala/org/knora/sipi/SipiIT.scala b/integration/src/test/scala/org/knora/sipi/SipiIT.scala
@@ -28,6 +28,7 @@ import org.knora.webapi.messages.admin.responder.sipimessages._
 import org.knora.webapi.messages.util.KnoraSystemInstances.Users.SystemUser
 import org.knora.webapi.routing.JwtService
 import org.knora.webapi.routing.JwtServiceLive
+import org.knora.webapi.testcontainers.SharedVolumes
 import org.knora.webapi.testcontainers.SipiTestContainer
 
 object SipiIT extends ZIOSpecDefault {
@@ -36,9 +37,6 @@ object SipiIT extends ZIOSpecDefault {
   private val infoTestfile  = "FGiLaT4zzuV-CqwbEDFAFeS.info"
   private val origTestfile  = "FGiLaT4zzuV-CqwbEDFAFeS.jp2.orig"
   private val prefix        = "0001"
-  private def copyTestFilesToSipi = ZIO.foreach(List(imageTestfile, infoTestfile, origTestfile))(
-    SipiTestContainer.copyFileToImageFolderInContainer(prefix, _)
-  )
 
   private def getWithoutAuthorization(path: Path) =
     SipiTestContainer
@@ -62,7 +60,6 @@ object SipiIT extends ZIOSpecDefault {
       ) {
         for {
           jwt <- getToken
-          _   <- copyTestFilesToSipi
           _   <- MockDspApiServer.resetAndAllowWithPermissionCode(prefix, imageTestfile, 2)
           response <-
             SipiTestContainer
@@ -96,7 +93,6 @@ object SipiIT extends ZIOSpecDefault {
       ) {
         for {
           jwt <- getToken
-          _   <- copyTestFilesToSipi
           _   <- MockDspApiServer.resetAndAllowWithPermissionCode(prefix, imageTestfile, 2)
           response <-
             SipiTestContainer
@@ -139,7 +135,6 @@ object SipiIT extends ZIOSpecDefault {
                  |}""".stripMargin.fromJson[Json]
             for {
               _        <- MockDspApiServer.resetAndAllowWithPermissionCode(prefix, imageTestfile, permissionCode = 2)
-              _        <- copyTestFilesToSipi
               response <- getWithoutAuthorization(Root / prefix / imageTestfile / "knora.json")
               json     <- response.body.asString.map(_.fromJson[Json])
               expected <- SipiTestContainer.portAndHost.map { case (port, host) => expectedJson(port, host) }
@@ -173,7 +168,6 @@ object SipiIT extends ZIOSpecDefault {
             val dspApiPermissionPath = s"/admin/files/$prefix/$imageTestfile"
             for {
               server   <- MockDspApiServer.resetAndStubGetResponse(dspApiPermissionPath, 200, dspApiResponse)
-              _        <- copyTestFilesToSipi
               response <- getWithoutAuthorization(Root / prefix / imageTestfile / "file")
             } yield assertTrue(
               response.status == Status.Ok,
@@ -189,7 +183,6 @@ object SipiIT extends ZIOSpecDefault {
             val dspApiPermissionPath = s"/admin/files/$prefix/$imageTestfile"
             for {
               server   <- MockDspApiServer.resetAndStubGetResponse(dspApiPermissionPath, 200, dspApiResponse)
-              _        <- copyTestFilesToSipi
               response <- getWithoutAuthorization(Root / prefix / imageTestfile / "file")
             } yield assertTrue(
               response.status == Status.Unauthorized,
@@ -204,7 +197,6 @@ object SipiIT extends ZIOSpecDefault {
             val dspApiPermissionPath = s"/admin/files/$prefix/$imageTestfile"
             for {
               server   <- MockDspApiServer.resetAndStubGetResponse(dspApiPermissionPath, 404)
-              _        <- copyTestFilesToSipi
               response <- getWithoutAuthorization(Root / prefix / imageTestfile / "file")
             } yield assertTrue(
               response.status == Status.NotFound,
@@ -239,7 +231,6 @@ object SipiIT extends ZIOSpecDefault {
           val dspApiPermissionPath = s"/admin/files/$prefix/$imageTestfile"
           for {
             server   <- MockDspApiServer.resetAndStubGetResponse(dspApiPermissionPath, 200, dspApiResponse)
-            _        <- copyTestFilesToSipi
             response <- getWithoutAuthorization(Root / prefix / imageTestfile / "full/max/0/default.jp2")
           } yield assertTrue(
             response.status == Status.Ok,
@@ -255,7 +246,6 @@ object SipiIT extends ZIOSpecDefault {
           val dspApiPermissionPath = s"/admin/files/$prefix/$imageTestfile"
           for {
             server   <- MockDspApiServer.resetAndStubGetResponse(dspApiPermissionPath, 200, dspApiResponse)
-            _        <- copyTestFilesToSipi
             response <- getWithoutAuthorization(Root / prefix / imageTestfile / "full/max/0/default.jp2")
           } yield assertTrue(
             response.status == Status.Unauthorized,
@@ -270,7 +260,6 @@ object SipiIT extends ZIOSpecDefault {
           val dspApiPermissionPath = s"/admin/files/$prefix/$imageTestfile"
           for {
             server   <- MockDspApiServer.resetAndStubGetResponse(dspApiPermissionPath, 404)
-            _        <- copyTestFilesToSipi
             response <- getWithoutAuthorization(Root / prefix / imageTestfile / "full/max/0/default.jp2")
           } yield assertTrue(
             response.status == Status.NotFound,
@@ -293,7 +282,9 @@ object SipiIT extends ZIOSpecDefault {
         } yield assertTrue(response.status.isSuccess, verifyNoInteractionWith(server))
       }
     )
-      .provideSomeLayerShared[Scope with Client with WireMockServer](SipiTestContainer.layer)
+      .provideSomeLayerShared[Scope with Client with WireMockServer](
+        SharedVolumes.Images.layer >+> SipiTestContainer.layer
+      )
       .provideSomeLayerShared[Scope with Client](MockDspApiServer.layer)
       .provideSomeLayer[Scope](Client.default) @@ TestAspect.sequential
 }

diff --git a/integration/src/test/scala/org/knora/webapi/ITKnoraLiveSpec.scala b/integration/src/test/scala/org/knora/webapi/ITKnoraLiveSpec.scala
@@ -8,6 +8,9 @@ package org.knora.webapi
 import com.typesafe.scalalogging.LazyLogging
 import com.typesafe.scalalogging.Logger
 import org.apache.pekko
+import org.apache.pekko.http.scaladsl.client.RequestBuilding
+import org.apache.pekko.http.scaladsl.model._
+import org.apache.pekko.testkit.TestKitBase
 import org.scalatest.BeforeAndAfterAll
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
@@ -31,15 +34,10 @@ import org.knora.webapi.messages.store.triplestoremessages.TriplestoreJsonProtoc
 import org.knora.webapi.messages.util.rdf.JsonLDDocument
 import org.knora.webapi.messages.util.rdf.JsonLDUtil
 import org.knora.webapi.routing.UnsafeZioRun
-import org.knora.webapi.testcontainers.SipiTestContainer
 import org.knora.webapi.testservices.FileToUpload
 import org.knora.webapi.testservices.TestClientService
 import org.knora.webapi.util.LogAspect
 
-import pekko.http.scaladsl.client.RequestBuilding
-import pekko.http.scaladsl.model._
-import pekko.testkit.TestKitBase
-
 /**
  * This class can be used in End-to-End testing. It starts the DSP stack and
  * provides access to configuration and logging.
@@ -83,9 +81,6 @@ abstract class ITKnoraLiveSpec
     config <- ZIO.service[AppConfig]
   } yield (router, config)
 
-  def copyFileToImageFolderInContainer(prefix: String, filename: String) =
-    UnsafeZioRun.runOrThrow(SipiTestContainer.copyFileToImageFolderInContainer(prefix, filename))
-
   /**
    * Create router and config by unsafe running them.
    */

diff --git a/integration/src/test/scala/org/knora/webapi/config/AppConfigForTestContainers.scala b/integration/src/test/scala/org/knora/webapi/config/AppConfigForTestContainers.scala
@@ -12,6 +12,7 @@ import zio.config.magnolia._
 import zio.config.typesafe.TypesafeConfigSource
 
 import org.knora.webapi.config.AppConfig.AppConfigurations
+import org.knora.webapi.testcontainers.DspIngestTestContainer
 import org.knora.webapi.testcontainers.FusekiTestContainer
 import org.knora.webapi.testcontainers.SipiTestContainer
 
@@ -23,19 +24,27 @@ object AppConfigForTestContainers {
   private def alterFusekiAndSipiPort(
     oldConfig: AppConfig,
     fusekiContainer: FusekiTestContainer,
-    sipiContainer: SipiTestContainer
+    sipiContainer: SipiTestContainer,
+    dspIngestContainer: DspIngestTestContainer
   ): UIO[AppConfig] = {
 
-    val newFusekiPort = fusekiContainer.getFirstMappedPort
-    val newSipiPort   = sipiContainer.container.getFirstMappedPort()
+    val newFusekiPort    = fusekiContainer.getFirstMappedPort
+    val newSipiPort      = sipiContainer.getFirstMappedPort
+    val newDspIngestPort = dspIngestContainer.getFirstMappedPort
 
     val alteredFuseki = oldConfig.triplestore.fuseki.copy(port = newFusekiPort)
 
     val alteredTriplestore = oldConfig.triplestore.copy(fuseki = alteredFuseki)
     val alteredSipi        = oldConfig.sipi.copy(internalPort = newSipiPort)
+    val alteredDspIngest   = oldConfig.dspIngest.copy(baseUrl = s"http://localhost:$newDspIngestPort")
 
     val newConfig: AppConfig =
-      oldConfig.copy(allowReloadOverHttp = true, triplestore = alteredTriplestore, sipi = alteredSipi)
+      oldConfig.copy(
+        allowReloadOverHttp = true,
+        triplestore = alteredTriplestore,
+        sipi = alteredSipi,
+        dspIngest = alteredDspIngest
+      )
 
     ZIO.succeed(newConfig)
   }
@@ -69,15 +78,17 @@ object AppConfigForTestContainers {
   private val config: UIO[AppConfig] = (read(descriptor[AppConfig].mapKey(toKebabCase) from source)).orDie
 
   /**
-   * Altered AppConfig with ports from TestContainers for Fuseki and Sipi.
+   * Altered AppConfig with ports from TestContainers for DSP-Ingest, Fuseki and Sipi.
    */
-  val testcontainers: ZLayer[FusekiTestContainer & SipiTestContainer, Nothing, AppConfigurations] = {
+  val testcontainers
+    : ZLayer[DspIngestTestContainer & FusekiTestContainer & SipiTestContainer, Nothing, AppConfigurations] = {
     val appConfigLayer = ZLayer {
       for {
-        appConfig       <- config
-        fusekiContainer <- ZIO.service[FusekiTestContainer]
-        sipiContainer   <- ZIO.service[SipiTestContainer]
-        alteredConfig   <- alterFusekiAndSipiPort(appConfig, fusekiContainer, sipiContainer)
+        appConfig          <- config
+        fusekiContainer    <- ZIO.service[FusekiTestContainer]
+        sipiContainer      <- ZIO.service[SipiTestContainer]
+        dspIngestContainer <- ZIO.service[DspIngestTestContainer]
+        alteredConfig      <- alterFusekiAndSipiPort(appConfig, fusekiContainer, sipiContainer, dspIngestContainer)
       } yield alteredConfig
     }
     AppConfig

diff --git a/integration/src/test/scala/org/knora/webapi/config/AppConfigForTestContainersZSpec.scala b/integration/src/test/scala/org/knora/webapi/config/AppConfigForTestContainersZSpec.scala
@@ -8,23 +8,31 @@ package org.knora.webapi.config
 import zio._
 import zio.test._
 
+import org.knora.webapi.testcontainers.DspIngestTestContainer
 import org.knora.webapi.testcontainers.FusekiTestContainer
+import org.knora.webapi.testcontainers.SharedVolumes
 import org.knora.webapi.testcontainers.SipiTestContainer
 
 object AppConfigForTestContainersZSpec extends ZIOSpecDefault {
 
-  def spec = suite("AppConfigForTestContainersSpec")(
+  def spec: Spec[Any, Nothing] = suite("AppConfigForTestContainersSpec")(
     test("successfully provide the adapted application configuration for using with test containers") {
       for {
-        appConfig     <- ZIO.service[AppConfig]
-        sipiContainer <- ZIO.service[SipiTestContainer]
+        appConfig          <- ZIO.service[AppConfig]
+        sipiContainer      <- ZIO.service[SipiTestContainer]
+        dspIngestContainer <- ZIO.service[DspIngestTestContainer]
       } yield {
-        assertTrue(appConfig.sipi.internalPort == sipiContainer.port)
+        assertTrue(
+          appConfig.sipi.internalPort == sipiContainer.getFirstMappedPort,
+          appConfig.dspIngest.baseUrl.endsWith(dspIngestContainer.getFirstMappedPort.toString)
+        )
       }
     }
   ).provide(
     AppConfigForTestContainers.testcontainers,
-    SipiTestContainer.layer,
-    FusekiTestContainer.layer
+    DspIngestTestContainer.layer,
+    FusekiTestContainer.layer,
+    SharedVolumes.Images.layer,
+    SipiTestContainer.layer
   )
 }
diff --git a/integration/src/test/scala/org/knora/webapi/core/LayersTest.scala b/integration/src/test/scala/org/knora/webapi/core/LayersTest.scala
@@ -60,7 +60,9 @@ import org.knora.webapi.store.iiif.impl.SipiServiceMock
 import org.knora.webapi.store.triplestore.api.TriplestoreService
 import org.knora.webapi.store.triplestore.impl.TriplestoreServiceLive
 import org.knora.webapi.store.triplestore.upgrade.RepositoryUpdater
+import org.knora.webapi.testcontainers.DspIngestTestContainer
 import org.knora.webapi.testcontainers.FusekiTestContainer
+import org.knora.webapi.testcontainers.SharedVolumes
 import org.knora.webapi.testcontainers.SipiTestContainer
 import org.knora.webapi.testservices.TestClientService
 
@@ -70,7 +72,10 @@ object LayersTest {
    * The `Environment`s that we require for the tests to run - with or without Sipi
    */
   type DefaultTestEnvironmentWithoutSipi = LayersLive.DspEnvironmentLive with FusekiTestContainer with TestClientService
-  type DefaultTestEnvironmentWithSipi    = DefaultTestEnvironmentWithoutSipi with SipiTestContainer
+  type DefaultTestEnvironmentWithSipi = DefaultTestEnvironmentWithoutSipi
+    with SipiTestContainer
+    with DspIngestTestContainer
+    with SharedVolumes.Images
 
   type CommonR0 = ActorSystem with AppConfigurations with SipiService with JwtService with StringFormatter
   type CommonR =
@@ -202,18 +207,23 @@ object LayersTest {
 
   private val fusekiAndSipiTestcontainers =
     ZLayer.make[
-      FusekiTestContainer
-        with SipiTestContainer
-        with AppConfigurations
+      AppConfigurations
+        with DspIngestTestContainer
+        with FusekiTestContainer
         with JwtService
+        with SharedVolumes.Images
         with SipiService
+        with SipiTestContainer
         with StringFormatter
     ](
       AppConfigForTestContainers.testcontainers,
+      DspIngestClientLive.layer,
+      DspIngestTestContainer.layer,
       FusekiTestContainer.layer,
       SipiTestContainer.layer,
       SipiServiceLive.layer,
       JwtServiceLive.layer,
+      SharedVolumes.Images.layer,
       StringFormatter.test
     )
 

diff --git a/integration/src/test/scala/org/knora/webapi/it/v2/KnoraSipiAuthenticationITSpec.scala b/integration/src/test/scala/org/knora/webapi/it/v2/KnoraSipiAuthenticationITSpec.scala
@@ -5,7 +5,9 @@
 
 package org.knora.webapi.it.v2
 
-import org.apache.pekko
+import org.apache.pekko.http.scaladsl.model._
+import org.apache.pekko.http.scaladsl.model.headers.BasicHttpCredentials
+import org.apache.pekko.http.scaladsl.unmarshalling.Unmarshal
 
 import java.nio.file.Files
 import java.nio.file.Paths
@@ -20,10 +22,6 @@ import org.knora.webapi.routing.Authenticator
 import org.knora.webapi.routing.UnsafeZioRun
 import org.knora.webapi.sharedtestdata.SharedTestDataADM
 
-import pekko.http.scaladsl.model._
-import pekko.http.scaladsl.model.headers.BasicHttpCredentials
-import pekko.http.scaladsl.unmarshalling.Unmarshal
-
 /**
  * Tests interaction between Knora and Sipi using Knora API v2.
  */
@@ -65,7 +63,6 @@ class KnoraSipiAuthenticationITSpec
     }
 
     "successfully get an image with provided credentials inside cookie" in {
-
       // using cookie to authenticate when accessing sipi (test for cookie parsing in sipi)
       val KnoraAuthenticationCookieName = UnsafeZioRun.runOrThrow(Authenticator.calculateCookieName())
       val cookieHeader                  = headers.Cookie(KnoraAuthenticationCookieName, loginToken)

diff --git a/integration/src/test/scala/org/knora/webapi/it/v2/KnoraSipiIntegrationV2ITSpec.scala b/integration/src/test/scala/org/knora/webapi/it/v2/KnoraSipiIntegrationV2ITSpec.scala
@@ -504,9 +504,6 @@ class KnoraSipiIntegrationV2ITSpec
     }
 
     "create a resource with a still image file that has already been ingested" in {
-      copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.jp2")
-      copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.info")
-      copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.png.orig")
       // Create the resource in the API.
       val jsonLdEntity = UploadFileRequest
         .make(fileType = FileType.StillImageFile(), internalFilename = "De6XyNL4H71-D9QxghOuOPJ.jp2")
@@ -522,9 +519,6 @@ class KnoraSipiIntegrationV2ITSpec
     }
 
     "not create a resource with a still image file that has already been ingested if the header is not provided" in {
-      copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.jp2")
-      copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.info")
-      copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.png.orig")
       // Create the resource in the API.
       val jsonLdEntity = UploadFileRequest
         .make(fileType = FileType.StillImageFile(), internalFilename = "De6XyNL4H71-D9QxghOuOPJ.jp2")

diff --git a/integration/src/test/scala/org/knora/webapi/messages/util/standoff/XMLUtilSpec.scala b/integration/src/test/scala/org/knora/webapi/messages/util/standoff/XMLUtilSpec.scala
@@ -3,23 +3,20 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.knora.webapi.util.standoff
+package org.knora.webapi.messages.util.standoff
 
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AnyWordSpec
 import org.xmlunit.builder.DiffBuilder
 import org.xmlunit.builder.Input
 import org.xmlunit.diff.Diff
 
 import java.nio.file.Paths
 
 import dsp.errors.StandoffConversionException
-import org.knora.webapi.CoreSpec
-import org.knora.webapi.messages.util.standoff.XMLUtil
 import org.knora.webapi.util.FileUtil
 
-/**
- * Tests [[org.knora.webapi.messages.util.standoff.XMLToStandoffUtil]].
- */
-class XMLUtilSpec extends CoreSpec {
+class XMLUtilSpec extends AnyWordSpec with Matchers {
 
   "The XML to standoff utility" should {