feat: Dsp-ingest integration for import/export of projects DEV-2297

docker-compose.yml

@@ -54,7 +54,7 @@ services:
     # command: --config=/sipi/config/sipi.docker-test-config.lua ## command variant to start the sipi container with test routes enabled
     command: --config=/sipi/config/sipi.docker-config.lua
 
-  dsp-ingest:
+  ingest:
     image: daschswiss/dsp-ingest:latest
     ports:
       - "3340:3340"
@@ -70,7 +70,6 @@ services:
       - JWT_AUDIENCE=http://localhost:3340
       - JWT_ISSUER=0.0.0.0:3333
       - JWT_SECRET=UP 4888, nice 4-8-4 steam engine
-      - JWT_DISABLE_AUTH=true
 
   api:
     image: daschswiss/knora-api:latest

docs/04-publishing-deployment/configuration.md

@@ -30,8 +30,9 @@ A number of core settings is additionally configurable through system environmen
 | app.bcrypt-password-strength           | KNORA_WEBAPI_BCRYPT_PASSWORD_STRENGTH           | 12                    |
 | app.jwt.secret                         | KNORA_WEBAPI_JWT_SECRET_KEY                     | super-secret-key      |
 | app.jwt.expiration                     | KNORA_WEBAPI_JWT_LONGEVITY                      | 30 days               |
-| app.jwt.dsp-issuer-audience            | KNORA_WEBAPI_JWT_DSP_INGEST_AUDIENCE            | http://localhost:3340 | 
 | app.jwt.issuer                         | KNORA_WEBAPI_JWT_ISSUER                         | 0.0.0.0:3333          | 
+| app.dsp-ingest.audience                | KNORA_WEBAPI_DSP_INGEST_AUDIENCE                | http://localhost:3340 | 
+| app.dsp-ingest.base-url                | KNORA_WEBAPI_DSP_INGEST_BASE_URL                | http://localhost:3340 | 
 | app.cookie-domain                      | KNORA_WEBAPI_COOKIE_DOMAIN                      | localhost             |
 | app.allow-reload-over-http             | KNORA_WEBAPI_ALLOW_RELOAD_OVER_HTTP             | false                 |
 | app.ark.resolver                       | KNORA_WEBAPI_ARK_RESOLVER_URL                   | http://0.0.0.0:3336   |

webapi/src/it/scala/org/knora/webapi/ITKnoraLiveSpec.scala

@@ -15,15 +15,16 @@ import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
 import spray.json._
 import zio._
+
 import java.util.concurrent.TimeUnit
 import scala.concurrent.Await
 import scala.concurrent.ExecutionContext
 import scala.concurrent.Future
 import scala.concurrent.duration.FiniteDuration
-
 import org.knora.webapi.config.AppConfig
 import org.knora.webapi.core.AppRouter
 import org.knora.webapi.core.AppServer
+import org.knora.webapi.core.LayersTest.DefaultTestEnvironmentWithSipi
 import org.knora.webapi.core.TestStartupUtils
 import org.knora.webapi.messages.store.sipimessages._
 import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject
@@ -68,11 +69,12 @@ abstract class ITKnoraLiveSpec
   private val bootstrap = util.Logger.text() >>> effectLayers
 
   // create a configured runtime
-  implicit val runtime = Unsafe.unsafe(implicit u => Runtime.unsafe.fromLayer(bootstrap))
+  implicit val runtime: Runtime.Scoped[DefaultTestEnvironmentWithSipi] =
+    Unsafe.unsafe(implicit u => Runtime.unsafe.fromLayer(bootstrap))
 
   // An effect for getting stuff out, so that we can pass them
   // to some legacy code
-  val routerAndConfig = for {
+  val routerAndConfig: ZIO[AppConfig with AppRouter, Nothing, (AppRouter, AppConfig)] = for {
     router <- ZIO.service[core.AppRouter]
     config <- ZIO.service[AppConfig]
   } yield (router, config)
@@ -81,13 +83,7 @@ abstract class ITKnoraLiveSpec
    * Create router and config by unsafe running them.
    */
   val (router: AppRouter, config: AppConfig) =
-    Unsafe.unsafe { implicit u =>
-      runtime.unsafe
-        .run(
-          routerAndConfig
-        )
-        .getOrThrowFiberFailure()
-    }
+    Unsafe.unsafe(implicit u => runtime.unsafe.run(routerAndConfig).getOrThrowFiberFailure())
 
   implicit lazy val system: akka.actor.ActorSystem     = router.system
   implicit lazy val executionContext: ExecutionContext = system.dispatcher

webapi/src/it/scala/org/knora/webapi/core/LayersTest.scala

@@ -5,129 +5,56 @@
 
 package org.knora.webapi.core
 
-import zio._
-
-import org.knora.webapi.config.AppConfig
 import org.knora.webapi.config.AppConfig.AppConfigurations
 import org.knora.webapi.config.AppConfigForTestContainers
 import org.knora.webapi.http.middleware.AuthenticationMiddleware
 import org.knora.webapi.messages.StringFormatter
-import org.knora.webapi.messages.util.ConstructResponseUtilV2
-import org.knora.webapi.messages.util.ConstructResponseUtilV2Live
-import org.knora.webapi.messages.util.PermissionUtilADM
-import org.knora.webapi.messages.util.PermissionUtilADMLive
-import org.knora.webapi.messages.util.ValueUtilV1
-import org.knora.webapi.messages.util.ValueUtilV1Live
+import org.knora.webapi.messages.util._
 import org.knora.webapi.messages.util.search.QueryTraverser
 import org.knora.webapi.messages.util.search.gravsearch.prequery.InferenceOptimizationService
-import org.knora.webapi.messages.util.search.gravsearch.transformers.OntologyInferencer
+import org.knora.webapi.messages.util.search.gravsearch.transformers.{ConstructTransformer, OntologyInferencer}
 import org.knora.webapi.messages.util.search.gravsearch.types.GravsearchTypeInspectionRunner
-import org.knora.webapi.messages.util.standoff.StandoffTagUtilV2
-import org.knora.webapi.messages.util.standoff.StandoffTagUtilV2Live
+import org.knora.webapi.messages.util.standoff.{StandoffTagUtilV2, StandoffTagUtilV2Live}
 import org.knora.webapi.responders.IriService
-import org.knora.webapi.responders.admin.GroupsResponderADM
-import org.knora.webapi.responders.admin.GroupsResponderADMLive
-import org.knora.webapi.responders.admin.ListsResponderADM
-import org.knora.webapi.responders.admin.ListsResponderADMLive
-import org.knora.webapi.responders.admin.PermissionsResponderADM
-import org.knora.webapi.responders.admin.PermissionsResponderADMLive
-import org.knora.webapi.responders.admin.ProjectsResponderADM
-import org.knora.webapi.responders.admin.ProjectsResponderADMLive
-import org.knora.webapi.responders.admin.SipiResponderADM
-import org.knora.webapi.responders.admin.SipiResponderADMLive
-import org.knora.webapi.responders.admin.StoresResponderADM
-import org.knora.webapi.responders.admin.StoresResponderADMLive
-import org.knora.webapi.responders.admin.UsersResponderADM
-import org.knora.webapi.responders.admin.UsersResponderADMLive
-import org.knora.webapi.responders.v1.CkanResponderV1
-import org.knora.webapi.responders.v1.CkanResponderV1Live
-import org.knora.webapi.responders.v1.ListsResponderV1
-import org.knora.webapi.responders.v1.ListsResponderV1Live
-import org.knora.webapi.responders.v1.OntologyResponderV1
-import org.knora.webapi.responders.v1.OntologyResponderV1Live
-import org.knora.webapi.responders.v1.ProjectsResponderV1
-import org.knora.webapi.responders.v1.ProjectsResponderV1Live
-import org.knora.webapi.responders.v1.ResourcesResponderV1
-import org.knora.webapi.responders.v1.ResourcesResponderV1Live
-import org.knora.webapi.responders.v1.SearchResponderV1
-import org.knora.webapi.responders.v1.SearchResponderV1Live
-import org.knora.webapi.responders.v1.StandoffResponderV1
-import org.knora.webapi.responders.v1.StandoffResponderV1Live
-import org.knora.webapi.responders.v1.UsersResponderV1
-import org.knora.webapi.responders.v1.UsersResponderV1Live
-import org.knora.webapi.responders.v1.ValuesResponderV1
-import org.knora.webapi.responders.v1.ValuesResponderV1Live
-import org.knora.webapi.responders.v2.ListsResponderV2
-import org.knora.webapi.responders.v2.ListsResponderV2Live
-import org.knora.webapi.responders.v2.OntologyResponderV2
-import org.knora.webapi.responders.v2.OntologyResponderV2Live
-import org.knora.webapi.responders.v2.ResourceUtilV2
-import org.knora.webapi.responders.v2.ResourceUtilV2Live
-import org.knora.webapi.responders.v2.ResourcesResponderV2
-import org.knora.webapi.responders.v2.ResourcesResponderV2Live
-import org.knora.webapi.responders.v2.SearchResponderV2
-import org.knora.webapi.responders.v2.SearchResponderV2Live
-import org.knora.webapi.responders.v2.StandoffResponderV2
-import org.knora.webapi.responders.v2.StandoffResponderV2Live
-import org.knora.webapi.responders.v2.ValuesResponderV2
-import org.knora.webapi.responders.v2.ValuesResponderV2Live
-import org.knora.webapi.responders.v2.ontology.CardinalityHandler
-import org.knora.webapi.responders.v2.ontology.CardinalityHandlerLive
-import org.knora.webapi.responders.v2.ontology.OntologyHelpers
-import org.knora.webapi.responders.v2.ontology.OntologyHelpersLive
-import org.knora.webapi.routing.ApiRoutes
-import org.knora.webapi.routing.Authenticator
-import org.knora.webapi.routing.AuthenticatorLive
-import org.knora.webapi.routing.JwtService
-import org.knora.webapi.routing.JwtServiceLive
-import org.knora.webapi.routing.admin.AuthenticatorService
-import org.knora.webapi.routing.admin.ProjectsRouteZ
-import org.knora.webapi.slice.admin.api.service.ProjectADMRestService
-import org.knora.webapi.slice.admin.api.service.ProjectsADMRestServiceLive
-import org.knora.webapi.slice.admin.domain.service.ProjectADMService
-import org.knora.webapi.slice.admin.domain.service.ProjectADMServiceLive
-import org.knora.webapi.slice.admin.domain.service.ProjectExportService
-import org.knora.webapi.slice.admin.domain.service.ProjectExportServiceLive
+import org.knora.webapi.responders.admin._
+import org.knora.webapi.responders.v1._
+import org.knora.webapi.responders.v2._
+import org.knora.webapi.responders.v2.ontology.{
+  CardinalityHandler,
+  CardinalityHandlerLive,
+  OntologyHelpers,
+  OntologyHelpersLive
+}
+import org.knora.webapi.routing._
+import org.knora.webapi.routing.admin.{AuthenticatorService, ProjectsRouteZ}
+import org.knora.webapi.slice.admin.api.service.{ProjectADMRestService, ProjectsADMRestServiceLive}
+import org.knora.webapi.slice.admin.domain.service._
 import org.knora.webapi.slice.admin.repo.service.KnoraProjectRepoLive
-import org.knora.webapi.slice.common.api.RestPermissionService
-import org.knora.webapi.slice.common.api.RestPermissionServiceLive
+import org.knora.webapi.slice.common.api.{RestPermissionService, RestPermissionServiceLive}
 import org.knora.webapi.slice.common.repo.service.PredicateObjectMapper
-import org.knora.webapi.slice.ontology.api.service.RestCardinalityService
-import org.knora.webapi.slice.ontology.api.service.RestCardinalityServiceLive
-import org.knora.webapi.slice.ontology.domain.service.CardinalityService
-import org.knora.webapi.slice.ontology.domain.service.OntologyRepo
-import org.knora.webapi.slice.ontology.repo.service.OntologyCache
-import org.knora.webapi.slice.ontology.repo.service.OntologyCacheLive
-import org.knora.webapi.slice.ontology.repo.service.OntologyRepoLive
-import org.knora.webapi.slice.ontology.repo.service.PredicateRepositoryLive
-import org.knora.webapi.slice.resourceinfo.api.ResourceInfoRoute
-import org.knora.webapi.slice.resourceinfo.api.RestResourceInfoService
-import org.knora.webapi.slice.resourceinfo.domain.IriConverter
-import org.knora.webapi.slice.resourceinfo.domain.ResourceInfoRepo
-import org.knora.webapi.store.cache.CacheServiceRequestMessageHandler
-import org.knora.webapi.store.cache.CacheServiceRequestMessageHandlerLive
+import org.knora.webapi.slice.ontology.api.service.{RestCardinalityService, RestCardinalityServiceLive}
+import org.knora.webapi.slice.ontology.domain.service.{CardinalityService, OntologyRepo}
+import org.knora.webapi.slice.ontology.repo.service.{
+  OntologyCache,
+  OntologyCacheLive,
+  OntologyRepoLive,
+  PredicateRepositoryLive
+}
+import org.knora.webapi.slice.resourceinfo.api.{ResourceInfoRoute, RestResourceInfoService}
+import org.knora.webapi.slice.resourceinfo.domain.{IriConverter, ResourceInfoRepo}
+import org.knora.webapi.store.cache.{CacheServiceRequestMessageHandler, CacheServiceRequestMessageHandlerLive}
 import org.knora.webapi.store.cache.api.CacheService
 import org.knora.webapi.store.cache.impl.CacheServiceInMemImpl
-import org.knora.webapi.store.iiif.IIIFRequestMessageHandler
-import org.knora.webapi.store.iiif.IIIFRequestMessageHandlerLive
+import org.knora.webapi.store.iiif.{IIIFRequestMessageHandler, IIIFRequestMessageHandlerLive}
 import org.knora.webapi.store.iiif.api.IIIFService
-import org.knora.webapi.store.iiif.impl.IIIFServiceMockImpl
-import org.knora.webapi.store.iiif.impl.IIIFServiceSipiImpl
-import org.knora.webapi.store.triplestore.TriplestoreRequestMessageHandler
-import org.knora.webapi.store.triplestore.TriplestoreRequestMessageHandlerLive
+import org.knora.webapi.store.iiif.impl.{IIIFServiceMockImpl, IIIFServiceSipiImpl}
+import org.knora.webapi.store.triplestore.{TriplestoreRequestMessageHandler, TriplestoreRequestMessageHandlerLive}
 import org.knora.webapi.store.triplestore.api.TriplestoreService
 import org.knora.webapi.store.triplestore.impl.TriplestoreServiceLive
 import org.knora.webapi.store.triplestore.upgrade.RepositoryUpdater
-import org.knora.webapi.testcontainers.FusekiTestContainer
-import org.knora.webapi.testcontainers.SipiTestContainer
+import org.knora.webapi.testcontainers.{FusekiTestContainer, SipiTestContainer}
 import org.knora.webapi.testservices.TestClientService
-import org.knora.webapi.messages.util.search.gravsearch.transformers.ConstructTransformer
-import org.knora.webapi.slice.admin.domain.service.AssetService
-import org.knora.webapi.slice.admin.domain.service.AssetServiceLive
-import org.knora.webapi.slice.admin.domain.service.ProjectExportStorageService
-import org.knora.webapi.slice.admin.domain.service.ProjectExportStorageServiceLive
-import org.knora.webapi.slice.admin.domain.service.ProjectImportService
-import org.knora.webapi.slice.admin.domain.service.ProjectImportServiceLive
+import zio._
 
 object LayersTest {
 
@@ -137,19 +64,19 @@ object LayersTest {
   type DefaultTestEnvironmentWithoutSipi = LayersLive.DspEnvironmentLive with FusekiTestContainer with TestClientService
   type DefaultTestEnvironmentWithSipi    = DefaultTestEnvironmentWithoutSipi with SipiTestContainer
 
-  type CommonR0 = ActorSystem with AppConfig with IIIFService with JwtService with StringFormatter
+  type CommonR0 = ActorSystem with AppConfigurations with IIIFService with JwtService with StringFormatter
   type CommonR =
     ApiRoutes
       with AppRouter
       with Authenticator
-      with AssetService
       with CacheService
       with CacheServiceRequestMessageHandler
       with CardinalityHandler
       with CardinalityService
       with CkanResponderV1
       with ConstructResponseUtilV2
       with ConstructTransformer
+      with DspIngestClient
       with GravsearchTypeInspectionRunner
       with GroupsResponderADM
       with HttpServer
@@ -211,14 +138,14 @@ object LayersTest {
       AuthenticationMiddleware.layer,
       AuthenticatorLive.layer,
       AuthenticatorService.layer,
-      AssetServiceLive.layer,
       CacheServiceInMemImpl.layer,
       CacheServiceRequestMessageHandlerLive.layer,
       CardinalityHandlerLive.layer,
       CardinalityService.layer,
       CkanResponderV1Live.layer,
       ConstructResponseUtilV2Live.layer,
       ConstructTransformer.layer,
+      DspIngestClientLive.layer,
       GravsearchTypeInspectionRunner.layer,
       GroupsResponderADMLive.layer,
       HttpServer.layer,

webapi/src/it/scala/org/knora/webapi/slice/admin/domain/service/ProjectImportServiceIT.scala

@@ -5,14 +5,14 @@
 
 package org.knora.webapi.slice.admin.domain.service
 
+import dsp.valueobjects.Project
+import org.knora.webapi.config.{Fuseki, Triplestore}
+import org.knora.webapi.testcontainers.FusekiTestContainer
 import zio._
 import zio.nio.file._
 import zio.test._
-import java.io.IOException
 
-import org.knora.webapi.config.Fuseki
-import org.knora.webapi.config.Triplestore
-import org.knora.webapi.testcontainers.FusekiTestContainer
+import java.io.IOException
 
 object ProjectImportServiceIT extends ZIOSpecDefault {
 
@@ -24,11 +24,21 @@ object ProjectImportServiceIT extends ZIOSpecDefault {
     } yield ProjectExportStorageServiceLive(exportDirectory)
   }
 
+  private val dspIngestClientLayer: ULayer[DspIngestClient] = ZLayer.succeed {
+    new DspIngestClient {
+      override def exportProject(shortCode: Project.ShortCode): ZIO[Scope, Throwable, Path] =
+        ZIO.succeed(Path("unused"))
+      override def importProject(shortCode: Project.ShortCode, fileToImport: Path): Task[Path] =
+        ZIO.succeed(Path("unused"))
+    }
+  }
+
   private val importServiceTestLayer
     : URLayer[FusekiTestContainer with ProjectExportStorageService, ProjectImportServiceLive] = ZLayer.fromZIO {
-    for {
+    (for {
       exportStorageService <- ZIO.service[ProjectExportStorageService]
       container            <- ZIO.service[FusekiTestContainer]
+      dspIngestClient      <- ZIO.service[DspIngestClient]
       config =
         Triplestore(
           dbtype = "tdb2",
@@ -45,7 +55,8 @@ object ProjectImportServiceIT extends ZIOSpecDefault {
           ),
           profileQueries = false
         )
-    } yield ProjectImportServiceLive(config, exportStorageService)
+    } yield ProjectImportServiceLive(config, exportStorageService, dspIngestClient))
+      .provideSomeLayer[FusekiTestContainer with ProjectExportStorageService](dspIngestClientLayer)
   }
 
   private val trigContent =

webapi/src/main/resources/application.conf

@@ -264,13 +264,18 @@ app {
 
     // configuration for the JWT authentication
     jwt {
-      secret = "UP 4888, nice 4-8-4 steam engine"
-      secret = ${?KNORA_WEBAPI_JWT_SECRET_KEY}
-      expiration = 30 days
-      expiration = ${?KNORA_WEBAPI_JWT_LONGEVITY}
-      issuer = ${?KNORA_WEBAPI_JWT_ISSUER}
-      dsp-ingest-audience = "http://localhost:3340"
-      dsp-ingest-audience = ${?KNORA_WEBAPI_JWT_DSP_INGEST_AUDIENCE}
+        secret = "UP 4888, nice 4-8-4 steam engine"
+        secret = ${?KNORA_WEBAPI_JWT_SECRET_KEY}
+        expiration = 30 days
+        expiration = ${?KNORA_WEBAPI_JWT_LONGEVITY}
+        issuer = ${?KNORA_WEBAPI_JWT_ISSUER}
+    }
+
+    dsp-ingest {
+        base-url = "http://localhost:3340"
+        base-url = ${?KNORA_WEBAPI_DSP_INGEST_BASE_URL}
+        audience = "http://localhost:3340"
+        audience = ${?KNORA_WEBAPI_DSP_INGEST_AUDIENCE}
     }
 
     knora-api {

webapi/src/main/scala/org/knora/webapi/Main.scala

@@ -23,7 +23,7 @@ object Main extends ZIOApp {
    * `Bootstrap` will ensure that everything is instantiated when the Runtime is created
    * and cleaned up when the Runtime is shutdown.
    */
-  override def bootstrap: ZLayer[Any, Nothing, LayersLive.DspEnvironmentLive] =
+  override def bootstrap: ZLayer[Any, Nothing, Environment] =
     Logger.fromEnv() >>> LayersLive.dspLayersLive
 
   /**