feat: add export of assets (DEV-2106) (#2668)

Co-authored-by: Balduin Landolt <33053745+BalduinLandolt@users.noreply.github.com>

docker-compose.yml

@@ -3,7 +3,7 @@ version: '3.7'
 services:
 
   app:
-    image: daschswiss/dsp-app:latest
+    image: daschswiss/dsp-app:v10.19.2
     ports:
       - "4200:4200"
     networks:
@@ -41,7 +41,10 @@ services:
       - SIPI_EXTERNAL_PROTOCOL=http
       - SIPI_EXTERNAL_HOSTNAME=0.0.0.0
       - SIPI_EXTERNAL_PORT=1024
-      - SIPI_WEBAPI_HOSTNAME=api
+      # Use the following line if you start the api as a docker container from this docker-compose.yml
+      # - SIPI_WEBAPI_HOSTNAME=api
+      # Use the following line if you start the api from your IDE
+      - SIPI_WEBAPI_HOSTNAME=host.docker.internal
       - SIPI_WEBAPI_PORT=3333
       - KNORA_WEBAPI_KNORA_API_EXTERNAL_HOST=0.0.0.0
       - KNORA_WEBAPI_KNORA_API_EXTERNAL_PORT=3333

project/Dependencies.scala

@@ -102,12 +102,18 @@ object Dependencies {
   val xmlunitCore    = "org.xmlunit"       % "xmlunit-core"     % "2.9.1"
 
   // test
-  val akkaHttpTestkit   = "com.typesafe.akka"     %% "akka-http-testkit"   % AkkaHttpVersion  // Scala 3 incompatible
-  val akkaStreamTestkit = "com.typesafe.akka"     %% "akka-stream-testkit" % AkkaActorVersion // Scala 3 compatible
-  val akkaTestkit       = "com.typesafe.akka"     %% "akka-testkit"        % AkkaActorVersion // Scala 3 compatible
-  val scalaTest         = "org.scalatest"         %% "scalatest"           % "3.2.15"         // Scala 3 compatible
-  val testcontainers    = "org.testcontainers"     % "testcontainers"      % "1.18.0"
-  val wiremock          = "com.github.tomakehurst" % "wiremock-jre8"       % "2.35.0"
+  val akkaHttpTestkit   = "com.typesafe.akka" %% "akka-http-testkit"   % AkkaHttpVersion  // Scala 3 incompatible
+  val akkaStreamTestkit = "com.typesafe.akka" %% "akka-stream-testkit" % AkkaActorVersion // Scala 3 compatible
+  val akkaTestkit       = "com.typesafe.akka" %% "akka-testkit"        % AkkaActorVersion // Scala 3 compatible
+  val scalaTest         = "org.scalatest"     %% "scalatest"           % "3.2.15"         // Scala 3 compatible
+  // The scoverage plugin actually adds its dependencies automatically.
+  // Add it redundantly to the IT dependencies in order to fix build issues with IntelliJ
+  // Fixes error message when running IT in IntelliJ
+  //  A needed class was not found. This could be due to an error in your runpath.Missing class: scoverage / Invoker$
+  //  java.lang.NoClassDefFoundError: scoverage / Invoker$
+  val scoverage      = "org.scoverage"         %% "scalac-scoverage-runtime" % "2.0.7"
+  val testcontainers = "org.testcontainers"     % "testcontainers"           % "1.18.0"
+  val wiremock       = "com.github.tomakehurst" % "wiremock-jre8"            % "2.35.0"
 
   // found/added by the plugin but deleted anyway
   val commonsLang3 = "org.apache.commons" % "commons-lang3" % "3.12.0"
@@ -118,6 +124,7 @@ object Dependencies {
     akkaTestkit,
     rdf4jClient,
     scalaTest,
+    scoverage,
     testcontainers,
     wiremock,
     xmlunitCore,

project/plugins.sbt

@@ -11,11 +11,12 @@ addSbtPlugin("io.kamon"          % "sbt-aspectj-runner"  % "1.1.2")
 addSbtPlugin("com.typesafe.play" % "sbt-twirl"           % "1.5.2")
 addSbtPlugin("com.eed3si9n"      % "sbt-assembly"        % "2.1.1")
 addSbtPlugin("com.lightbend.sbt" % "sbt-javaagent"       % "0.1.6")
-addSbtPlugin("org.scoverage"     % "sbt-scoverage"       % "2.0.7")
-addSbtPlugin("com.eed3si9n"      % "sbt-buildinfo"       % "0.11.0")
-addSbtPlugin("org.scalameta"     % "sbt-scalafmt"        % "2.5.0")
-addSbtPlugin("ch.epfl.scala"     % "sbt-scalafix"        % "0.10.4")
-addSbtPlugin("de.heikoseeberger" % "sbt-header"          % "5.9.0")
+// also update the scalac-scoverage-runtime version in build.sbt
+addSbtPlugin("org.scoverage"     % "sbt-scoverage" % "2.0.7")
+addSbtPlugin("com.eed3si9n"      % "sbt-buildinfo" % "0.11.0")
+addSbtPlugin("org.scalameta"     % "sbt-scalafmt"  % "2.5.0")
+addSbtPlugin("ch.epfl.scala"     % "sbt-scalafix"  % "0.10.4")
+addSbtPlugin("de.heikoseeberger" % "sbt-header"    % "5.9.0")
 
 // ad-hoc plugins - uncomment on demenad and keep it commented out in main branch
 

sipi/scripts/authentication.lua

@@ -7,6 +7,8 @@ require "send_response"
 require "strings"
 require "util"
 
+local NO_TOKEN_FOUND_ERROR = "No token found"
+
 --- Extracts a JSON web token (JWT) from the HTTP request: header ('Authorization' and 'Cookie') or query param 'token'.
 -- If present decodes token and validates claims exp, aud, iss.
 -- Sends an HTTP error if the token is invalid.
@@ -22,7 +24,7 @@ function _token()
     local nil_token = { raw = nil, decoded = nil }
     local jwt_raw = _get_jwt_string_from_header_params_or_cookie()
     if jwt_raw == nil then
-        return nil_token, "No token found"
+        return nil_token, NO_TOKEN_FOUND_ERROR
     else
         local decoded, error = _decode_jwt(jwt_raw)
         if decoded == nil then
@@ -81,6 +83,8 @@ function _decode_jwt(token_str)
     if decoded_token["iss"] ~= token_issuer then
         return _send_unauthorized_error(401, "Invalid 'iss' (issuer) in token, expected: " .. token_issuer .. ".")
     end
+
+    log("authentication: decoded jwt token for 'sub' " .. decoded_token["sub"], server.loglevel.LOG_DEBUG)
     return decoded_token
 end
 

webapi/src/it/scala/org/knora/webapi/core/LayersTest.scala

@@ -121,6 +121,9 @@ import org.knora.webapi.testcontainers.FusekiTestContainer
 import org.knora.webapi.testcontainers.SipiTestContainer
 import org.knora.webapi.testservices.TestClientService
 import org.knora.webapi.messages.util.search.gravsearch.transformers.ConstructTransformer
+import org.knora.webapi.slice.admin.domain.service.AssetService
+import org.knora.webapi.slice.admin.domain.service.AssetServiceLive
+
 object LayersTest {
 
   /**
@@ -134,6 +137,7 @@ object LayersTest {
     ApiRoutes
       with AppRouter
       with Authenticator
+      with AssetService
       with CacheService
       with CacheServiceRequestMessageHandler
       with CardinalityHandler
@@ -200,6 +204,7 @@ object LayersTest {
       AuthenticationMiddleware.layer,
       AuthenticatorLive.layer,
       AuthenticatorService.layer,
+      AssetServiceLive.layer,
       CacheServiceInMemImpl.layer,
       CacheServiceRequestMessageHandlerLive.layer,
       CardinalityHandlerLive.layer,

webapi/src/it/scala/org/knora/webapi/it/v1/KnoraSipiIntegrationV1ITSpec.scala

@@ -182,27 +182,25 @@ class KnoraSipiIntegrationV1ITSpec
     }
   }
 
+  private def getLoginToken() = {
+    val params =
+      s"""
+         |{
+         |    "email": "$userEmail",
+         |    "password": "$password"
+         |}
+              """.stripMargin
+    val loginResponse: HttpResponse = singleAwaitingRequest(
+      Post(baseApiUrl + s"/v2/authentication", HttpEntity(ContentTypes.`application/json`, params))
+    )
+    assert(loginResponse.status == StatusCodes.OK)
+    Await.result(Unmarshal(loginResponse.entity).to[LoginResponse], 1.seconds).token
+  }
+
   "Knora and Sipi" should {
-    var loginToken: String = ""
+    lazy val loginToken: String = getLoginToken()
 
     "log in as a Knora user" in {
-      /* Correct username and correct password */
-
-      val params =
-        s"""
-           |{
-           |    "email": "$userEmail",
-           |    "password": "$password"
-           |}
-                """.stripMargin
-
-      val request                = Post(baseApiUrl + s"/v2/authentication", HttpEntity(ContentTypes.`application/json`, params))
-      val response: HttpResponse = singleAwaitingRequest(request)
-      assert(response.status == StatusCodes.OK)
-
-      val lr: LoginResponse = Await.result(Unmarshal(response.entity).to[LoginResponse], 1.seconds)
-      loginToken = lr.token
-
       loginToken.nonEmpty should be(true)
     }
 

webapi/src/it/scala/org/knora/webapi/it/v2/StandoffRouteV2ITSpec.scala

@@ -44,8 +44,7 @@ import org.knora.webapi.util.MutableTestIri
  */
 class StandoffRouteV2ITSpec extends ITKnoraLiveSpec with AuthenticationV2JsonProtocol {
 
-  private implicit val stringFormatter: StringFormatter = StringFormatter.getGeneralInstance
-  val validationFun: (String, => Nothing) => String     = (s, e) => Iri.validateAndEscapeIri(s).getOrElse(e)
+  val validationFun: (String, => Nothing) => String = (s, e) => Iri.validateAndEscapeIri(s).getOrElse(e)
 
   private val anythingUser      = SharedTestDataADM.anythingUser1
   private val anythingUserEmail = anythingUser.email

webapi/src/it/scala/org/knora/webapi/routing/AuthenticatorSpec.scala

@@ -8,11 +8,12 @@ package org.knora.webapi.routing
 import akka.testkit.ImplicitSender
 import akka.util.Timeout
 import org.scalatest.PrivateMethodTester
+
 import dsp.errors.BadCredentialsException
 import dsp.errors.BadRequestException
-
 import org.knora.webapi._
 import org.knora.webapi.messages.StringFormatter
+import org.knora.webapi.messages.admin.responder.usersmessages.UserADM
 import org.knora.webapi.messages.admin.responder.usersmessages.UserIdentifierADM
 import org.knora.webapi.messages.v2.routing.authenticationmessages.KnoraCredentialsV2.KnoraJWTTokenCredentialsV2
 import org.knora.webapi.messages.v2.routing.authenticationmessages.KnoraCredentialsV2.KnoraPasswordCredentialsV2
@@ -32,6 +33,8 @@ class AuthenticatorSpec extends CoreSpec with ImplicitSender with PrivateMethodT
 
   implicit val stringFormatter: StringFormatter = StringFormatter.getGeneralInstance
 
+  private def testUserAdmFromIri(iri: String) = UserADM(iri, "", "", "", "", false, "")
+
   "During Authentication" when {
     "called, the 'getUserADMByEmail' method " should {
       "succeed with the correct 'email' " in {
@@ -89,7 +92,8 @@ class AuthenticatorSpec extends CoreSpec with ImplicitSender with PrivateMethodT
       "succeed with correct token" in {
         val resF = UnsafeZioRun.runToFuture(
           for {
-            token     <- JwtService.createToken("http://rdfh.ch/users/X-T8IkfQTKa86UWuISpbOA")
+            token <-
+              JwtService.createJwt(testUserAdmFromIri("http://rdfh.ch/users/X-T8IkfQTKa86UWuISpbOA")).map(_.jwtString)
             tokenCreds = KnoraJWTTokenCredentialsV2(token)
             result    <- Authenticator.authenticateCredentialsV2(Some(tokenCreds))
           } yield result
@@ -101,7 +105,8 @@ class AuthenticatorSpec extends CoreSpec with ImplicitSender with PrivateMethodT
         assertThrows[BadCredentialsException] {
           throw UnsafeZioRun
             .run(for {
-              token     <- JwtService.createToken("http://rdfh.ch/users/X-T8IkfQTKa86UWuISpbOA")
+              token <-
+                JwtService.createJwt(testUserAdmFromIri("http://rdfh.ch/users/X-T8IkfQTKa86UWuISpbOA")).map(_.jwtString)
               tokenCreds = KnoraJWTTokenCredentialsV2(token)
               _          = CacheUtil.put(AUTHENTICATION_INVALIDATION_CACHE_NAME, tokenCreds.jwtToken, tokenCreds.jwtToken)
               result    <- Authenticator.authenticateCredentialsV2(Some(tokenCreds))

webapi/src/it/scala/org/knora/webapi/routing/JwtServiceSpec.scala

@@ -20,7 +20,7 @@ class JwtServiceSpec extends CoreSpec with ImplicitSender {
 
     "create a token" in {
       val runZio = for {
-        token   <- JwtService.createToken(SharedTestDataADM.anythingUser1.id, Map("foo" -> JsString("bar")))
+        token   <- JwtService.createJwt(SharedTestDataADM.anythingUser1, Map("foo" -> JsString("bar"))).map(_.jwtString)
         useriri <- JwtService.extractUserIriFromToken(token)
       } yield useriri
 

webapi/src/it/scala/org/knora/webapi/store/iiif/impl/IIIFServiceMockImpl.scala

@@ -6,9 +6,12 @@
 package org.knora.webapi.store.iiif.impl
 
 import zio._
+import zio.nio.file.Path
 
+import org.knora.webapi.messages.admin.responder.usersmessages.UserADM
 import org.knora.webapi.messages.store.sipimessages._
 import org.knora.webapi.messages.v2.responder.SuccessResponseV2
+import org.knora.webapi.slice.admin.domain.service.Asset
 import org.knora.webapi.store.iiif.api.IIIFService
 import org.knora.webapi.store.iiif.errors.SipiException
 
@@ -56,6 +59,8 @@ case class IIIFServiceMockImpl() extends IIIFService {
   override def getTextFileRequest(textFileRequest: SipiGetTextFileRequest): Task[SipiGetTextFileResponse] = ???
 
   override def getStatus(): Task[IIIFServiceStatusResponse] = ZIO.succeed(IIIFServiceStatusOK)
+
+  override def downloadAsset(asset: Asset, targetDir: Path, user: UserADM): Task[Option[Path]] = ???
 }
 
 object IIIFServiceMockImpl {

webapi/src/main/scala/org/knora/webapi/core/LayersLive.scala

@@ -84,6 +84,8 @@ import org.knora.webapi.routing.admin.AuthenticatorService
 import org.knora.webapi.routing.admin.ProjectsRouteZ
 import org.knora.webapi.slice.admin.api.service.ProjectADMRestService
 import org.knora.webapi.slice.admin.api.service.ProjectsADMRestServiceLive
+import org.knora.webapi.slice.admin.domain.service.AssetService
+import org.knora.webapi.slice.admin.domain.service.AssetServiceLive
 import org.knora.webapi.slice.admin.domain.service.KnoraProjectRepo
 import org.knora.webapi.slice.admin.domain.service.ProjectADMService
 import org.knora.webapi.slice.admin.domain.service.ProjectADMServiceLive
@@ -129,6 +131,7 @@ object LayersLive {
       with ApiRoutes
       with AppConfig
       with AppRouter
+      with AssetService
       with Authenticator
       with CacheService
       with CacheServiceRequestMessageHandler
@@ -200,6 +203,7 @@ object LayersLive {
       ApiRoutes.layer,
       AppConfig.layer,
       AppRouter.layer,
+      AssetServiceLive.layer,
       AuthenticationMiddleware.layer,
       AuthenticatorLive.layer,
       AuthenticatorService.layer,

webapi/src/main/scala/org/knora/webapi/routing/Authenticator.scala

@@ -198,16 +198,15 @@ final case class AuthenticatorLive(
     val credentials: Option[KnoraCredentialsV2] = extractCredentialsV2(requestContext)
 
     for {
-      userADM      <- getUserADMThroughCredentialsV2(credentials)
-      userProfile   = userADM.asUserProfileV1
-      cookieDomain  = Some(appConfig.cookieDomain)
-      sessionToken <- jwtService.createToken(userProfile.userData.user_id.get)
+      userADM     <- getUserADMThroughCredentialsV2(credentials)
+      cookieDomain = Some(appConfig.cookieDomain)
+      jwt         <- jwtService.createJwt(userADM)
       httpResponse = HttpResponse(
                        headers = List(
                          headers.`Set-Cookie`(
                            HttpCookie(
                              calculateCookieName(),
-                             sessionToken,
+                             jwt.jwtString,
                              domain = cookieDomain,
                              path = Some("/"),
                              httpOnly = true
@@ -220,8 +219,8 @@ final case class AuthenticatorLive(
                          JsObject(
                            "status"      -> JsNumber(0),
                            "message"     -> JsString("credentials are OK"),
-                           "sid"         -> JsString(sessionToken),
-                           "userProfile" -> userProfile.ofType(UserProfileTypeV1.RESTRICTED).toJsValue
+                           "sid"         -> JsString(jwt.jwtString),
+                           "userProfile" -> userADM.asUserProfileV1.ofType(UserProfileTypeV1.RESTRICTED).toJsValue
                          ).compactPrint
                        )
                      )
@@ -240,14 +239,14 @@ final case class AuthenticatorLive(
       _           <- authenticateCredentialsV2(credentials = Some(credentials))
       userADM     <- getUserByIdentifier(credentials.identifier)
       cookieDomain = Some(appConfig.cookieDomain)
-      token       <- jwtService.createToken(userADM.id)
+      jwtString   <- jwtService.createJwt(userADM).map(_.jwtString)
 
       httpResponse = HttpResponse(
                        headers = List(
                          headers.`Set-Cookie`(
                            HttpCookie(
                              calculateCookieName(),
-                             token,
+                             jwtString,
                              domain = cookieDomain,
                              path = Some("/"),
                              httpOnly = true
@@ -258,7 +257,7 @@ final case class AuthenticatorLive(
                        entity = HttpEntity(
                          ContentTypes.`application/json`,
                          JsObject(
-                           "token" -> JsString(token)
+                           "token" -> JsString(jwtString)
                          ).compactPrint
                        )
                      )
@@ -753,6 +752,8 @@ object AuthenticatorLive {
     ZLayer.fromFunction(AuthenticatorLive.apply _)
 }
 
+case class Jwt(jwtString: String, expiration: Long)
+
 /**
  * Provides functions for creating, decoding, and validating JWT tokens.
  */
@@ -762,11 +763,11 @@ trait JwtService {
   /**
    * Creates a JWT.
    *
-   * @param userIri   the user IRI that will be encoded into the token.
+   * @param user the user IRI that will be encoded into the token.
    * @param content   any other content to be included in the token.
    * @return a [[String]] containing the JWT.
    */
-  def createToken(userIri: IRI, content: Map[String, JsValue] = Map.empty): Task[String]
+  def createJwt(user: UserADM, content: Map[String, JsValue] = Map.empty): UIO[Jwt]
 
   /**
    * Validates a JWT, taking the invalidation cache into account. The invalidation cache holds invalidated
@@ -797,13 +798,7 @@ final case class JwtServiceLive(private val config: AppConfig, stringFormatter:
 
   private val logger = Logger(LoggerFactory.getLogger(this.getClass))
 
-  /**
-   * Creates a JWT.
-   *
-   * @param userIri   the user IRI that will be encoded into the token.
-   * @return a [[String]] containing the JWT.
-   */
-  override def createToken(userIri: IRI, content: Map[String, JsValue] = Map.empty): Task[String] =
+  override def createJwt(user: UserADM, content: Map[String, JsValue] = Map.empty): UIO[Jwt] =
     for {
       now  <- Clock.instant
       uuid <- ZIO.random.flatMap(_.nextUUID)
@@ -812,13 +807,13 @@ final case class JwtServiceLive(private val config: AppConfig, stringFormatter:
       claim = JwtClaim(
                 content = JsObject(content).compactPrint,
                 issuer = Some(issuer),
-                subject = Some(userIri),
+                subject = Some(user.id),
                 audience = Some(Set("Knora", "Sipi")),
                 issuedAt = Some(now.getEpochSecond),
                 expiration = Some(exp),
                 jwtId = jwtId
               ).toJson
-    } yield JwtSprayJson.encode(header, claim, secret, algorithm)
+    } yield Jwt(JwtSprayJson.encode(header, claim, secret, algorithm), exp)
 
   /**
    * Validates a JWT, taking the invalidation cache into account. The invalidation cache holds invalidated

webapi/src/main/scala/org/knora/webapi/slice/admin/api/service/ProjectsADMRestService.scala

@@ -238,7 +238,7 @@ final case class ProjectsADMRestServiceLive(
     projectIri <-
       IriIdentifier.fromString(projectIri).toZIO.orElseFail(BadRequestException(s"Invalid project IRI: $projectIri"))
     project <- projectRepo.findById(projectIri).someOrFail(NotFoundException(s"Project $projectIri not found."))
-    zipFile <- projectExportService.exportProject(project)
+    zipFile <- projectExportService.exportProject(project, requestingUser)
   } yield ProjectExportResponse(zipFile.toString)
 }