feat: Make Sipi handling optional during FileValue creation (Dev-2945) (

#2960)

Co-authored-by: Balduin Landolt <33053745+BalduinLandolt@users.noreply.github.com>
Co-authored-by: Christian Kleinbölting <seakayone@users.noreply.github.com>

integration/src/test/resources/sipi/testfiles/De6XyNL4H71-D9QxghOuOPJ.info

@@ -0,0 +1,7 @@
+{
+   "originalInternalFilename": "De6XyNL4H71-D9QxghOuOPJ.png.orig",
+   "internalFilename": "De6XyNL4H71-D9QxghOuOPJ.jp2",
+   "checksumOriginal": "710aa7c06f7a7293f4cab70855ab98e166762a6b3ab7b6bdb4f6750bfc7028ed",
+   "originalFilename": "dog.png",
+   "checksumDerivative": "a89a2522adefe4510016ebdf337d82d3af0eef2a5c008c8c76553db4adb64155"
+}

integration/src/test/scala/org/knora/webapi/ITKnoraLiveSpec.scala

@@ -31,6 +31,7 @@ import org.knora.webapi.messages.store.triplestoremessages.TriplestoreJsonProtoc
 import org.knora.webapi.messages.util.rdf.JsonLDDocument
 import org.knora.webapi.messages.util.rdf.JsonLDUtil
 import org.knora.webapi.routing.UnsafeZioRun
+import org.knora.webapi.testcontainers.SipiTestContainer
 import org.knora.webapi.testservices.FileToUpload
 import org.knora.webapi.testservices.TestClientService
 import org.knora.webapi.util.LogAspect
@@ -82,6 +83,9 @@ abstract class ITKnoraLiveSpec
     config <- ZIO.service[AppConfig]
   } yield (router, config)
 
+  def copyFileToImageFolderInContainer(prefix: String, filename: String) =
+    UnsafeZioRun.runOrThrow(SipiTestContainer.copyFileToImageFolderInContainer(prefix, filename))
+
   /**
    * Create router and config by unsafe running them.
    */

integration/src/test/scala/org/knora/webapi/it/v2/KnoraSipiIntegrationV2ITSpec.scala

@@ -5,7 +5,9 @@
 
 package org.knora.webapi.it.v2
 
-import org.apache.pekko
+import org.apache.pekko.http.scaladsl.model._
+import org.apache.pekko.http.scaladsl.model.headers.BasicHttpCredentials
+import org.apache.pekko.http.scaladsl.unmarshalling.Unmarshal
 
 import java.net.URLEncoder
 import java.nio.file.Paths
@@ -30,10 +32,6 @@ import org.knora.webapi.sharedtestdata.SharedTestDataADM
 import org.knora.webapi.testservices.FileToUpload
 import org.knora.webapi.util.MutableTestIri
 
-import pekko.http.scaladsl.model._
-import pekko.http.scaladsl.model.headers.BasicHttpCredentials
-import pekko.http.scaladsl.unmarshalling.Unmarshal
-
 /**
  * Tests interaction between Knora and Sipi using Knora API v2.
  */
@@ -505,6 +503,38 @@ class KnoraSipiIntegrationV2ITSpec
       assert(savedImage.internalFilename == uploadedFile.filename)
     }
 
+    "create a resource with a still image file that has already been ingested" in {
+      copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.jp2")
+      copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.info")
+      copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.png.orig")
+      // Create the resource in the API.
+      val jsonLdEntity = UploadFileRequest
+        .make(fileType = FileType.StillImageFile(), internalFilename = "De6XyNL4H71-D9QxghOuOPJ.jp2")
+        .toJsonLd(className = Some("ThingPicture"), ontologyName = "anything")
+      val request = Post(s"$baseApiUrl/v2/resources", HttpEntity(RdfMediaTypes.`application/ld+json`, jsonLdEntity)) ~>
+        addCredentials(BasicHttpCredentials(anythingUserEmail, password)) ~>
+        addHeader("X-Asset-Ingested", "true")
+      val responseJsonDoc: JsonLDDocument = getResponseJsonLD(request)
+      // Get the resource from the API.
+      val resIri     = UnsafeZioRun.runOrThrow(responseJsonDoc.body.getRequiredIdValueAsKnoraDataIri).toString
+      val getRequest = Get(s"$baseApiUrl/v2/resources/${URLEncoder.encode(resIri, "UTF-8")}")
+      checkResponseOK(getRequest)
+    }
+
+    "not create a resource with a still image file that has already been ingested if the header is not provided" in {
+      copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.jp2")
+      copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.info")
+      copyFileToImageFolderInContainer("0001", "De6XyNL4H71-D9QxghOuOPJ.png.orig")
+      // Create the resource in the API.
+      val jsonLdEntity = UploadFileRequest
+        .make(fileType = FileType.StillImageFile(), internalFilename = "De6XyNL4H71-D9QxghOuOPJ.jp2")
+        .toJsonLd(className = Some("ThingPicture"), ontologyName = "anything")
+      val request = Post(s"$baseApiUrl/v2/resources", HttpEntity(RdfMediaTypes.`application/ld+json`, jsonLdEntity)) ~>
+        addCredentials(BasicHttpCredentials(anythingUserEmail, password)) // no X-Asset-Ingested header
+      val res = singleAwaitingRequest(request)
+      assert(res.status == StatusCodes.BadRequest)
+    }
+
     "reject an image file with the wrong file extension" in {
       val exception = intercept[BadRequestException] {
         uploadToSipi(

integration/src/test/scala/org/knora/webapi/responders/v2/ValuesResponderV2Spec.scala

@@ -5,7 +5,7 @@
 
 package org.knora.webapi.responders.v2
 
-import org.apache.pekko
+import org.apache.pekko.testkit.ImplicitSender
 
 import java.time.Instant
 import java.util.UUID
@@ -26,6 +26,7 @@ import org.knora.webapi.messages.util.DatePrecisionYear
 import org.knora.webapi.messages.util.KnoraSystemInstances
 import org.knora.webapi.messages.util.PermissionUtilADM
 import org.knora.webapi.messages.util.search.gravsearch.GravsearchParser
+import org.knora.webapi.messages.v2.responder.resourcemessages.CreateResourceRequestV2.AssetIngestState
 import org.knora.webapi.messages.v2.responder.resourcemessages._
 import org.knora.webapi.messages.v2.responder.standoffmessages._
 import org.knora.webapi.messages.v2.responder.valuemessages._
@@ -39,8 +40,6 @@ import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Select
 import org.knora.webapi.util.MutableTestIri
 import org.knora.webapi.util.ZioScalaTestUtil.assertFailsWithA
 
-import pekko.testkit.ImplicitSender
-
 /**
  * Tests [[ValuesResponderV2]].
  */
@@ -457,7 +456,8 @@ class ValuesResponderV2Spec extends CoreSpec with ImplicitSender {
       val resourceClassIri = "http://0.0.0.0:3333/ontology/0001/anything/v2#Thing".toSmartIri
       val propertyIri      = "http://0.0.0.0:3333/ontology/0001/anything/v2#hasInteger".toSmartIri
       val intVal           = IntegerValueContentV2(ApiV2Complex, 4)
-      val duplicateValue   = CreateValueV2(resourceIri, resourceClassIri, propertyIri, intVal)
+      val duplicateValue =
+        CreateValueV2(resourceIri, resourceClassIri, propertyIri, intVal, ingestState = AssetIngestState.AssetInTemp)
 
       val actual = UnsafeZioRun.run(ValuesResponderV2.createValueV2(duplicateValue, anythingUser1, randomUUID))
       assertFailsWithA[DuplicateValueException](actual)

integration/src/test/scala/org/knora/webapi/store/iiif/impl/SipiServiceMock.scala

@@ -11,6 +11,7 @@ import zio.nio.file.Path
 import org.knora.webapi.messages.admin.responder.usersmessages.UserADM
 import org.knora.webapi.messages.store.sipimessages._
 import org.knora.webapi.messages.v2.responder.SuccessResponseV2
+import org.knora.webapi.slice.admin.domain.model.KnoraProject
 import org.knora.webapi.slice.admin.domain.service.Asset
 import org.knora.webapi.store.iiif.api.FileMetadataSipiResponse
 import org.knora.webapi.store.iiif.api.SipiService
@@ -27,7 +28,7 @@ case class SipiServiceMock() extends SipiService {
    */
   private val FAILURE_FILENAME: String = "failure.jp2"
 
-  override def getFileMetadata(ignoredByMock: String): Task[FileMetadataSipiResponse] =
+  override def getFileMetadataFromTemp(filename: String): Task[FileMetadataSipiResponse] =
     ZIO.succeed(
       FileMetadataSipiResponse(
         originalFilename = Some("test2.tiff"),
@@ -62,6 +63,9 @@ case class SipiServiceMock() extends SipiService {
   override def getStatus(): Task[IIIFServiceStatusResponse] = ZIO.succeed(IIIFServiceStatusOK)
 
   override def downloadAsset(asset: Asset, targetDir: Path, user: UserADM): Task[Option[Path]] = ???
+
+  override def getFileMetadata(filename: String, shortcode: KnoraProject.Shortcode): Task[FileMetadataSipiResponse] =
+    ???
 }
 
 object SipiServiceMock {

integration/src/test/scala/org/knora/webapi/testservices/TestClientService.scala

@@ -148,6 +148,9 @@ final case class TestClientService(config: AppConfig, httpClient: CloseableHttpC
           .mapError(error =>
             throw AssertionException(s"Got HTTP ${response.status.intValue}\n REQUEST: $request, \n RESPONSE: $error")
           )
+      _ <- ZIO
+             .fail(AssertionException(s"Got HTTP ${response.status.intValue}\n REQUEST: $request, \n RESPONSE: $body"))
+             .when(response.status.isFailure())
     } yield body
 
   /**

sipi/scripts/sipi.init.lua

@@ -96,11 +96,18 @@ function pre_flight(prefix, identifier, cookie)
     end
 
     log("pre_flight - filepath: " .. filepath, server.loglevel.LOG_DEBUG)
+
     if prefix == "tmp" then
         log("pre_flight - always allow access to tmp folder", server.loglevel.LOG_DEBUG)
         return 'allow', filepath
     end
 
+    local token, error = auth_get_jwt_decoded()
+    if error == nil and token ~= nil and token["sub"] == "http://www.knora.org/ontology/knora-admin#SystemUser" then
+        log("pre_flight - always allow access for system user", server.loglevel.LOG_DEBUG)
+        return 'allow', filepath
+    end
+
     local jwt_raw = auth_get_jwt_raw()
     local permission_info = get_permission_on_file(prefix, identifier, jwt_raw)
     if permission_info == nil then

webapi/src/main/scala/org/knora/webapi/messages/v2/responder/resourcemessages/ResourceMessagesV2.scala

@@ -34,6 +34,8 @@ import org.knora.webapi.messages.util.rdf.*
 import org.knora.webapi.messages.util.standoff.StandoffTagUtilV2
 import org.knora.webapi.messages.util.standoff.XMLUtil
 import org.knora.webapi.messages.v2.responder.*
+import org.knora.webapi.messages.v2.responder.resourcemessages.CreateResourceRequestV2.AssetIngestState
+import org.knora.webapi.messages.v2.responder.resourcemessages.CreateResourceRequestV2.AssetIngestState.AssetInTemp
 import org.knora.webapi.messages.v2.responder.standoffmessages.MappingXMLtoStandoff
 import org.knora.webapi.messages.v2.responder.valuemessages.*
 import org.knora.webapi.slice.resourceinfo.domain.IriConverter
@@ -656,23 +658,31 @@ case class CreateResourceV2(
 case class CreateResourceRequestV2(
   createResource: CreateResourceV2,
   requestingUser: UserADM,
-  apiRequestID: UUID
+  apiRequestID: UUID,
+  ingestState: AssetIngestState = AssetInTemp
 ) extends ResourcesResponderRequestV2
 
 object CreateResourceRequestV2 {
+  sealed trait AssetIngestState
+  object AssetIngestState {
+    case object AssetIngested extends AssetIngestState
+    case object AssetInTemp   extends AssetIngestState
+  }
 
   /**
    * Converts JSON-LD input to a [[CreateResourceRequestV2]].
    *
    * @param jsonLDDocument       the JSON-LD input.
    * @param apiRequestID         the UUID of the API request.
    * @param requestingUser       the user making the request.
+   * @param ingestState indicates the state of the file, either ingested or in temp folder
    * @return a case class instance representing the input.
    */
   def fromJsonLd(
     jsonLDDocument: JsonLDDocument,
     apiRequestID: UUID,
-    requestingUser: UserADM
+    requestingUser: UserADM,
+    ingestState: AssetIngestState = AssetInTemp
   ): ZIO[IriConverter & SipiService & StringFormatter & MessageRelay, Throwable, CreateResourceRequestV2] =
     ZIO.serviceWithZIO[StringFormatter] { implicit stringFormatter =>
       val validationFun: (String, => Nothing) => String =
@@ -775,8 +785,12 @@ object CreateResourceRequestV2 {
                                              )
                                            )
                                        }
+                  fileInfo <- ValueContentV2
+                                .getFileInfo(projectInfoResponse.project.shortcode, ingestState, valueJsonLDObject)
+                                .option
 
-                  valueContent <- ValueContentV2.fromJsonLdObject(valueJsonLDObject, requestingUser)
+                  valueContent <-
+                    ValueContentV2.fromJsonLdObject(ingestState, valueJsonLDObject, requestingUser, fileInfo)
 
                   maybeCustomValueIri <- valueJsonLDObject.getIdValueAsKnoraDataIri
                                            .mapError(BadRequestException(_))
@@ -841,7 +855,8 @@ object CreateResourceRequestV2 {
           creationDate = creationDate
         ),
         requestingUser = maybeAttachedToUser.getOrElse(requestingUser),
-        apiRequestID = apiRequestID
+        apiRequestID = apiRequestID,
+        ingestState = ingestState
       )
     }
 }