Skip to content

Commit

Permalink
use dsp ingest to get the metadata in case of AssetIngested
Browse files Browse the repository at this point in the history
  • Loading branch information
seakayone committed Dec 22, 2023
1 parent c39bf55 commit e16e2ce
Show file tree
Hide file tree
Showing 7 changed files with 147 additions and 129 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ object LayersTest {
with StringFormatter
](
AppConfigForTestContainers.testcontainers,
DspIngestClientLive.layer,
FusekiTestContainer.layer,
SipiTestContainer.layer,
SipiServiceLive.layer,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import zio.nio.file.Path
import org.knora.webapi.messages.admin.responder.usersmessages.UserADM
import org.knora.webapi.messages.store.sipimessages._
import org.knora.webapi.messages.v2.responder.SuccessResponseV2
import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.AssetId
import org.knora.webapi.slice.admin.domain.model.KnoraProject
import org.knora.webapi.slice.admin.domain.service.Asset
import org.knora.webapi.store.iiif.api.FileMetadataSipiResponse
Expand All @@ -28,7 +29,7 @@ case class SipiServiceMock() extends SipiService {
*/
private val FAILURE_FILENAME: String = "failure.jp2"

override def getFileMetadataFromTemp(filename: String): Task[FileMetadataSipiResponse] =
override def getFileMetadataFromSipiTemp(filename: String): Task[FileMetadataSipiResponse] =
ZIO.succeed(
FileMetadataSipiResponse(
originalFilename = Some("test2.tiff"),
Expand Down Expand Up @@ -64,8 +65,10 @@ case class SipiServiceMock() extends SipiService {

override def downloadAsset(asset: Asset, targetDir: Path, user: UserADM): Task[Option[Path]] = ???

override def getFileMetadata(filename: String, shortcode: KnoraProject.Shortcode): Task[FileMetadataSipiResponse] =
???
override def getFileMetadataFromDspIngestApi(
shortcode: KnoraProject.Shortcode,
assetId: AssetId
): Task[FileMetadataSipiResponse] = ???

Check warning on line 71 in integration/src/test/scala/org/knora/webapi/store/iiif/impl/SipiServiceMock.scala

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

integration/src/test/scala/org/knora/webapi/store/iiif/impl/SipiServiceMock.scala#L71

Usage of ??? operator.
}

object SipiServiceMock {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ import org.knora.webapi.messages.v2.responder.resourcemessages.ReadResourceV2
import org.knora.webapi.messages.v2.responder.standoffmessages.*
import org.knora.webapi.routing.RouteUtilV2
import org.knora.webapi.routing.RouteUtilZ
import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.AssetId
import org.knora.webapi.slice.admin.domain.model.KnoraProject.Shortcode
import org.knora.webapi.slice.resourceinfo.domain.IriConverter
import org.knora.webapi.store.iiif.api.FileMetadataSipiResponse
Expand Down Expand Up @@ -1055,7 +1056,6 @@ object ValueContentV2 {
/**
* Converts a JSON-LD object to a [[ValueContentV2]].
*
* @param ingestState indicates the state of the file, either ingested or in temp folder
* @param jsonLdObject the JSON-LD object.
* @param requestingUser the user making the request.
* @return a [[ValueContentV2]].
Expand Down Expand Up @@ -1134,17 +1134,18 @@ object ValueContentV2 {
jsonLdObject: JsonLDObject
): ZIO[SipiService, Throwable, FileInfo] =
for {
internalFilename <- ZIO.attempt {
val validationFun: (IRI, => Nothing) => IRI =
(s, errorFun) => Iri.toSparqlEncodedString(s).getOrElse(errorFun)
jsonLdObject.requireStringWithValidation(FileValueHasFilename, validationFun)
}
internalFilename <- {
val fileNameEncoded = jsonLdObject
.getRequiredString(FileValueHasFilename)
.flatMap(it => Iri.toSparqlEncodedString(it).toRight(s"$FileValueHasFilename is invalid."))
ZIO.fromEither(fileNameEncoded).mapError(BadRequestException(_))
}
metadata <- ingestState match {
case AssetIngestState.AssetIngested =>
SipiService.getFileMetadata(internalFilename, Shortcode.unsafeFrom(shortcode))
case AssetIngestState.AssetInTemp => SipiService.getFileMetadataFromTemp(internalFilename)
val assetId = AssetId.unsafeFrom(internalFilename.substring(0, internalFilename.indexOf('.')))
SipiService.getFileMetadataFromDspIngestApi(Shortcode.unsafeFrom(shortcode), assetId)
case AssetIngestState.AssetInTemp => SipiService.getFileMetadataFromSipiTemp(internalFilename)
}

} yield FileInfo(internalFilename, metadata)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import zio.nio.file.Path
import org.knora.webapi.messages.admin.responder.usersmessages.UserADM
import org.knora.webapi.messages.store.sipimessages.*
import org.knora.webapi.messages.v2.responder.SuccessResponseV2
import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.AssetId
import org.knora.webapi.slice.admin.domain.model.KnoraProject.Shortcode
import org.knora.webapi.slice.admin.domain.service.Asset
import org.knora.webapi.store.iiif.errors.SipiException
Expand Down Expand Up @@ -68,16 +69,16 @@ trait SipiService {
* @param filename the path to the file.
* @return a [[FileMetadataSipiResponse]] containing the requested metadata.
*/
def getFileMetadataFromTemp(filename: String): Task[FileMetadataSipiResponse]
def getFileMetadataFromSipiTemp(filename: String): Task[FileMetadataSipiResponse]

/**
* Asks Sipi for metadata about a file in permanent location, served from the 'knora.json' route.
* Asks dsp-ingest for metadata about a file in permanent location, served from the 'knora.json' route.
*
* @param filename the path to the file.
* @param shortcode the shortcode of the project.
* @param assetId for the file.
* @return a [[FileMetadataSipiResponse]] containing the requested metadata.
*/
def getFileMetadata(filename: String, shortcode: Shortcode): Task[FileMetadataSipiResponse]
def getFileMetadataFromDspIngestApi(shortcode: Shortcode, assetId: AssetId): Task[FileMetadataSipiResponse]

/**
* Asks Sipi to move a file from temporary storage to permanent storage.
Expand Down Expand Up @@ -111,10 +112,12 @@ trait SipiService {

/**
* Downloads an asset from Sipi.
* @param asset The asset to download.
*
* @param asset The asset to download.
* @param targetDir The target directory in which the asset should be stored.
* @param user The user who is downloading the asset.
* @param user The user who is downloading the asset.
* @return The path to the downloaded asset. If the asset could not be downloaded, [[None]] is returned.
*/
def downloadAsset(asset: Asset, targetDir: Path, user: UserADM): Task[Option[Path]]

}
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,10 @@ import org.knora.webapi.messages.util.KnoraSystemInstances
import org.knora.webapi.messages.v2.responder.SuccessResponseV2
import org.knora.webapi.routing.Jwt
import org.knora.webapi.routing.JwtService
import org.knora.webapi.slice.admin.domain.model.KnoraProject
import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.AssetId
import org.knora.webapi.slice.admin.domain.model.KnoraProject.Shortcode
import org.knora.webapi.slice.admin.domain.service.Asset
import org.knora.webapi.slice.admin.domain.service.DspIngestClient
import org.knora.webapi.store.iiif.api.FileMetadataSipiResponse
import org.knora.webapi.store.iiif.api.SipiService
import org.knora.webapi.store.iiif.errors.SipiException
Expand All @@ -58,7 +60,8 @@ import org.knora.webapi.util.ZScopedJavaIoStreams
final case class SipiServiceLive(
private val sipiConfig: Sipi,
private val jwtService: JwtService,
private val httpClient: CloseableHttpClient
private val httpClient: CloseableHttpClient,
private val dspIngestClient: DspIngestClient
) extends SipiService {

private object SipiRoutes {
Expand All @@ -75,30 +78,31 @@ final case class SipiServiceLive(
* @param filename the file name
* @return a [[FileMetadataSipiResponse]] containing the requested metadata.
*/
override def getFileMetadataFromTemp(filename: String): Task[FileMetadataSipiResponse] =
getFileMetadataFromUrl(s"${sipiConfig.internalBaseUrl}/tmp/$filename/knora.json")

/**
* Asks Sipi for metadata about a file in permanent location, served from the 'knora.json' route.
*
* @param filename the path to the file.
* @param shortcode the shortcode of the project.
* @return a [[FileMetadataSipiResponse]] containing the requested metadata.
*/
override def getFileMetadata(filename: String, shortcode: KnoraProject.Shortcode): Task[FileMetadataSipiResponse] =
getFileMetadataFromUrl(s"${sipiConfig.internalBaseUrl}/${shortcode.value}/$filename/knora.json")

private def getFileMetadataFromUrl(url: String): Task[FileMetadataSipiResponse] =
override def getFileMetadataFromSipiTemp(filename: String): Task[FileMetadataSipiResponse] =
for {
jwt <- jwtService.createJwt(KnoraSystemInstances.Users.SystemUser)
request = new HttpGet(url)
request = new HttpGet(s"${sipiConfig.internalBaseUrl}/tmp/$filename/knora.json")

Check warning on line 84 in webapi/src/main/scala/org/knora/webapi/store/iiif/impl/SipiServiceLive.scala

View check run for this annotation

Codecov / codecov/patch

webapi/src/main/scala/org/knora/webapi/store/iiif/impl/SipiServiceLive.scala#L84

Added line #L84 was not covered by tests
_ = request.addHeader(new BasicHeader("Authorization", s"Bearer ${jwt.jwtString}"))
bodyStr <- doSipiRequest(request)
res <- ZIO
.fromEither(bodyStr.fromJson[FileMetadataSipiResponse])
.mapError(e => SipiException(s"Invalid response from Sipi: $e, $bodyStr"))
} yield res

override def getFileMetadataFromDspIngestApi(shortcode: Shortcode, assetId: AssetId): Task[FileMetadataSipiResponse] =
for {
response <- dspIngestClient.getAssetInfo(shortcode, assetId)
} yield FileMetadataSipiResponse(
Some(response.originalFilename),
response.originalMimeType,
response.internalMimeType.getOrElse("application/octet-stream"),
response.width,
response.height,
None,
response.duration.map(BigDecimal(_)),
response.fps.map(BigDecimal(_))

Check warning on line 103 in webapi/src/main/scala/org/knora/webapi/store/iiif/impl/SipiServiceLive.scala

View check run for this annotation

Codecov / codecov/patch

webapi/src/main/scala/org/knora/webapi/store/iiif/impl/SipiServiceLive.scala#L94-L103

Added lines #L94 - L103 were not covered by tests
)

/**
* Asks Sipi to move a file from temporary storage to permanent storage.
*
Expand Down Expand Up @@ -408,12 +412,13 @@ object SipiServiceLive {
private def release(httpClient: CloseableHttpClient): UIO[Unit] =
ZIO.attemptBlocking(httpClient.close()).logError.ignore <* ZIO.logInfo(">>> Release Sipi IIIF Service <<<")

val layer: URLayer[AppConfig & JwtService, SipiService] =
val layer: URLayer[AppConfig & DspIngestClient & JwtService, SipiService] =
ZLayer.scoped {
for {
config <- ZIO.serviceWith[AppConfig](_.sipi)
jwtService <- ZIO.service[JwtService]
httpClient <- ZIO.acquireRelease(acquire(config))(release)
} yield SipiServiceLive(config, jwtService, httpClient)
config <- ZIO.serviceWith[AppConfig](_.sipi)
jwtService <- ZIO.service[JwtService]
httpClient <- ZIO.acquireRelease(acquire(config))(release)
dspIngestClient <- ZIO.service[DspIngestClient]
} yield SipiServiceLive(config, jwtService, httpClient, dspIngestClient)

Check warning on line 422 in webapi/src/main/scala/org/knora/webapi/store/iiif/impl/SipiServiceLive.scala

View check run for this annotation

Codecov / codecov/patch

webapi/src/main/scala/org/knora/webapi/store/iiif/impl/SipiServiceLive.scala#L418-L422

Added lines #L418 - L422 were not covered by tests
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*
* Copyright © 2021 - 2023 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors.
* SPDX-License-Identifier: Apache-2.0
*/

package org.knora.webapi.messages.v2.responder.valuemessages

import zio.*
import zio.nio.file.Path
import zio.test.*

import dsp.errors.AssertionException
import org.knora.webapi.messages.admin.responder.usersmessages.UserADM
import org.knora.webapi.messages.store.sipimessages.*
import org.knora.webapi.messages.util.rdf.JsonLDUtil
import org.knora.webapi.messages.v2.responder.SuccessResponseV2
import org.knora.webapi.messages.v2.responder.resourcemessages.CreateResourceRequestV2.AssetIngestState
import org.knora.webapi.messages.v2.responder.resourcemessages.CreateResourceRequestV2.AssetIngestState.AssetInTemp
import org.knora.webapi.messages.v2.responder.resourcemessages.CreateResourceRequestV2.AssetIngestState.AssetIngested
import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.AssetId
import org.knora.webapi.slice.admin.domain.model.KnoraProject
import org.knora.webapi.slice.admin.domain.service.Asset
import org.knora.webapi.store.iiif.api.FileMetadataSipiResponse
import org.knora.webapi.store.iiif.api.SipiService

object ValueContentV2Spec extends ZIOSpecDefault {

private val assetId = AssetId.unsafeFrom("4sAf4AmPeeg-ZjDn3Tot1Zt")

private val jsonLd = JsonLDUtil
.parseJsonLD(s"{\"http://api.knora.org/ontology/knora-api/v2#fileValueHasFilename\" : \"$assetId.txt\"}")
.body

private val expected: FileMetadataSipiResponse =
FileMetadataSipiResponse(Some("originalFilename"), None, "", None, None, None, None, None)

override def spec: Spec[Any, Throwable] =
suite("ValueContentV2.getFileInfo")(
suite("Given the asset is present in the tmp folder of Sipi")(
test("When getting file metadata with AssetInTemp from Sipi, then it should succeed") {
for {
temp <- ValueContentV2.getFileInfo("0001", AssetInTemp, jsonLd)
} yield assertTrue(temp.metadata == expected)
},
test("When getting file metadata with AssetIngested from dsp-ingest, then it should fail") {
for {
ingested <- ValueContentV2.getFileInfo("0001", AssetIngested, jsonLd).exit
} yield assertTrue(ingested.isFailure)
}
).provide(mockSipi(AssetInTemp)),
suite("Given the asset is ingested")(
test("When getting file metadata with AssetInTemp from Sipi, then it should fail") {
for {
temp <- ValueContentV2.getFileInfo("0001", AssetInTemp, jsonLd).exit
} yield assertTrue(temp.isFailure)
},
test("When getting file metadata with AssetIngested from dsp-ingest, then it should succeed") {
for {
ingested <- ValueContentV2.getFileInfo("0001", AssetIngested, jsonLd)
} yield assertTrue(ingested.metadata == expected)
}
).provide(mockSipi(AssetIngested))
)

private def mockSipi(flag: AssetIngestState) = ZLayer.succeed(new SipiService {

override def getFileMetadataFromSipiTemp(filename: String): Task[FileMetadataSipiResponse] =
if (flag == AssetInTemp) { ZIO.succeed(expected) }
else { ZIO.fail(AssertionException("fail")) }

override def getFileMetadataFromDspIngestApi(
shortcode: KnoraProject.Shortcode,
assetId: AssetId
): Task[FileMetadataSipiResponse] =
if (flag == AssetIngested) { ZIO.succeed(expected) }
else { ZIO.fail(AssertionException("fail")) }

// The following are unsupported operations because they are not used in the test
def moveTemporaryFileToPermanentStorage(
moveTemporaryFileToPermanentStorageRequestV2: MoveTemporaryFileToPermanentStorageRequest
): Task[SuccessResponseV2] =
ZIO.dieMessage("unsupported operation")
def deleteTemporaryFile(
deleteTemporaryFileRequestV2: DeleteTemporaryFileRequest
): Task[SuccessResponseV2] =
ZIO.dieMessage("unsupported operation")
def getTextFileRequest(textFileRequest: SipiGetTextFileRequest): Task[SipiGetTextFileResponse] =
ZIO.dieMessage("unsupported operation")
def getStatus(): Task[IIIFServiceStatusResponse] =
ZIO.dieMessage("unsupported operation")
def downloadAsset(asset: Asset, targetDir: Path, user: UserADM): Task[Option[Path]] =
ZIO.dieMessage("unsupported operation")
})
}

This file was deleted.

0 comments on commit e16e2ce

Please sign in to comment.