Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(xmlupload): clean up handle media information (DEV-3086) #677

Merged
101 changes: 71 additions & 30 deletions src/dsp_tools/commands/xmlupload/resource_multimedia.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,40 +15,56 @@
logger = get_logger(__name__)


def _upload_bitstream(
def handle_media_info(
resource: XMLResource,
media_previously_uploaded: bool,
sipi_server: Sipi,
imgdir: str,
permissions_lookup: dict[str, Permissions],
) -> BitstreamInfo | None:
) -> tuple[bool, None | BitstreamInfo]:
"""
This function uploads a specified bitstream file to SIPI and then returns the file information from SIPI.
This function checks if a resource has a bitstream.
If not, it reports success and returns None.
If the file has been uploaded it returns the internal ID.
Else it uploads it and returns the internal ID.


Args:
resource: resource with that has a bitstream
resource: resource holding the bitstream
media_previously_uploaded: True if the image is already in SIPI
sipi_server: server to upload
imgdir: directory of the file
Nora-Olivia-Ammann marked this conversation as resolved.
Show resolved Hide resolved
permissions_lookup: dictionary that contains the permission name as string and the corresponding Python object

Returns:
The information from sipi which is needed to establish a link from the resource
If the bitstream could be processed successfully, then the function returns True and the new internal ID.
If there was no bitstream, it returns True and None.
If the upload was not successful, it returns False and None.
"""
pth = resource.bitstream.value # type: ignore[union-attr]
img: Optional[dict[Any, Any]] = try_network_action(
sipi_server.upload_bitstream,
filepath=str(Path(imgdir) / Path(pth)),
)
internal_file_name_bitstream = img["uploadedFiles"][0]["internalFilename"] # type: ignore[index]
return resource.get_bitstream_information(
internal_file_name_bitstream=internal_file_name_bitstream,
permissions_lookup=permissions_lookup,
)
bitstream = resource.bitstream
success = True
bitstream_information: None | BitstreamInfo = None

if not bitstream:
return success, bitstream_information
if not media_previously_uploaded:
bitstream_information = _handle_media_upload(
resource=resource,
bitstream=bitstream,
permissions_lookup=permissions_lookup,
sipi_server=sipi_server,
imgdir=imgdir,
)
if not bitstream_information:
success = False
else:
bitstream_information = resource.get_bitstream_information(bitstream.value, permissions_lookup)
return success, bitstream_information


def handle_bitstream(
def _handle_media_upload(
resource: XMLResource,
bitstream: XMLBitstream,
preprocessing_done: bool,
permissions_lookup: dict[str, Permissions],
sipi_server: Sipi,
imgdir: str,
Expand All @@ -59,7 +75,6 @@ def handle_bitstream(
Args:
resource: resource holding the bitstream
bitstream: the bitstream object
preprocessing_done: whether the preprocessing is done already
permissions_lookup: dictionary that contains the permission name as string and the corresponding Python object
sipi_server: server to upload
imgdir: directory of the file
Expand All @@ -68,22 +83,48 @@ def handle_bitstream(
The information from sipi which is needed to establish a link from the resource
"""
try:
if preprocessing_done:
resource_bitstream = resource.get_bitstream_information(bitstream.value, permissions_lookup)
else:
resource_bitstream = _upload_bitstream(
resource=resource,
sipi_server=sipi_server,
imgdir=imgdir,
permissions_lookup=permissions_lookup,
)
msg = f"Uploaded file '{bitstream.value}'"
print(f"{datetime.now()}: {msg}")
logger.info(msg)
resource_bitstream = _upload_bitstream(
resource=resource,
sipi_server=sipi_server,
imgdir=imgdir,
permissions_lookup=permissions_lookup,
)
msg = f"Uploaded file '{bitstream.value}'"
print(f"{datetime.now()}: {msg}")
logger.info(msg)
return resource_bitstream
except BaseError as err:
err_msg = err.orig_err_msg_from_api or err.message
msg = f"Unable to upload file '{bitstream.value}' of resource '{resource.label}' ({resource.id})"
print(f"{datetime.now()}: WARNING: {msg}: {err_msg}")
logger.warning(msg, exc_info=True)
return None


def _upload_bitstream(
Nora-Olivia-Ammann marked this conversation as resolved.
Show resolved Hide resolved
resource: XMLResource,
sipi_server: Sipi,
imgdir: str,
permissions_lookup: dict[str, Permissions],
) -> BitstreamInfo | None:
"""
This function uploads a specified bitstream file to SIPI and then returns the file information from SIPI.

Args:
resource: resource with that has a bitstream
sipi_server: server to upload
imgdir: directory of the file
permissions_lookup: dictionary that contains the permission name as string and the corresponding Python object

Returns:
The information from sipi which is needed to establish a link from the resource
"""
img: Optional[dict[Any, Any]] = try_network_action(
sipi_server.upload_bitstream,
filepath=str(Path(imgdir) / Path(resource.bitstream.value)), # type: ignore[union-attr]
)
internal_file_name_bitstream = img["uploadedFiles"][0]["internalFilename"] # type: ignore[index]
return resource.get_bitstream_information(
internal_file_name_bitstream=internal_file_name_bitstream,
permissions_lookup=permissions_lookup,
)
24 changes: 9 additions & 15 deletions src/dsp_tools/commands/xmlupload/xmlupload.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from dsp_tools.commands.xmlupload.project_client import ProjectClient, ProjectClientLive
from dsp_tools.commands.xmlupload.read_validate_xml_file import validate_and_parse_xml_file
from dsp_tools.commands.xmlupload.resource_create_client import ResourceCreateClient
from dsp_tools.commands.xmlupload.resource_multimedia import handle_bitstream
from dsp_tools.commands.xmlupload.resource_multimedia import handle_media_info
from dsp_tools.commands.xmlupload.stash.stash_circular_references import (
identify_circular_references,
stash_circular_references,
Expand Down Expand Up @@ -335,24 +335,18 @@ def _upload_resources(
)

for i, resource in enumerate(resources):
bitstream_information = None
if bitstream := resource.bitstream:
bitstream_information = handle_bitstream(
resource=resource,
bitstream=bitstream,
preprocessing_done=config.preprocessing_done,
permissions_lookup=permissions_lookup,
sipi_server=sipi_server,
imgdir=imgdir,
)
if not bitstream_information:
failed_uploads.append(resource.id)
continue
success, media_info = handle_media_info(
resource, config.preprocessing_done, sipi_server, imgdir, permissions_lookup
)
if not success:
failed_uploads.append(resource.id)
continue

res = _create_resource(resource, bitstream_information, resource_create_client)
res = _create_resource(resource, media_info, resource_create_client)
if not res:
failed_uploads.append(resource.id)
continue

iri, label = res
id_to_iri_resolver.update(resource.id, iri)

Expand Down