Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set project as busy / not busy when uploading (1) #525

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,3 +182,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe
## Sprint (2022-09-02 - 2022-09-16)

- Add storage usage information in the Units listing table for Super Admin ([#523](https://github.com/ScilifelabDataCentre/dds_cli/pull/523))
- Set project as busy / not busy when starting / finishing an upload ([#525](https://github.com/ScilifelabDataCentre/dds_cli/pull/525))
1 change: 1 addition & 0 deletions dds_cli/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ class DDSEndpoint:

# Project specific urls
PROJ_ACCESS = BASE_ENDPOINT + "/proj/access"
PROJ_BUSY = BASE_ENDPOINT + "/proj/busy"

# Listing urls
LIST_PROJ = BASE_ENDPOINT + "/proj/list"
Expand Down
38 changes: 37 additions & 1 deletion dds_cli/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import logging
import os
import pathlib
import typing

# Installed
import http
Expand Down Expand Up @@ -121,8 +122,12 @@ def __enter__(self):
return self

def __exit__(self, exc_type, exc_value, tb, max_fileerrs: int = 40):
"""Finish and print out delivery summary."""
"""Finish and print out delivery summary.

This is not entered if there's an error during __init__.
"""
if self.method in ["put", "get"]:
self.cleanup_busy_status()
self.__printout_delivery_summary()

# Exception is not handled
Expand All @@ -132,6 +137,37 @@ def __exit__(self, exc_type, exc_value, tb, max_fileerrs: int = 40):

return True

# Public methods ################################# Public methods #

def cleanup_busy_status(self):
"""Reset busy to False."""
# Set project to busy
set_to_not_busy: bool = self.change_busy_status(busy=False)
if not set_to_not_busy:
LOG.warning(
"Failed to clear busy status. New actions in project "
f"'{self.project}' may be blocked. Contact the responsible unit."
)
LOG.debug(f"Project '{self.project}' busy status reset: {set_to_not_busy}")

def change_busy_status(self, busy: bool) -> bool:
"""Set project as busy."""
response, _ = dds_cli.utils.perform_request(
endpoint=DDSEndpoint.PROJ_BUSY,
method="put",
headers=self.token,
params={"project": self.project},
json={"busy": busy},
error_message="Failed setting project as busy.",
)
LOG.debug(
response.get(
"message",
"No message was returned from the ProjectBusy endpoint, there's an error somewhere.",
)
)
return response.get("ok", False)

# Private methods ############################### Private methods #
def __get_safespring_keys(self):
"""Get safespring keys."""
Expand Down
88 changes: 51 additions & 37 deletions dds_cli/data_putter.py
Original file line number Diff line number Diff line change
Expand Up @@ -208,49 +208,63 @@ def __init__(
if self.method != "put":
raise exceptions.AuthenticationError(f"Unauthorized method: '{self.method}'")

# Start file prep progress
with Progress(
"[bold]{task.description}",
SpinnerColumn(spinner_name="dots12", style="white"),
console=dds_cli.utils.stderr_console,
) as progress:
# Spinner while collecting file info
wait_task = progress.add_task("Collecting and preparing data", step="prepare")

# Get file info
self.filehandler = fhl.LocalFileHandler(
user_input=(source, source_path_file),
project=self.project,
temporary_destination=self.dds_directory.directories["FILES"],
# Set project to busy
set_to_busy: bool = self.change_busy_status(busy=True)
if not set_to_busy:
raise exceptions.DDSCLIException(
message=(
"Cannot upload data at this time: "
f"The '{self.project}' is currently busy with another task."
)
)
LOG.debug(f"Project '{self.project}' set to busy: {set_to_busy}")
try:
# Start file prep progress
with Progress(
"[bold]{task.description}",
SpinnerColumn(spinner_name="dots12", style="white"),
console=dds_cli.utils.stderr_console,
) as progress:
# Spinner while collecting file info
wait_task = progress.add_task("Collecting and preparing data", step="prepare")

# Get file info
self.filehandler = fhl.LocalFileHandler(
user_input=(source, source_path_file),
project=self.project,
temporary_destination=self.dds_directory.directories["FILES"],
)

# Verify that the Safespring S3 bucket exists
# self.verify_bucket_exist()

# Check which, if any, files exist in the db
files_in_db = self.filehandler.check_previous_upload(token=self.token)
# Verify that the Safespring S3 bucket exists
# self.verify_bucket_exist()

# Quit if error and flag
if files_in_db and self.break_on_fail and not self.overwrite:
raise exceptions.UploadError(
"Some files have already been uploaded (or have identical names to "
"previously uploaded files) and the '--break-on-fail' flag was used. "
"Try again with the '--overwrite' flag if you want to upload these files."
)
# Check which, if any, files exist in the db
files_in_db = self.filehandler.check_previous_upload(token=self.token)

# Generate status dict
self.status = self.filehandler.create_upload_status_dict(
existing_files=files_in_db, overwrite=self.overwrite
)
# Quit if error and flag
if files_in_db and self.break_on_fail and not self.overwrite:
raise exceptions.UploadError(
"Some files have already been uploaded (or have identical names to "
"previously uploaded files) and the '--break-on-fail' flag was used. "
"Try again with the '--overwrite' flag if you want to upload these files."
)

# Remove spinner
progress.remove_task(wait_task)
# Generate status dict
self.status = self.filehandler.create_upload_status_dict(
existing_files=files_in_db, overwrite=self.overwrite
)

if not self.filehandler.data:
if self.temporary_directory and self.temporary_directory.is_dir():
LOG.debug(f"Deleting temporary folder {self.temporary_directory}.")
dds_cli.utils.delete_folder(self.temporary_directory)
raise exceptions.UploadError("No data to upload.")
# Remove spinner
progress.remove_task(wait_task)

if not self.filehandler.data:
if self.temporary_directory and self.temporary_directory.is_dir():
LOG.debug(f"Deleting temporary folder {self.temporary_directory}.")
dds_cli.utils.delete_folder(self.temporary_directory)
raise exceptions.UploadError("No data to upload.")
except:
self.cleanup_busy_status()
raise

# Public methods ###################### Public methods #
@verify_proceed
Expand Down