Skip to content

Commit

Permalink
Merge branch 'RESTAPI-1234-deref-compress' into 'master'
Browse files Browse the repository at this point in the history 
Add dereference option in compress endpoints

See merge request firecrest/firecrest!305
  • Loading branch information
@ekouts
ekouts committed Jun 13, 2024
2 parents ee65502 + fca015f commit a6d9e3c
Show file tree
Hide file tree
Showing 4 changed files with 36 additions and 12 deletions.
8 changes: 8 additions & 0 deletions doc/openapi/firecrest-api.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1366,6 +1366,10 @@ paths:
targetPath:
type: string
description: Absolute filesystem path of the compressed file
dereference:
type: boolean
description: Follow symbolic links
default: false
required:
- sourcePath
- targetPath
Expand Down Expand Up @@ -2327,6 +2331,10 @@ paths:
targetPath:
type: string
description: Absolute path to destination
dereference:
type: boolean
description: Follow symbolic links
default: false
jobName:
type: string
description: job name the copy operation
Expand Down
8 changes: 8 additions & 0 deletions doc/openapi/firecrest-developers-api.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1354,6 +1354,10 @@ paths:
targetPath:
type: string
description: Absolute filesystem path of the compressed file
dereference:
type: boolean
description: Follow symbolic links
default: false
required:
- sourcePath
- targetPath
Expand Down Expand Up @@ -2315,6 +2319,10 @@ paths:
targetPath:
type: string
description: Absolute path to destination
dereference:
type: boolean
description: Follow symbolic links
default: false
jobName:
type: string
description: job name the copy operation
Expand Down
14 changes: 9 additions & 5 deletions src/storage/storage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,15 @@

# Internal microservices communication
## certificator
CERTIFICATOR_HOST = os.environ.get("F7T_CERTIFICATOR_HOST","127.0.0.1")
CERTIFICATOR_HOST = os.environ.get("F7T_CERTIFICATOR_HOST","127.0.0.1")
CERTIFICATOR_PORT = os.environ.get("F7T_CERTIFICATOR_PORT","5000")
CERTIFICATOR_URL = f"{F7T_SCHEME_PROTOCOL}://{CERTIFICATOR_HOST}:{CERTIFICATOR_PORT}"
## tasks
TASKS_HOST = os.environ.get("F7T_TASKS_HOST","127.0.0.1")
TASKS_HOST = os.environ.get("F7T_TASKS_HOST","127.0.0.1")
TASKS_PORT = os.environ.get("F7T_TASKS_PORT","5003")
TASKS_URL = f"{F7T_SCHEME_PROTOCOL}://{TASKS_HOST}:{TASKS_PORT}"
## compute
COMPUTE_HOST = os.environ.get("F7T_COMPUTE_HOST","127.0.0.1")
COMPUTE_HOST = os.environ.get("F7T_COMPUTE_HOST","127.0.0.1")
COMPUTE_PORT = os.environ.get("F7T_COMPUTE_PORT","5006")
COMPUTE_URL = f"{F7T_SCHEME_PROTOCOL}://{COMPUTE_HOST}:{COMPUTE_PORT}"

Expand Down Expand Up @@ -266,7 +266,7 @@ def create_staging():
# For S3:
# For S3:
S3_PRIVATE_URL = os.environ.get("F7T_S3_PRIVATE_URL")
# For S3:
# For S3:
S3_PRIVATE_URL = os.environ.get("F7T_S3_PRIVATE_URL")
S3_PUBLIC_URL = os.environ.get("F7T_S3_PUBLIC_URL")
S3_PRIVATE_URL = os.environ.get("F7T_S3_PRIVATE_URL", S3_PUBLIC_URL)
Expand Down Expand Up @@ -1026,7 +1026,11 @@ def internal_operation(request, command):
elif command == "compress":
basedir = os.path.dirname(sourcePath)
file_path = os.path.basename(sourcePath)
actual_command = f"tar -czf '{targetPath}' -C '{basedir}' '{file_path}'"
deref = ""
if get_boolean_var(request.form.get("dereference", False)):
deref = "--dereference"

actual_command = f"tar {deref} -czf '{targetPath}' -C '{basedir}' '{file_path}'"
else:
extraction_type = request.form.get("type", "auto")
actual_command = extract_command(sourcePath, targetPath, type=extraction_type)
Expand Down
18 changes: 11 additions & 7 deletions src/utilities/utilities.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@

# Internal microservices communication
## certificator
CERTIFICATOR_HOST = os.environ.get("F7T_CERTIFICATOR_HOST","127.0.0.1")
CERTIFICATOR_HOST = os.environ.get("F7T_CERTIFICATOR_HOST","127.0.0.1")
CERTIFICATOR_PORT = os.environ.get("F7T_CERTIFICATOR_PORT","5000")
CERTIFICATOR_URL = f"{F7T_SCHEME_PROTOCOL}://{CERTIFICATOR_HOST}:{CERTIFICATOR_PORT}"

Expand Down Expand Up @@ -180,7 +180,7 @@ def ls_parse_folder(folder_content:str,path:str=""):
file_pattern = (r'^(?P<type>\S)(?P<permissions>\S+)\s+\d+\s+(?P<user>\S+)\s+'
r'(?P<group>\S+)\s+(?P<size>\d+)\s+(?P<last_modified>(\d|-|T|:)+)\s+(?P<filename>.+)$')
matches = re.finditer(file_pattern, folder_content, re.MULTILINE)

for m in matches:
tokens = shlex.split(m.group("filename"))
if len(tokens) == 1:
Expand Down Expand Up @@ -220,7 +220,7 @@ def ls_parse(request, retval):
# total 1
# -rw-rw-r-- 1 username groupname 0 2023-07-24T11:45:35 "file_in_folder.txt"
# ...

def remove_prefix(text, prefix):
return text[text.startswith(prefix) and len(prefix):]

Expand All @@ -238,7 +238,7 @@ def remove_prefix(text, prefix):
else:
file_list += ls_parse_folder(retval["msg"])


totalSize = len(file_list)
logging.info(f"Length of file list: {len(file_list)}")

Expand Down Expand Up @@ -453,7 +453,11 @@ def common_fs_operation(request, command):
elif command == "compress":
basedir = os.path.dirname(sourcePath)
file_path = os.path.basename(sourcePath)
action = f"tar -czvf '{targetPath}' -C '{basedir}' '{file_path}'"
deref = ""
if get_boolean_var(request.form.get("dereference", False)):
deref = "--dereference"

action = f"tar {deref} -czvf '{targetPath}' -C '{basedir}' '{file_path}'"
success_code = 201
elif command == "extract":
extraction_type = request.form.get("type", "auto")
Expand All @@ -469,7 +473,7 @@ def common_fs_operation(request, command):
opt = ""
bytes = request.args.get("bytes", None)
lines = request.args.get("lines", None)
if command == "head":
if command == "head":
reverse_mode = get_boolean_var(request.args.get("skip_ending", None))
else:
reverse_mode = get_boolean_var(request.args.get("skip_beginning", None))
Expand Down Expand Up @@ -502,7 +506,7 @@ def common_fs_operation(request, command):
opt = f" --lines='+{lines}' "
else:
opt = f" --lines='{lines}' "

grep_exp=request.args.get("grep", None)
if grep_exp:
grep = f"| grep --fixed-strings --regexp='{grep_exp}'"
Expand Down

0 comments on commit a6d9e3c

Please sign in to comment.