Skip to content

Commit

Permalink
Apply some heuristics to make certain datasets look more like a binder
Browse files Browse the repository at this point in the history
  • Loading branch information
Xarthisius committed Dec 13, 2019
1 parent f0cc56c commit 2b51e0d
Showing 1 changed file with 40 additions and 0 deletions.
40 changes: 40 additions & 0 deletions server/tasks/import_binder.py
Expand Up @@ -17,6 +17,8 @@
from fs.mode import Mode
from fs.path import basename
from fs.permissions import Permissions
from fs.tarfs import ReadTarFS
from fs.zipfs import ReadZipFS
from girderfs.core import WtDmsGirderFS
from girder_client import GirderClient
from girder.constants import AccessType
Expand All @@ -34,6 +36,43 @@
from ..utils import getOrCreateRootFolder


def sanitize_binder(root):
root_listdir = root.listdir("/")

if len(root_listdir) != 1:
return

single_file_or_dir = root_listdir[0]

if root.isdir(single_file_or_dir):
# moving files recursively doesn't work with webdav :(
# it should be done via simple move_fs(subdir, root)...
with root.opendir(single_file_or_dir) as subdir:
dirs = sorted([_ for _ in subdir.walk.dirs()], key=len, reverse=True)
for directory in dirs:
root.makedirs(directory, recreate=True)
for path in subdir.walk.files():
root.move("/" + single_file_or_dir + "/" + path, "/" + path)
for directory in dirs:
subdir.removedir(directory)
root.removedir("/" + single_file_or_dir)
sanitize_binder(root)

if root.isfile(single_file_or_dir):
if single_file_or_dir.endswith(".zip"):
archive_fs = ReadZipFS
elif ".tar" in root.listdir[0]:
archive_fs = ReadTarFS
else:
archive_fs = None

if archive_fs is not None:
with archive_fs(root.openbin(single_file_or_dir)) as archive:
copy_fs(archive, root)
root.remove("/" + single_file_or_dir)
sanitize_binder(root)


def run(job):
jobModel = Job()
jobModel.updateJob(job, status=JobStatus.RUNNING)
Expand Down Expand Up @@ -120,6 +159,7 @@ def run(job):
str(session["_id"]), girder_root + "/api/v1", str(token["_id"])
) as source_fs:
copy_fs(source_fs, destination_fs)
sanitize_binder(destination_fs)

Session().deleteSession(user, session)

Expand Down

0 comments on commit 2b51e0d

Please sign in to comment.