Add OpenTTD#22: allow uploading .zip / .tar / .tar.gz additional to i…

…ndividual files

Theze archives are extracted and the individual files are added
to the list. Folder-structure is maintained.

The most common way to make tar-balls is to have a single folder
as root-folder. Therefor, this folder is skipped (but only if
there is exactly one folder and no files in the root).

bananas_api/new_upload/extract.py

@@ -0,0 +1,120 @@
+import tarfile
+import os
+import secrets
+import zipfile
+
+TAR_STORAGE_PATH = "data/tar"
+
+
+def _find_root_folder(info_list, get_name=None, is_file=None):
+    """
+    Tar-files are often made of a whole folder. This means that we would
+    prefix all files with that folder name, which in 99% of the cases
+    won't be the expected outcome. So first do a scan, to see if there
+    are any files or more than one directory in the root folder. If not,
+    skip the root-folder while extracting.
+
+    To keep things more the same, do this also for any other format.
+    """
+
+    root_folder = None
+
+    for info in info_list:
+        if not is_file(info):
+            continue
+
+        if root_folder is None:
+            root_folder = get_name(info).split("/")[0]
+
+        if not get_name(info).startswith(f"{root_folder}/"):
+            root_folder = None
+            break
+
+    return root_folder
+
+
+def _extact_files(info_list, root_folder, extractor, extractor_kwargs, get_name=None, set_name=None, is_file=None):
+    files = []
+
+    for info in info_list:
+        if not is_file(info):
+            continue
+
+        # Chance on collision is really low, but would be really annoying. So
+        # simply protect against it by looking for an unused UUID.
+        uuid = secrets.token_hex(16)
+        while os.path.isfile(os.path.join(TAR_STORAGE_PATH, uuid)):
+            uuid = secrets.token_hex(16)
+
+        internal_filename = os.path.join(TAR_STORAGE_PATH, uuid)
+
+        new_file = {
+            "uuid": uuid,
+            "filename": get_name(info),
+            "internal_filename": internal_filename,
+            "errors": [],
+        }
+
+        # Remove the root-folder from the filename if needed.
+        if root_folder:
+            new_file["filename"] = new_file["filename"][len(root_folder) + 1:]
+
+        # Change the filename and extract to it; this flattens everything,
+        # which means we won't have empty folders to deal with.
+        set_name(info, uuid)
+        extractor.extract(info, TAR_STORAGE_PATH, **extractor_kwargs)
+
+        new_file["filesize"] = os.stat(internal_filename).st_size
+        files.append(new_file)
+
+    return files
+
+
+def extract_tarball(file_info):
+    def set_tar_name(info, value):
+        info.name = value
+
+    files = []
+
+    with tarfile.open(file_info["internal_filename"]) as tar:
+        root_folder = _find_root_folder(
+            tar,
+            get_name=lambda info: info.name,
+            is_file=lambda info: info.isfile(),
+        )
+
+        files = _extact_files(
+            tar,
+            root_folder,
+            extractor=tar,
+            extractor_kwargs={"set_attrs": False},
+            get_name=lambda info: info.name,
+            set_name=set_tar_name,
+            is_file=lambda info: info.isfile(),
+        )
+
+    return files
+
+
+def extract_zip(file_info):
+    def set_zip_name(info, value):
+        info.filename = value
+
+    with zipfile.ZipFile(file_info["internal_filename"]) as zip:
+        root_folder = _find_root_folder(
+            zip.infolist(),
+            get_name=lambda info: info.filename,
+            is_file=lambda info: not info.is_dir(),
+        )
+
+        files = _extact_files(
+            zip.infolist(),
+            root_folder,
+            extractor=zip,
+            extractor_kwargs={},
+            get_name=lambda info: info.filename,
+            set_name=set_zip_name,
+            is_file=lambda info: not info.is_dir(),
+        )
+
+    return files

bananas_api/new_upload/session.py

@@ -4,9 +4,16 @@
 import os
 import secrets
 
-from collections import defaultdict
+from collections import (
+    Counter,
+    defaultdict,
+)
 
 from .exceptions import ValidationException
+from .extract import (
+    extract_tarball,
+    extract_zip,
+)
 from .session_publish import (
     create_package,
     create_tarball,
@@ -50,7 +57,8 @@ def cleanup_session(session):
 
     for file_info in session["files"]:
         os.unlink(file_info["internal_filename"])
-        os.unlink(f"{file_info['internal_filename']}.info")
+        if not file_info["internal_filename"].startswith("data/tar/"):
+            os.unlink(f"{file_info['internal_filename']}.info")
 
     del _tokens[session["token"]]
     del _sessions[session["user"].full_id]
@@ -224,7 +232,16 @@ def add_file(session, uuid, filename, filesize, internal_filename, announcing=Fa
         if uuid in session["announced-files"]:
             del session["announced-files"][uuid]
 
-        session["files"].append(new_file)
+        if filename.endswith((".tar", ".tar.gz")):
+            session["files"].extend(extract_tarball(new_file))
+            os.unlink(new_file["internal_filename"])
+            os.unlink(f"{new_file['internal_filename']}.info")
+        elif filename.endswith((".zip")):
+            session["files"].extend(extract_zip(new_file))
+            os.unlink(new_file["internal_filename"])
+            os.unlink(f"{new_file['internal_filename']}.info")
+        else:
+            session["files"].append(new_file)
 
 
 def update_session(session, data):

regression/200_success_ai_and_utf8.yaml

@@ -1,9 +1,7 @@
 steps:
 - api: user/login
 - api: new-package/start
-- file-upload: main.nut
-- file-upload: info-ai.nut
-  name: info.nut
+- file-upload: ai.tar
 - file-upload: utf8_with_bom.nut
 - api: new-package/update
   name: "test"

regression/211_missing_main.yaml

@@ -1,7 +1,7 @@
 steps:
 - api: user/login
 - api: new-package/start
-- file-upload: info-ai.nut
+- file-upload: info-gs.nut
   name: info.nut
 - api: new-package/publish
   error: "Expected exact 1 main-script file(s), but 0 were found."

regression/220_utf8_without_bom.yaml

@@ -2,7 +2,7 @@ steps:
 - api: user/login
 - api: new-package/start
 - file-upload: main.nut
-- file-upload: info-ai.nut
+- file-upload: info-gs.nut
   name: info.nut
 - file-upload: utf8_without_bom.nut
 - api: new-package/publish

regression/ai-tarball/main.nut

@@ -0,0 +1 @@
+/* A nearly empty file. All you got is this pound (£) sign. This file is saved in latin-1 encoding. */