feat!: Remove fast-xmlupload command (DEV-3324) (#830)

.github/workflows/tests-on-push.yml

@@ -92,26 +92,3 @@ jobs:
         uses: ./.github/actions/setup-from-wheel
       - name: distribution tests
         run: pytest test/distribution/
-
-
-  fast-xmlupload:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout dsp-tools repo
-        uses: actions/checkout@v4
-      - name: Install poetry, python, and dependencies
-        uses: ./.github/actions/setup
-      - uses: FedericoCarboni/setup-ffmpeg@v3
-      - id: search-cached-imagemagick
-        # If a cache is found that matches "key", the cached files are restored to "path".
-        # If no cache is found, this step is skipped, and when all jobs are done, the files in "path" are cached under "key".
-        uses: actions/cache@v4
-        with:
-          path: ~/bin/magick        # AppImage containing all binaries + dependencies
-          key: imagemagick          # don't check for a new version, because version checking is fairly complex
-      - uses: mfinelli/setup-imagemagick@v5  # downloads the "magick" AppImage to ~/bin/magick
-        if: ${{ steps.search-cached-imagemagick.outputs.cache-hit != 'true' }}
-      - name: start stack
-        run: poetry run dsp-tools start-stack --no-prune
-      - name: run fast-xmlupload test
-        run: poetry run pytest test/fast_xmlupload/

README.md

@@ -32,7 +32,6 @@ To get started quickly, without reading the details, just execute these commands
 - `poetry install`
 - `poetry shell`
 - `pre-commit install`
-- `brew install imagemagick ffmpeg` (A prior installation of [Homebrew](https://brew.sh/) is required.)
 - `npm install -g markdown-link-validator`
 
 The remainder of this README explains these commands in more detail.
@@ -146,8 +145,6 @@ Tests can be run in three different ways:
 This is useful in combination with the debugging mode of your IDE 
 if you want to examine why a single line of code in a test method fails.
 
-Some e2e tests require ImageMagick and ffmpeg to be installed: `brew install imagemagick ffmpeg`
-
 
 
 ## Code Style

docs/cli-commands.md

@@ -347,26 +347,3 @@ dsp-tools rosetta
 ```
 
 A DSP stack must be running before executing this command.
-
-
-
-## `process-files`
-
-DaSCH internal command to process multimedia files locally,
-before uploading them to a DSP server.
-See [here](./internal/fast-xmlupload.md) for more information.
-
-
-
-## `upload-files`
-
-DaSCH internal command to upload processed multimedia files to a DSP server.
-See [here](./internal/fast-xmlupload.md) for more information.
-
-
-
-## `fast-xmlupload`
-
-DaSCH internal command to create the resources of an XML file
-after the processed multimedia files have been uploaded already.
-See [here](./internal/fast-xmlupload.md) for more information.

docs/developers/user-data.md

@@ -12,7 +12,6 @@ Here is an overview of its structure:
 | docker                     | `start-stack`    | files necessary to startup Docker containers                                               |
 | rosetta                    | `rosetta`        | a clone of [the rosetta test project](https://github.com/dasch-swiss/082e-rosetta-scripts) |
 | logging.log, logging.log.1 | several ones     | These two grow up to 3 MB, then the oldest entries are deleted                             |
-| fast-xmlupload             | several ones     | shell script for local processing                                                          |
 
 
 Remark: Docker is normally not able to access files 

mkdocs.yml

@@ -20,8 +20,6 @@ nav:
   - Incremental xmlupload: incremental-xmlupload.md
   - excel2xml module: excel2xml-module.md
   - Running DSP locally: start-stack.md
-  - DaSCH-internal commands:
-      - Fast XML Upload: internal/fast-xmlupload.md
   - Information for developers:
       - Developers documentation: developers/index.md
       - Dependencies, packaging & distribution: developers/packaging.md

src/dsp_tools/cli/call_action.py

@@ -6,9 +6,6 @@
 from dsp_tools.commands.excel2json.properties import excel2properties
 from dsp_tools.commands.excel2json.resources import excel2resources
 from dsp_tools.commands.excel2xml.excel2xml_cli import excel2xml
-from dsp_tools.commands.fast_xmlupload.process_files import process_files
-from dsp_tools.commands.fast_xmlupload.upload_files import upload_files
-from dsp_tools.commands.fast_xmlupload.upload_xml import fast_xmlupload
 from dsp_tools.commands.id2iri import id2iri
 from dsp_tools.commands.ingest_xmlupload.upload_xml import ingest_xmlupload
 from dsp_tools.commands.project.create.project_create import create_project
@@ -64,12 +61,6 @@ def call_requested_action(args: argparse.Namespace) -> bool:
             result = _call_stop_stack()
         case "get":
             result = _call_get(args)
-        case "process-files":
-            result = _call_process_files(args)
-        case "upload-files":
-            result = _call_upload_files(args)
-        case "fast-xmlupload":
-            result = _call_fast_xmlupload(args)
         case "ingest-xmlupload":
             result = _call_ingest_xmlupload(args)
         case "template":
@@ -160,36 +151,6 @@ def _call_ingest_xmlupload(args: argparse.Namespace) -> bool:
     return True
 
 
-def _call_fast_xmlupload(args: argparse.Namespace) -> bool:
-    return fast_xmlupload(
-        xml_file=args.xml_file,
-        user=args.user,
-        password=args.password,
-        dsp_url=args.server,
-        sipi_url=args.sipi_url,
-    )
-
-
-def _call_upload_files(args: argparse.Namespace) -> bool:
-    return upload_files(
-        dir_with_processed_files=args.processed_dir,
-        nthreads=args.nthreads,
-        user=args.user,
-        password=args.password,
-        dsp_url=args.server,
-        sipi_url=args.sipi_url,
-    )
-
-
-def _call_process_files(args: argparse.Namespace) -> bool:
-    return process_files(
-        input_dir=args.input_dir,
-        output_dir=args.output_dir,
-        xml_file=args.xml_file,
-        nthreads=args.nthreads,
-    )
-
-
 def _call_xmlupload(args: argparse.Namespace) -> bool:
     if args.validate_only:
         return validate_xml(args.xmlfile)

src/dsp_tools/cli/create_parsers.py

@@ -47,12 +47,6 @@ def make_parser(
 
     _add_ingest_xmlupload(subparsers, default_dsp_api_url, root_user_email, root_user_pw)
 
-    _add_fast_xmlupload(subparsers, default_dsp_api_url, root_user_email, root_user_pw)
-
-    _add_process_files(subparsers)
-
-    _add_upload_files(subparsers, default_dsp_api_url, root_user_email, root_user_pw)
-
     _add_excel2json(subparsers)
 
     _add_excel2lists(subparsers)
@@ -197,55 +191,6 @@ def _add_ingest_xmlupload(
     subparser.add_argument("xml_file", help="path to XML file containing the data")
 
 
-def _add_fast_xmlupload(
-    subparsers: _SubParsersAction[ArgumentParser],
-    default_dsp_api_url: str,
-    root_user_email: str,
-    root_user_pw: str,
-) -> None:
-    subparser = subparsers.add_parser(
-        name="fast-xmlupload",
-        help="For internal use only: create resources with already uploaded files",
-    )
-    subparser.set_defaults(action="fast-xmlupload")
-    subparser.add_argument("-s", "--server", default=default_dsp_api_url, help=dsp_server_text)
-    subparser.add_argument("-u", "--user", default=root_user_email, help=username_text)
-    subparser.add_argument("-p", "--password", default=root_user_pw, help=password_text)
-    subparser.add_argument("xml_file", help="path to XML file containing the data")
-
-
-def _add_upload_files(
-    subparsers: _SubParsersAction[ArgumentParser],
-    default_dsp_api_url: str,
-    root_user_email: str,
-    root_user_pw: str,
-) -> None:
-    subparser = subparsers.add_parser(
-        name="upload-files",
-        help="For internal use only: upload already processed files",
-    )
-    subparser.set_defaults(action="upload-files")
-    subparser.add_argument("-d", "--processed-dir", help="path to the directory with the processed files")
-    subparser.add_argument("-n", "--nthreads", type=int, default=4, help="number of threads to use")
-    subparser.add_argument("-s", "--server", default=default_dsp_api_url, help=dsp_server_text)
-    subparser.add_argument("-u", "--user", default=root_user_email, help=username_text)
-    subparser.add_argument("-p", "--password", default=root_user_pw, help=password_text)
-
-
-def _add_process_files(subparsers: _SubParsersAction[ArgumentParser]) -> None:
-    subparser = subparsers.add_parser(
-        name="process-files",
-        help="For internal use only: process all files referenced in an XML file",
-    )
-    subparser.set_defaults(action="process-files")
-    subparser.add_argument("--input-dir", help="path to the input directory where the files should be read from")
-    subparser.add_argument(
-        "--output-dir", help="path to the output directory where the processed/transformed files should be written to"
-    )
-    subparser.add_argument("--nthreads", type=int, default=None, help="number of threads to use")
-    subparser.add_argument("xml_file", help="path to XML file containing the data")
-
-
 def _add_xmlupload(
     subparsers: _SubParsersAction[ArgumentParser],
     default_dsp_api_url: str,