diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml new file mode 100644 index 0000000..74ccaaf --- /dev/null +++ b/.github/workflows/ci-dev.yml @@ -0,0 +1,60 @@ +name: CI (dev) + +on: + push: + branches: [ "dev" ] + pull_request: + branches: [ "dev" ] + schedule: + - cron: "0 3 * * *" + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install tooling + run: | + python -m pip install --upgrade pip + pip install ruff mypy + - name: Ruff check + run: ruff check automation_file tests + - name: Ruff format check + run: ruff format --check automation_file tests + - name: Mypy + run: mypy automation_file + + pytest: + needs: lint + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + python-version: [ "3.10", "3.11", "3.12" ] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel + pip install -r dev_requirements.txt + pip install pytest pytest-cov + - name: Run pytest with coverage + run: python -m pytest tests/ -v --tb=short --cov=automation_file --cov-report=term-missing --cov-report=xml + - name: Upload coverage artifact + if: matrix.python-version == '3.12' + uses: actions/upload-artifact@v4 + with: + name: coverage-xml + path: coverage.xml diff --git a/.github/workflows/ci-stable.yml b/.github/workflows/ci-stable.yml new file mode 100644 index 0000000..62e8cab --- /dev/null +++ b/.github/workflows/ci-stable.yml @@ -0,0 +1,131 @@ +name: CI (stable) + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: "0 3 * * *" + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install tooling + run: | + python -m pip install --upgrade pip + pip install ruff mypy + - name: Ruff check + run: ruff check automation_file tests + - name: Ruff format check + run: ruff format --check automation_file tests + - name: Mypy + run: mypy automation_file + + pytest: + needs: lint + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + python-version: [ "3.10", "3.11", "3.12" ] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel + pip install -r requirements.txt + pip install pytest pytest-cov + - name: Run pytest with coverage + run: python -m pytest tests/ -v --tb=short --cov=automation_file --cov-report=term-missing --cov-report=xml + - name: Upload coverage artifact + if: matrix.python-version == '3.12' + uses: actions/upload-artifact@v4 + with: + name: coverage-xml + path: coverage.xml + + publish: + needs: pytest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install build tools + run: | + python -m pip install --upgrade pip + pip install build twine + - name: Bump patch version in stable.toml and dev.toml + id: version + run: | + python - <<'PY' + import os + import pathlib + import re + + def bump(path: pathlib.Path) -> str: + text = path.read_text(encoding="utf-8") + match = re.search(r'^version = "(\d+)\.(\d+)\.(\d+)"', text, re.MULTILINE) + if match is None: + raise SystemExit(f"no version line found in {path}") + major, minor, patch = (int(g) for g in match.groups()) + new = f"{major}.{minor}.{patch + 1}" + path.write_text(text.replace(match.group(0), f'version = "{new}"', 1), encoding="utf-8") + return new + + stable_version = bump(pathlib.Path("stable.toml")) + dev_version = bump(pathlib.Path("dev.toml")) + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as fp: + fp.write(f"version={stable_version}\n") + fp.write(f"dev_version={dev_version}\n") + print(f"stable.toml -> {stable_version}") + print(f"dev.toml -> {dev_version}") + PY + - name: Commit bumped versions back to main + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add stable.toml dev.toml + git commit -m "Bump version to v${{ steps.version.outputs.version }} [skip ci]" + git push origin HEAD:main + - name: Use stable.toml as pyproject.toml + run: cp stable.toml pyproject.toml + - name: Build sdist and wheel + run: python -m build + - name: Twine check + run: twine check dist/* + - name: Twine upload to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: twine upload --non-interactive dist/* + - name: Create GitHub Release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release create "v${{ steps.version.outputs.version }}" dist/* \ + --title "v${{ steps.version.outputs.version }}" \ + --generate-notes diff --git a/.github/workflows/file_automation_dev_python3_10.yml b/.github/workflows/file_automation_dev_python3_10.yml deleted file mode 100644 index bbd2157..0000000 --- a/.github/workflows/file_automation_dev_python3_10.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: FileAutomation Dev Python3.10 - -on: - push: - branches: [ "dev" ] - pull_request: - branches: [ "dev" ] - schedule: - - cron: "0 3 * * *" - -permissions: - contents: read - -jobs: - build_dev_version: - runs-on: windows-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip wheel - pip install -r dev_requirements.txt - - name: Dir Module Test - run: python ./tests/unit_test/local/dir/dir_test.py - - name: File Module Test - run: python ./tests/unit_test/local/file/test_file.py - - name: Zip Module Test - run: python ./tests/unit_test/local/zip/zip_test.py \ No newline at end of file diff --git a/.github/workflows/file_automation_dev_python3_11.yml b/.github/workflows/file_automation_dev_python3_11.yml deleted file mode 100644 index b9f397a..0000000 --- a/.github/workflows/file_automation_dev_python3_11.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: FileAutomation Dev Python3.11 - -on: - push: - branches: [ "dev" ] - pull_request: - branches: [ "dev" ] - schedule: - - cron: "0 3 * * *" - -permissions: - contents: read - -jobs: - build_dev_version: - runs-on: windows-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.11 - uses: actions/setup-python@v3 - with: - python-version: "3.11" - - name: Install dependencies - run: | - python -m pip install --upgrade pip wheel - pip install -r dev_requirements.txt - - name: Dir Module Test - run: python ./tests/unit_test/local/dir/dir_test.py - - name: File Module Test - run: python ./tests/unit_test/local/file/test_file.py - - name: Zip Module Test - run: python ./tests/unit_test/local/zip/zip_test.py \ No newline at end of file diff --git a/.github/workflows/file_automation_dev_python3_12.yml b/.github/workflows/file_automation_dev_python3_12.yml deleted file mode 100644 index b7ca17e..0000000 --- a/.github/workflows/file_automation_dev_python3_12.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: FileAutomation Dev Python3.12 - -on: - push: - branches: [ "dev" ] - pull_request: - branches: [ "dev" ] - schedule: - - cron: "0 3 * * *" - -permissions: - contents: read - -jobs: - build_dev_version: - runs-on: windows-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.12 - uses: actions/setup-python@v3 - with: - python-version: "3.12" - - name: Install dependencies - run: | - python -m pip install --upgrade pip wheel - pip install -r dev_requirements.txt - - name: Dir Module Test - run: python ./tests/unit_test/local/dir/dir_test.py - - name: File Module Test - run: python ./tests/unit_test/local/file/test_file.py - - name: Zip Module Test - run: python ./tests/unit_test/local/zip/zip_test.py \ No newline at end of file diff --git a/.github/workflows/file_automation_stable_python3_10.yml b/.github/workflows/file_automation_stable_python3_10.yml deleted file mode 100644 index 8c844f3..0000000 --- a/.github/workflows/file_automation_stable_python3_10.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: FileAutomation Stable Python3.10 - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: "0 3 * * *" - -permissions: - contents: read - -jobs: - build_stable_version: - runs-on: windows-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip wheel - pip install -r requirements.txt - - name: Dir Module Test - run: python ./tests/unit_test/local/dir/dir_test.py - - name: File Module Test - run: python ./tests/unit_test/local/file/test_file.py - - name: Zip Module Test - run: python ./tests/unit_test/local/zip/zip_test.py \ No newline at end of file diff --git a/.github/workflows/file_automation_stable_python3_11.yml b/.github/workflows/file_automation_stable_python3_11.yml deleted file mode 100644 index ed8002b..0000000 --- a/.github/workflows/file_automation_stable_python3_11.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: FileAutomation Stable Python3.11 - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: "0 3 * * *" - -permissions: - contents: read - -jobs: - build_stable_version: - runs-on: windows-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.11 - uses: actions/setup-python@v3 - with: - python-version: "3.11" - - name: Install dependencies - run: | - python -m pip install --upgrade pip wheel - pip install -r requirements.txt - - name: Dir Module Test - run: python ./tests/unit_test/local/dir/dir_test.py - - name: File Module Test - run: python ./tests/unit_test/local/file/test_file.py - - name: Zip Module Test - run: python ./tests/unit_test/local/zip/zip_test.py \ No newline at end of file diff --git a/.github/workflows/file_automation_stable_python3_12.yml b/.github/workflows/file_automation_stable_python3_12.yml deleted file mode 100644 index 9c1a4cd..0000000 --- a/.github/workflows/file_automation_stable_python3_12.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: FileAutomation Stable Python3.12 - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: "0 3 * * *" - -permissions: - contents: read - -jobs: - build_stable_version: - runs-on: windows-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.12 - uses: actions/setup-python@v3 - with: - python-version: "3.12" - - name: Install dependencies - run: | - python -m pip install --upgrade pip wheel - pip install -r requirements.txt - - name: Dir Module Test - run: python ./tests/unit_test/local/dir/dir_test.py - - name: File Module Test - run: python ./tests/unit_test/local/file/test_file.py - - name: Zip Module Test - run: python ./tests/unit_test/local/zip/zip_test.py diff --git a/.gitignore b/.gitignore index 3e651e2..c9b638b 100644 --- a/.gitignore +++ b/.gitignore @@ -156,3 +156,5 @@ token.json credentials.json **/token.json **/credentials.json + +.claude/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..db10b24 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-toml + - id: check-added-large-files + args: ["--maxkb=500"] + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.9 + hooks: + - id: ruff + args: ["--fix"] + - id: ruff-format + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.11.2 + hooks: + - id: mypy + additional_dependencies: [] + args: ["--config-file=mypy.ini"] + files: ^automation_file/ diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..e1b9b88 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,42 @@ +[MAIN] +# PySide6 is a C-extension binding — pylint's static inference cannot see the +# generated classes, so treat it as a trusted extension package. +extension-pkg-allow-list=PySide6,PySide6.QtCore,PySide6.QtGui,PySide6.QtWidgets + +# Pylint should not try to parse requirement manifests as Python modules. +ignore-paths=docs/requirements\.txt + +[MESSAGES CONTROL] +# Disabled rules, with rationale: +# C0114/C0115/C0116 — docstring requirements are enforced by review, not lint. +# C0415 — ``import-outside-toplevel`` is used deliberately for +# lazy imports of heavy / optional modules (see CLAUDE.md). +# R0903 — too-few-public-methods; dataclasses and frozen option +# objects are allowed to have no methods. +# W0511 — TODO/FIXME markers; tracked via issues, not lint. +disable= + C0114, + C0115, + C0116, + C0415, + R0903, + W0511 + +[TYPECHECK] +# googleapiclient's ``Resource`` object exposes ``files()``, ``permissions()``, +# etc. dynamically — pylint can't see them, so whitelist the names rather than +# littering the Drive ops modules with ``# pylint: disable=no-member``. +generated-members=files,permissions + +[DESIGN] +# Align with CLAUDE.md's code-quality checklist. +max-args=7 +max-locals=15 +max-returns=8 +max-branches=15 +max-statements=50 +max-attributes=17 +max-public-methods=25 + +[FORMAT] +max-line-length=100 diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..1299634 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,275 @@ +# FileAutomation + +Automation-first Python library for local file / directory / zip operations, HTTP downloads, and remote storage (Google Drive, S3, Azure Blob, Dropbox, SFTP). Actions are defined as JSON and dispatched through a central registry so they can be executed in-process, from disk, over a TCP socket, or over HTTP. + +## Architecture + +**Layered architecture with Facade + Registry + Command + Strategy patterns:** + +``` +automation_file/ +├── __init__.py # Public API facade (every name users import) +├── __main__.py # CLI entry (argparse dispatcher, subcommands + legacy flags) +├── exceptions.py # Exception hierarchy (FileAutomationException base) +├── logging_config.py # file_automation_logger (file + stderr handlers) +├── core/ +│ ├── action_registry.py # ActionRegistry — name -> callable (Registry + Command) +│ ├── action_executor.py # ActionExecutor — runs JSON action lists (Facade + Template Method) +│ ├── callback_executor.py # CallbackExecutor — trigger then callback composition +│ ├── package_loader.py # PackageLoader — dynamically registers package members +│ ├── json_store.py # Thread-safe read/write of JSON action files +│ ├── retry.py # retry_on_transient — capped exponential back-off decorator +│ └── quota.py # Quota — size + time budget guards +├── local/ # Strategy modules — each file is a batch of pure operations +│ ├── file_ops.py +│ ├── dir_ops.py +│ ├── zip_ops.py +│ └── safe_paths.py # safe_join / is_within — path traversal guard +├── remote/ +│ ├── url_validator.py # SSRF guard for outbound URLs +│ ├── http_download.py # SSRF-validated HTTP download with size/timeout caps + retry +│ ├── google_drive/ +│ │ ├── client.py # GoogleDriveClient (Singleton Facade) +│ │ ├── delete_ops.py +│ │ ├── download_ops.py +│ │ ├── folder_ops.py +│ │ ├── search_ops.py +│ │ ├── share_ops.py +│ │ └── upload_ops.py +│ ├── s3/ # S3 (boto3) — auto-registered in build_default_registry() +│ │ ├── client.py # S3Client +│ │ ├── upload_ops.py +│ │ ├── download_ops.py +│ │ ├── delete_ops.py +│ │ └── list_ops.py +│ ├── azure_blob/ # Azure Blob — auto-registered in build_default_registry() +│ │ └── {client,upload,download,delete,list}_ops.py +│ ├── dropbox_api/ # Dropbox — auto-registered in build_default_registry() +│ │ └── {client,upload,download,delete,list}_ops.py +│ └── sftp/ # SFTP (paramiko + RejectPolicy) — auto-registered in build_default_registry() +│ └── {client,upload,download,delete,list}_ops.py +├── server/ +│ ├── tcp_server.py # Loopback-only TCP server executing JSON actions (optional shared-secret auth) +│ └── http_server.py # Loopback-only HTTP server (POST /actions, optional Bearer auth) +├── project/ +│ ├── project_builder.py # ProjectBuilder (Builder pattern) +│ └── templates.py # Scaffolding templates +├── ui/ # PySide6 GUI (required dep) +│ ├── launcher.py # launch_ui(argv) — boots QApplication + MainWindow +│ ├── main_window.py # MainWindow — tabbed control surface over every feature +│ ├── worker.py # ActionWorker(QRunnable) + _WorkerSignals +│ ├── log_widget.py # LogPanel — timestamped, read-only log stream +│ └── tabs/ # One tab per domain: local / http / drive / s3 / +│ # azure / dropbox / sftp / +│ # JSON actions / servers +└── utils/ + └── file_discovery.py # Recursive file listing by extension +``` + +**Key design patterns in use:** +- **Facade**: `automation_file/__init__.py` re-exports every supported name (`execute_action`, `driver_instance`, `start_autocontrol_socket_server`, …). +- **Registry + Command**: `ActionRegistry` maps action name → callable. JSON action lists are command objects (`[name, kwargs]` / `[name, [args]]` / `[name]`) dispatched through the registry. +- **Template Method**: `ActionExecutor._execute_event` defines the single-action lifecycle (resolve → call → wrap result); `execute_action` is the outer iteration template. +- **Strategy**: Each `local/*_ops.py` and `remote/google_drive/*_ops.py` module is an independent strategy that plugs into the registry. +- **Singleton (module-level)**: `driver_instance`, `executor`, `callback_executor`, `package_manager` are shared instances wired in `__init__.py` so `callback_executor.registry is executor.registry`. +- **Builder**: `ProjectBuilder` assembles the `keyword/` + `executor/` skeleton. + +## Key types + +- `ActionRegistry` — mutable name → callable mapping. `register`, `register_many`, `resolve`, `unregister`, `event_dict` (live view for legacy callers). +- `ActionExecutor` — holds a registry and runs JSON action lists. `execute_action(list|dict, validate_first=False, dry_run=False)`, `execute_action_parallel(list, max_workers=None)`, `validate(list) -> list[str]`, `execute_files(paths)`, `add_command_to_executor(mapping)`. +- `CallbackExecutor` — runs a registered trigger, then a user callback, sharing the executor's registry. +- `PackageLoader` — imports a package by name and registers its top-level functions / classes / builtins as `_`. +- `GoogleDriveClient` — wraps OAuth2 credential loading; exposes `service` lazily. `later_init(token_path, credentials_path)` bootstraps; `require_service()` raises if not initialised. +- `S3Client` / `AzureBlobClient` / `DropboxClient` / `SFTPClient` — singleton wrappers around the required SDKs. Each exposes `later_init(...)` plus `close()` where relevant. Their ops are auto-registered by `build_default_registry()`; `register__ops(registry)` is still exported so callers can populate custom registries. +- `MainWindow` — PySide6 tabbed control surface (`ui/main_window.py`). Nine tabs — Local, HTTP, Google Drive, S3, Azure Blob, Dropbox, SFTP, JSON actions, Servers — share a `LogPanel` and dispatch work through `ActionWorker(QRunnable)` on the global `QThreadPool`. +- `launch_ui(argv=None)` — boots / reuses a `QApplication`, shows `MainWindow`, and returns the exec code. Exposed lazily on the facade via `__getattr__` so the Qt runtime isn't paid for by non-UI importers. +- `TCPActionServer` — threaded TCP server that deserialises a JSON action list per connection. Defaults to loopback; optional `shared_secret` enforces `AUTH \n` prefix. +- `HTTPActionServer` — `ThreadingHTTPServer` exposing `POST /actions`. Defaults to loopback; optional `shared_secret` enforces `Authorization: Bearer `. +- `Quota` — frozen dataclass capping bytes and wall-clock seconds per action or block (`check_size`, `time_budget` context manager, `wraps` decorator). `0` disables each cap. +- `retry_on_transient(max_attempts, backoff_base, backoff_cap, retriable)` — decorator that retries with capped exponential back-off and raises `RetryExhaustedException` chained to the last error. +- `safe_join(root, user_path)` / `is_within(root, path)` — path traversal guard; `safe_join` raises `PathTraversalException` when the resolved path escapes `root`. + +## Branching & CI + +- `main` branch: stable releases, publishes `automation_file` to PyPI (version in `stable.toml`). +- `dev` branch: development, publishes `automation_file_dev` to PyPI (version in `dev.toml`). +- Keep `dependencies` and `[project.optional-dependencies]` (`dev`) in sync across both TOMLs. Backends (`boto3`, `azure-storage-blob`, `dropbox`, `paramiko`) and `PySide6` are first-class runtime deps — do not move them back under extras. +- **Version bumping is automatic.** The stable publish job bumps the patch in both `stable.toml` and `dev.toml`, commits the bump back to `main` with `[skip ci]`, then builds and releases. Do not hand-bump before merging to `main`. +- CI: GitHub Actions (Windows, Python 3.10 / 3.11 / 3.12) — one matrix workflow per branch: `.github/workflows/ci-dev.yml`, `.github/workflows/ci-stable.yml`. +- CI steps: `lint` (ruff check + ruff format --check + mypy) → `pytest` with coverage → uploads `coverage.xml` as an artifact. +- Stable branch additionally runs a `publish` job on push to `main`: auto-bumps the patch in both TOMLs and commits back, then builds the sdist + wheel, `twine check`, `twine upload` using `PYPI_API_TOKEN`, then `gh release create v --generate-notes`. +- `pre-commit` is configured (`.pre-commit-config.yaml`): trailing-whitespace, eof-fixer, check-yaml, check-toml, check-added-large-files, ruff, ruff-format, mypy. Install with `pre-commit install` after cloning. + +## Development + +```bash +python -m pip install -r dev_requirements.txt pytest pytest-cov +python -m pip install -e ".[dev]" # ruff, mypy, pre-commit +python -m pytest tests/ -v --tb=short +ruff check automation_file/ tests/ +ruff format --check automation_file/ tests/ +mypy automation_file/ +python -m automation_file --help +``` + +**Testing:** +- Unit tests live under `tests/` (pytest). Fixtures in `tests/conftest.py` (`sample_file`, `sample_dir`). +- Tests cover every module in `core/`, `local/`, `remote/url_validator`, `project/`, `server/`, `utils/`, plus a facade smoke test, retry/quota/safe_paths, HTTP+TCP auth, and optional-backend registration. +- Google Drive / HTTP-download / S3 / Azure / Dropbox / SFTP code paths that require real credentials or network access are **not** exercised in CI — only their URL-validation, auth, and guard-clause behaviour are. +- Run all tests before submitting changes: `python -m pytest tests/ -v`. + +## Conventions + +- Python 3.10+ — use `X | Y` union syntax, not `Union[X, Y]`. +- Use `from __future__ import annotations` at the top of every module for deferred type evaluation. +- Exception hierarchy: all custom exceptions inherit from `FileAutomationException`; never `raise Exception(...)` directly. +- Logging: use `file_automation_logger` from `automation_file.logging_config`. Never `print()` for diagnostics. +- Action-list shape: `[name]`, `[name, {kwargs}]`, or `[name, [args]]` — nothing else. +- Delete all unused code — no dead imports, commented-out blocks, unreachable branches, or `_old_`-prefixed names. Git history is the archive. +- Prefer updating the registry over extending the executor class. Plugins register via `add_command_to_executor({name: callable})`. + +## Security + +All code must follow secure-by-default principles. Review every change against the checklist below. + +### General rules +- Never use `eval()`, `exec()`, or `pickle.loads()` on untrusted data. +- Never use `subprocess.Popen(..., shell=True)` — always pass argument lists. +- Never log or display secrets, tokens, passwords, or API keys. OAuth2 tokens handled by `GoogleDriveClient` are kept on disk only at the caller-supplied `token_path`. +- Use `json.loads()` / `json.dumps()` for serialisation — never pickle. +- Validate all user input at system boundaries (CLI args, URL inputs, TCP payloads). + +### Network requests (SSRF prevention) +- **All** outbound HTTP requests to user-specified URLs must validate the target first via `automation_file.remote.url_validator.validate_http_url`: + 1. Only `http://` and `https://` schemes — rejects `file://`, `ftp://`, `data:`, `gopher://`. + 2. Resolve the hostname and reject IPs in private / loopback / link-local / reserved / multicast / unspecified ranges. +- `http_download.download_file` calls the validator, uses `allow_redirects=False`, enforces a default 20 MB response cap and 15 s connection timeout, and never downgrades TLS verification. +- Never pass user-supplied URLs directly to `urlopen()` / `requests.*` without the validator. + +### Network requests (TLS) +- All HTTPS requests must use default TLS verification — never set `verify=False`. +- No bespoke SSH logic in this project; if added, match PyBreeze's `InteractiveHostKeyPolicy` pattern. + +### Subprocess execution +- This library does not spawn subprocesses on the hot path. If you add one, pass argument lists (never `shell=True`), set an explicit `timeout`, and never interpolate user input into a command string. + +### TCP server +- `TCPActionServer` binds to `localhost` by default. `start_autocontrol_socket_server(host=…)` raises `ValueError` if the resolved address is not loopback unless `allow_non_loopback=True` is passed explicitly. +- Do not remove the loopback guard to "make it easier to test remotely". The server dispatches arbitrary registry commands; exposing it to the network is equivalent to exposing a Python REPL. +- The server accepts a single JSON payload per connection (`recv(8192)`). Do not raise that limit without also adding a length-framed protocol. +- `quit_server` triggers an orderly shutdown; do not add an administrative bypass that skips the loopback check. +- Optional `shared_secret=` enforces an `AUTH \n` prefix; the comparison uses `hmac.compare_digest` (constant time). Never log the secret or the raw payload. + +### HTTP server +- `HTTPActionServer` / `start_http_action_server` mirror the TCP server's posture: loopback-only by default, `allow_non_loopback=True` required to bind elsewhere, optional `shared_secret` enforced as `Authorization: Bearer ` using `hmac.compare_digest`. +- Only `POST /actions` is handled. Request body capped at 1 MB — do not raise without also switching to a streaming parser. +- Responses are JSON. Auth failures return `401`; malformed JSON returns `400`; unknown paths return `404`. + +### Path traversal +- Any caller resolving a user-supplied path against a trusted root must go through `automation_file.local.safe_paths.safe_join` (raises `PathTraversalException`) or the `is_within` check. Never concatenate + `Path.resolve()` yourself and skip the containment check — symlinks and `..` segments bypass naive string checks. + +### SFTP host verification +- `SFTPClient` uses `paramiko.RejectPolicy()` — unknown hosts are rejected, never auto-added. Callers pass `known_hosts=` explicitly or rely on `~/.ssh/known_hosts`. Do not swap in `AutoAddPolicy` for convenience. + +### Reliability (retry / quota) +- `retry_on_transient` only retries the exception types passed via `retriable=(…)`. Never widen to bare `Exception` — masks logic bugs as transient failures. Always exhausts to `RetryExhaustedException` chained with `raise ... from err`. +- `Quota(max_bytes=…, max_seconds=…)` — prefer `Quota.wraps(...)` over inline checks when guarding a whole operation. `0` disables each cap. + +### Google Drive +- Credentials are stored at the caller-supplied `token_path` with `encoding="utf-8"`. Never log or print the token contents. +- `GoogleDriveClient.require_service()` raises rather than silently operating with a `None` service — do not paper over it by catching `RuntimeError` at the call site. + +### File I/O +- Always use `pathlib.Path` for path manipulation; never string-concatenate paths with user input. +- Use `with open(...) as f:` for every file operation; close via context manager. +- Always pass `encoding="utf-8"` when reading or writing text. +- Never follow symlinks from untrusted sources — resolve and re-check the parent. +- JSON writes go through `automation_file.core.json_store.write_action_json` which holds a module-level lock. + +### Plugin / package loading +- `PackageLoader.add_package_to_executor(package)` registers every function / class / builtin of a package under `_`. Treat it as eval-grade power: never expose it to arbitrary clients (e.g. via the TCP server). If you add a remote plugin-load command, gate it behind an explicit admin flag and authenticated transport. + +### Secrets and credentials +- Google OAuth tokens live on disk at the user-supplied path; keep the path out of logs. +- API keys / credentials must come from env vars or caller-supplied paths; never hardcode. + +### Dependency security +- Pin dependencies in `requirements.txt` / `dev_requirements.txt`. +- Do not add new dependencies without reviewing their security posture. +- Avoid transitive bloat — prefer stdlib when the alternative is a single-function dependency. + +## Code quality (SonarQube / Codacy compliance) + +All code must satisfy common static-analysis rules. Review every change against the checklist below. + +### Complexity & size +- Cyclomatic complexity per function: ≤ 15 (hard cap 20). Break large branches into helpers. +- Cognitive complexity per function: ≤ 15. Flatten nested `if`/`for`/`try` chains with early returns. +- Function length: ≤ 75 lines of code (excluding docstring / blank lines). Extract helpers past that. +- Parameter count: ≤ 7 per function/method. Use a dataclass when more are needed. +- Nesting depth: ≤ 4 levels. Refactor with early returns instead of pyramids. +- File length: ≤ 1000 lines. + +### Exception handling +- Never use bare `except:` — always specify exception types. +- Avoid catching `Exception` / `BaseException` unless immediately logging and re-raising, or running at a top-level dispatcher boundary (the `ActionExecutor.execute_action` loop is one of these — it intentionally records per-action failures without aborting the batch). +- Never `pass` silently inside `except` — log via `file_automation_logger` at minimum. +- Do not `return` / `break` / `continue` inside a `finally` block — it swallows exceptions. +- Custom exceptions must inherit from `FileAutomationException`. +- Use `raise ... from err` (or `raise ... from None`) when re-raising to preserve / suppress the chain explicitly. + +### Pythonic correctness +- Compare with `None` using `is` / `is not`, never `==` / `!=`. +- Type checks use `isinstance(obj, T)`, never `type(obj) == T`. +- Never use mutable default arguments — use `None` and initialise inside. +- Prefer f-strings over `%` formatting or `str.format()` (except inside lazy log calls: `logger.info("x=%s", x)`). +- Use context managers for every file / socket / lock. +- Use `enumerate()` instead of `range(len(...))` when the index is needed. +- Use `dict.get(key, default)` over `key in dict and dict[key]`. + +### Naming & style (PEP 8) +- `snake_case` for functions, methods, variables, module names. +- `PascalCase` for classes. +- `UPPER_SNAKE_CASE` for module-level constants. +- `_leading_underscore` for protected / internal members. +- Do not shadow built-ins (`id`, `type`, `list`, `dict`, `input`, `file`, `open`, etc.). + +### Duplication & dead code +- String literal used 3+ times in the same module → extract a module-level constant. +- Identical 6+ line blocks in 2+ places → extract a helper. +- Remove unused imports, unused parameters, unused local variables, unreachable code after `return` / `raise`. +- No commented-out code blocks — delete them. +- No `TODO` / `FIXME` / `XXX` without an issue reference (`# TODO(#123): …`). + +### Logging, printing, assertions +- Never use `print()` for diagnostics in library code — use `file_automation_logger`. +- Use lazy logging (`logger.debug("x=%s", x)`) to avoid eager f-string formatting on hot paths. +- Never use `assert` for runtime validation; `assert` is for tests only. + +### Hardcoded values & secrets +- No hardcoded passwords, tokens, API keys, or secrets. +- No hardcoded IPs / hostnames outside of documented `localhost` / loopback defaults. +- Magic numbers (except 0, 1, -1) should be named constants when repeated or non-obvious. + +### Boolean & return hygiene +- `return bool(cond)` or `return cond`, not `if cond: return True else: return False`. +- `if x` / `if not x`, not `if x == True` / `if x == False`. +- A function should have a consistent return type. + +### Imports +- One import per line; grouped `from x import a, b` is fine. +- Order: stdlib → third-party → first-party (`automation_file.*`) — separated by blank lines. +- No wildcard imports outside `__init__.py` re-exports. +- Max one level of relative import. + +### Running the linter +- Before committing any non-trivial change, run `ruff check automation_file/ tests/` locally. +- When adding a `# noqa: RULE`, justify it in the comment — never blanket-disable. + +## Commit & PR rules + +- Commit messages: short imperative sentence (e.g., "Fix rename_file overwrite bug", "Update stable version"). +- Do not mention any AI tools, assistants, or co-authors in commit messages or PR descriptions. +- Do not add `Co-Authored-By` headers referencing any AI. +- PR target: `dev` for development work, `main` for stable releases. diff --git a/README.md b/README.md index b7b3345..e781ad8 100644 --- a/README.md +++ b/README.md @@ -1,90 +1,306 @@ # FileAutomation -This project provides a modular framework for file automation and Google Drive integration. -It supports local file and directory operations, ZIP archive handling, -Google Drive CRUD (create, search, upload, download, delete, share), -and remote execution through a TCP Socket Server. - -# Features -## Local File and Directory Operations -- Create, delete, copy, and rename files -- Create, delete, and copy directories -- Recursively search for files by extension - -## ZIP Archive Handling -- Create ZIP archives -- Extract single files or entire archives -- Set ZIP archive passwords -- Read archive information - -## Google Drive Integration -- Upload: single files, entire directories, to root or specific folders -- Download: single files or entire folders -- Search: by name, MIME type, or custom fields -- Delete: remove files from Drive -- Share: with specific users, domains, or via public link -- Folder Management: create new folders in Drive - -## Automation Executors -- Executor: central manager for all executable functions, supports action lists -- CallbackExecutor: supports callback functions for flexible workflows -- PackageManager: dynamically loads packages and registers functions into executors - -# JSON Configuration -- Read and write JSON-based action lists -- Define automation workflows in JSON format - -# TCP Socket Server -- Start a TCP server to receive JSON commands and execute corresponding actions -- Supports remote control and returns execution results - -## Installation and Requirements - -- Requirements - - Python 3.9+ - - Google API Client - - Google Drive API enabled and credentials.json downloaded +A modular automation framework for local file / directory / ZIP operations, +SSRF-validated HTTP downloads, remote storage (Google Drive, S3, Azure Blob, +Dropbox, SFTP), and JSON-driven action execution over embedded TCP / HTTP +servers. Ships with a PySide6 GUI that exposes every feature through tabs. +All public functionality is re-exported from the top-level `automation_file` +facade. +- Local file / directory / ZIP operations with path traversal guard (`safe_join`) +- Validated HTTP downloads with SSRF protections, retry, and size / time caps +- Google Drive CRUD (upload, download, search, delete, share, folders) +- First-class S3, Azure Blob, Dropbox, and SFTP backends — installed by default +- JSON action lists executed by a shared `ActionExecutor` — validate, dry-run, parallel +- Loopback-first TCP **and** HTTP servers that accept JSON command batches with optional shared-secret auth +- Reliability primitives: `retry_on_transient` decorator, `Quota` size / time budgets +- PySide6 GUI (`python -m automation_file ui`) with a tab per backend plus a JSON-action runner +- Rich CLI with one-shot subcommands plus legacy JSON-batch flags +- Project scaffolding (`ProjectBuilder`) for executor-based automations + +## Architecture + +```mermaid +flowchart LR + User[User / CLI / JSON batch] + + subgraph Facade["automation_file (facade)"] + Public["Public API
execute_action, execute_action_parallel,
validate_action, driver_instance,
start_autocontrol_socket_server,
start_http_action_server, Quota,
retry_on_transient, safe_join, ..."] + end + + subgraph Core["core"] + Registry[(ActionRegistry
FA_* commands)] + Executor[ActionExecutor] + Callback[CallbackExecutor] + Loader[PackageLoader] + Json[json_store] + Retry[retry] + QuotaMod[quota] + end + + subgraph Local["local"] + FileOps[file_ops] + DirOps[dir_ops] + ZipOps[zip_ops] + Safe[safe_paths] + end + + subgraph Remote["remote"] + UrlVal[url_validator] + Http[http_download] + Drive["google_drive
client + *_ops"] + S3["s3"] + Azure["azure_blob"] + Dropbox["dropbox_api"] + SFTP["sftp"] + end + + subgraph Server["server"] + TCP[TCPActionServer] + HTTP[HTTPActionServer] + end + + subgraph UI["ui (PySide6)"] + Launcher[launch_ui] + MainWindow["MainWindow
9-tab control surface"] + end + + subgraph Project["project / utils"] + Builder[ProjectBuilder] + Templates[templates] + Discovery[file_discovery] + end + + User --> Public + User --> Launcher + Launcher --> MainWindow + MainWindow --> Public + Public --> Executor + Public --> Callback + Public --> Loader + Public --> TCP + Public --> HTTP + + Executor --> Registry + Executor --> Retry + Executor --> QuotaMod + Callback --> Registry + Loader --> Registry + TCP --> Executor + HTTP --> Executor + Executor --> Json + + Registry --> FileOps + Registry --> DirOps + Registry --> ZipOps + Registry --> Safe + Registry --> Http + Registry --> Drive + Registry --> S3 + Registry --> Azure + Registry --> Dropbox + Registry --> SFTP + Registry --> Builder + + Http --> UrlVal + Http --> Retry + Builder --> Templates + Builder --> Discovery +``` + +The `ActionRegistry` built by `build_default_registry()` is the single source +of truth for every `FA_*` command. `ActionExecutor`, `CallbackExecutor`, +`PackageLoader`, `TCPActionServer`, and `HTTPActionServer` all resolve commands +through the same shared registry instance exposed as `executor.registry`. ## Installation -> pip install automation_file -# Usage +```bash +pip install automation_file +``` + +A single install pulls in every backend (Google Drive, S3, Azure Blob, Dropbox, +SFTP) and the PySide6 GUI — no extras required for day-to-day use. + +```bash +pip install "automation_file[dev]" # ruff, mypy, pre-commit, pytest-cov, build, twine +``` + +Requirements: +- Python 3.10+ +- Bundled dependencies: `google-api-python-client`, `google-auth-oauthlib`, + `requests`, `tqdm`, `boto3`, `azure-storage-blob`, `dropbox`, `paramiko`, + `PySide6` + +## Usage + +### Execute a JSON action list +```python +from automation_file import execute_action + +execute_action([ + ["FA_create_file", {"file_path": "test.txt"}], + ["FA_copy_file", {"source": "test.txt", "target": "copy.txt"}], +]) +``` + +### Validate, dry-run, parallel +```python +from automation_file import execute_action, execute_action_parallel, validate_action + +# Fail-fast: aborts before any action runs if any name is unknown. +execute_action(actions, validate_first=True) + +# Dry-run: log what would be called without invoking commands. +execute_action(actions, dry_run=True) + +# Parallel: run independent actions through a thread pool. +execute_action_parallel(actions, max_workers=4) + +# Manual validation — returns the list of resolved names. +names = validate_action(actions) +``` + +### Initialize Google Drive and upload +```python +from automation_file import driver_instance, drive_upload_to_drive + +driver_instance.later_init("token.json", "credentials.json") +drive_upload_to_drive("example.txt") +``` + +### Validated HTTP download (with retry) +```python +from automation_file import download_file + +download_file("https://example.com/file.zip", "file.zip") +``` + +### Start the loopback TCP server (optional shared-secret auth) +```python +from automation_file import start_autocontrol_socket_server + +server = start_autocontrol_socket_server( + host="127.0.0.1", port=9943, shared_secret="optional-secret", +) +``` + +Clients must prefix each payload with `AUTH \n` when `shared_secret` +is set. Non-loopback binds require `allow_non_loopback=True` explicitly. -1. Initialize Google Drive +### Start the HTTP action server ```python -from automation_file.remote.google_drive.driver_instance import driver_instance +from automation_file import start_http_action_server -driver_instance.later_init("token.json", "credentials.json") +server = start_http_action_server( + host="127.0.0.1", port=9944, shared_secret="optional-secret", +) + +# curl -H 'Authorization: Bearer optional-secret' \ +# -d '[["FA_create_dir",{"dir_path":"x"}]]' \ +# http://127.0.0.1:9944/actions ``` -2. Upload a File +### Retry and quota primitives ```python -from automation_file.remote.google_drive.upload.upload_to_driver import drive_upload_to_drive +from automation_file import retry_on_transient, Quota + +@retry_on_transient(max_attempts=5, backoff_base=0.5) +def flaky_network_call(): ... -drive_upload_to_drive("example.txt") +quota = Quota(max_bytes=50 * 1024 * 1024, max_seconds=30.0) +with quota.time_budget("bulk-upload"): + bulk_upload_work() ``` -3. Search Files +### Path traversal guard ```python -from automation_file.remote.google_drive.search.search_drive import drive_search_all_file +from automation_file import safe_join + +target = safe_join("/data/jobs", user_supplied_path) +# raises PathTraversalException if the resolved path escapes /data/jobs. +``` + +### Cloud / SFTP backends +Every backend is auto-registered by `build_default_registry()`, so `FA_s3_*`, +`FA_azure_blob_*`, `FA_dropbox_*`, and `FA_sftp_*` actions are available out +of the box — no separate `register_*_ops` call needed. + +```python +from automation_file import execute_action, s3_instance + +s3_instance.later_init(region_name="us-east-1") + +execute_action([ + ["FA_s3_upload_file", {"local_path": "report.csv", "bucket": "reports", "key": "report.csv"}], +]) +``` + +All backends (`s3`, `azure_blob`, `dropbox_api`, `sftp`) expose the same five +operations: `upload_file`, `upload_dir`, `download_file`, `delete_*`, `list_*`. +SFTP uses `paramiko.RejectPolicy` — unknown hosts are rejected, not auto-added. -files = drive_search_all_file() -print(files) +### GUI +```bash +python -m automation_file ui # or: python main_ui.py ``` -4. Start TCP Server ```python -from automation_file.utils.socket_server.file_automation_socket_server import start_autocontrol_socket_server +from automation_file import launch_ui +launch_ui() +``` + +Tabs: Local, HTTP, Google Drive, S3, Azure Blob, Dropbox, SFTP, JSON actions, +Servers. A persistent log panel at the bottom streams every result and error. + +### Scaffold an executor-based project +```python +from automation_file import create_project_dir + +create_project_dir("my_workflow") +``` + +## CLI -server = start_autocontrol_socket_server("localhost", 9943) +```bash +# Subcommands (one-shot operations) +python -m automation_file ui +python -m automation_file zip ./src out.zip --dir +python -m automation_file unzip out.zip ./restored +python -m automation_file download https://example.com/file.bin file.bin +python -m automation_file create-file hello.txt --content "hi" +python -m automation_file server --host 127.0.0.1 --port 9943 +python -m automation_file http-server --host 127.0.0.1 --port 9944 +python -m automation_file drive-upload my.txt --token token.json --credentials creds.json + +# Legacy flags (JSON action lists) +python -m automation_file --execute_file actions.json +python -m automation_file --execute_dir ./actions/ +python -m automation_file --execute_str '[["FA_create_dir",{"dir_path":"x"}]]' +python -m automation_file --create_project ./my_project ``` -# Example JSON Action +## JSON action format + +Each entry is either a bare command name, a `[name, kwargs]` pair, or a +`[name, args]` list: + ```json [ ["FA_create_file", {"file_path": "test.txt"}], ["FA_drive_upload_to_drive", {"file_path": "test.txt"}], ["FA_drive_search_all_file"] ] -``` \ No newline at end of file +``` + +## Documentation + +Full API documentation lives under `docs/` and can be built with Sphinx: + +```bash +pip install -r docs/requirements.txt +sphinx-build -b html docs/source docs/_build/html +``` + +See [`CLAUDE.md`](CLAUDE.md) for architecture notes, conventions, and security +considerations. diff --git a/automation_file/__init__.py b/automation_file/__init__.py index 033b2f2..7419549 100644 --- a/automation_file/__init__.py +++ b/automation_file/__init__.py @@ -1,33 +1,191 @@ -from automation_file.local.dir.dir_process import copy_dir, rename_dir, create_dir, remove_dir_tree -from automation_file.local.file.file_process import copy_file, remove_file, rename_file, copy_specify_extension_file, \ - copy_all_file_to_dir -from automation_file.local.zip.zip_process import zip_dir, zip_file, zip_info, zip_file_info, set_zip_password, \ - read_zip_file, unzip_file, unzip_all -from automation_file.remote.google_drive.delete.delete_manager import drive_delete_file -from automation_file.remote.google_drive.dir.folder_manager import drive_add_folder -from automation_file.remote.google_drive.download.download_file import drive_download_file, \ - drive_download_file_from_folder -from automation_file.remote.google_drive.driver_instance import driver_instance -from automation_file.remote.google_drive.search.search_drive import \ - drive_search_all_file, drive_search_field, drive_search_file_mimetype -from automation_file.remote.google_drive.share.share_file import \ - drive_share_file_to_anyone, drive_share_file_to_domain, drive_share_file_to_user -from automation_file.remote.google_drive.upload.upload_to_driver import \ - drive_upload_dir_to_folder, drive_upload_to_folder, drive_upload_dir_to_drive, drive_upload_to_drive -from automation_file.utils.executor.action_executor import execute_action, execute_files, add_command_to_executor -from automation_file.utils.file_process.get_dir_file_list import get_dir_files_as_list -from automation_file.utils.json.json_file import read_action_json -from automation_file.utils.project.create_project_structure import create_project_dir -from automation_file.remote.download.file import download_file +"""Public API for automation_file. + +This module is the facade: every publicly supported function, class, or shared +singleton is re-exported from here, so callers only ever need +``from automation_file import X``. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from automation_file.core.action_executor import ( + ActionExecutor, + add_command_to_executor, + execute_action, + execute_action_parallel, + execute_files, + executor, + validate_action, +) +from automation_file.core.action_registry import ActionRegistry, build_default_registry +from automation_file.core.callback_executor import CallbackExecutor +from automation_file.core.json_store import read_action_json, write_action_json +from automation_file.core.package_loader import PackageLoader +from automation_file.core.quota import Quota +from automation_file.core.retry import retry_on_transient +from automation_file.local.dir_ops import copy_dir, create_dir, remove_dir_tree, rename_dir +from automation_file.local.file_ops import ( + copy_all_file_to_dir, + copy_file, + copy_specify_extension_file, + create_file, + remove_file, + rename_file, +) +from automation_file.local.safe_paths import is_within, safe_join +from automation_file.local.zip_ops import ( + read_zip_file, + set_zip_password, + unzip_all, + unzip_file, + zip_dir, + zip_file, + zip_file_info, + zip_info, +) +from automation_file.project.project_builder import ProjectBuilder, create_project_dir +from automation_file.remote.azure_blob import ( + AzureBlobClient, + azure_blob_instance, + register_azure_blob_ops, +) +from automation_file.remote.dropbox_api import ( + DropboxClient, + dropbox_instance, + register_dropbox_ops, +) +from automation_file.remote.google_drive.client import GoogleDriveClient, driver_instance +from automation_file.remote.google_drive.delete_ops import drive_delete_file +from automation_file.remote.google_drive.download_ops import ( + drive_download_file, + drive_download_file_from_folder, +) +from automation_file.remote.google_drive.folder_ops import drive_add_folder +from automation_file.remote.google_drive.search_ops import ( + drive_search_all_file, + drive_search_field, + drive_search_file_mimetype, +) +from automation_file.remote.google_drive.share_ops import ( + drive_share_file_to_anyone, + drive_share_file_to_domain, + drive_share_file_to_user, +) +from automation_file.remote.google_drive.upload_ops import ( + drive_upload_dir_to_drive, + drive_upload_dir_to_folder, + drive_upload_to_drive, + drive_upload_to_folder, +) +from automation_file.remote.http_download import download_file +from automation_file.remote.s3 import S3Client, register_s3_ops, s3_instance +from automation_file.remote.sftp import SFTPClient, register_sftp_ops, sftp_instance +from automation_file.remote.url_validator import validate_http_url +from automation_file.server.http_server import HTTPActionServer, start_http_action_server +from automation_file.server.tcp_server import ( + TCPActionServer, + start_autocontrol_socket_server, +) +from automation_file.utils.file_discovery import get_dir_files_as_list + +if TYPE_CHECKING: + from automation_file.ui.launcher import ( + launch_ui as launch_ui, # pylint: disable=useless-import-alias + ) + +# Shared callback executor + package loader wired to the shared registry. +callback_executor: CallbackExecutor = CallbackExecutor(executor.registry) +package_manager: PackageLoader = PackageLoader(executor.registry) + + +def __getattr__(name: str) -> Any: + if name == "launch_ui": + from automation_file.ui.launcher import launch_ui as _launch_ui + + return _launch_ui + raise AttributeError(f"module 'automation_file' has no attribute {name!r}") + __all__ = [ - "copy_file", "rename_file", "remove_file", "copy_all_file_to_dir", "copy_specify_extension_file", - "copy_dir", "create_dir", "remove_dir_tree", "zip_dir", "zip_file", "zip_info", - "zip_file_info", "set_zip_password", "unzip_file", "read_zip_file", "rename_dir", - "unzip_all", "driver_instance", "drive_search_all_file", "drive_search_field", "drive_search_file_mimetype", - "drive_upload_dir_to_folder", "drive_upload_to_folder", "drive_upload_dir_to_drive", "drive_upload_to_drive", - "drive_add_folder", "drive_share_file_to_anyone", "drive_share_file_to_domain", "drive_share_file_to_user", - "drive_delete_file", "drive_download_file", "drive_download_file_from_folder", "execute_action", "execute_files", - "add_command_to_executor", "read_action_json", "get_dir_files_as_list", "create_project_dir", - "download_file" + # Core + "ActionExecutor", + "ActionRegistry", + "CallbackExecutor", + "PackageLoader", + "Quota", + "build_default_registry", + "execute_action", + "execute_action_parallel", + "execute_files", + "validate_action", + "retry_on_transient", + "add_command_to_executor", + "read_action_json", + "write_action_json", + "executor", + "callback_executor", + "package_manager", + # Local + "copy_file", + "rename_file", + "remove_file", + "copy_all_file_to_dir", + "copy_specify_extension_file", + "create_file", + "copy_dir", + "create_dir", + "remove_dir_tree", + "rename_dir", + "zip_dir", + "zip_file", + "zip_info", + "zip_file_info", + "set_zip_password", + "unzip_file", + "read_zip_file", + "unzip_all", + "safe_join", + "is_within", + # Remote + "download_file", + "validate_http_url", + "GoogleDriveClient", + "driver_instance", + "drive_search_all_file", + "drive_search_field", + "drive_search_file_mimetype", + "drive_upload_dir_to_folder", + "drive_upload_to_folder", + "drive_upload_dir_to_drive", + "drive_upload_to_drive", + "drive_add_folder", + "drive_share_file_to_anyone", + "drive_share_file_to_domain", + "drive_share_file_to_user", + "drive_delete_file", + "drive_download_file", + "drive_download_file_from_folder", + "S3Client", + "s3_instance", + "register_s3_ops", + "AzureBlobClient", + "azure_blob_instance", + "register_azure_blob_ops", + "DropboxClient", + "dropbox_instance", + "register_dropbox_ops", + "SFTPClient", + "sftp_instance", + "register_sftp_ops", + # Server / Project / Utils + "TCPActionServer", + "start_autocontrol_socket_server", + "HTTPActionServer", + "start_http_action_server", + "ProjectBuilder", + "create_project_dir", + "get_dir_files_as_list", + # UI (lazy-loaded) + "launch_ui", ] diff --git a/automation_file/__main__.py b/automation_file/__main__.py index 95068b5..c050b8a 100644 --- a/automation_file/__main__.py +++ b/automation_file/__main__.py @@ -1,68 +1,218 @@ -# argparse +"""CLI entry point (``python -m automation_file``). + +Supports three invocation styles: + +* Legacy flags (``-e``, ``-d``, ``-c``, ``--execute_str``) — run JSON action + lists without writing Python. +* Subcommands (``zip``, ``unzip``, ``download``, ``server``, ``http-server``, + ``drive-upload``, ``ui``) — wrap the most common facade calls so users do + not need to hand-author JSON for one-shot operations. +* No arguments — prints help and exits non-zero. +""" + +from __future__ import annotations + import argparse import json import sys +import time +from collections.abc import Callable +from typing import Any + +from automation_file.core.action_executor import execute_action, execute_files +from automation_file.core.json_store import read_action_json +from automation_file.exceptions import ArgparseException +from automation_file.local.file_ops import create_file +from automation_file.local.zip_ops import unzip_all, zip_dir, zip_file +from automation_file.project.project_builder import create_project_dir +from automation_file.remote.http_download import download_file +from automation_file.utils.file_discovery import get_dir_files_as_list + + +def _execute_file(path: str) -> Any: + return execute_action(read_action_json(path)) + + +def _execute_dir(path: str) -> Any: + return execute_files(get_dir_files_as_list(path)) + + +def _execute_str(raw: str) -> Any: + return execute_action(json.loads(raw)) + + +_LEGACY_DISPATCH: dict[str, Callable[[str], Any]] = { + "execute_file": _execute_file, + "execute_dir": _execute_dir, + "execute_str": _execute_str, + "create_project": create_project_dir, +} + + +def _cmd_zip(args: argparse.Namespace) -> int: + if args.source_is_dir: + zip_dir(args.source, args.target) + else: + zip_file(args.source, args.target) + return 0 + + +def _cmd_unzip(args: argparse.Namespace) -> int: + unzip_all(args.archive, args.target_dir, password=args.password) + return 0 + + +def _cmd_download(args: argparse.Namespace) -> int: + ok = download_file(args.url, args.output) + return 0 if ok else 1 + + +def _cmd_create_file(args: argparse.Namespace) -> int: + create_file(args.path, args.content or "") + return 0 + + +def _cmd_server(args: argparse.Namespace) -> int: + from automation_file.server.tcp_server import start_autocontrol_socket_server + + start_autocontrol_socket_server( + host=args.host, + port=args.port, + allow_non_loopback=args.allow_non_loopback, + shared_secret=args.shared_secret, + ) + _sleep_forever() + return 0 + + +def _cmd_http_server(args: argparse.Namespace) -> int: + from automation_file.server.http_server import start_http_action_server + + start_http_action_server( + host=args.host, + port=args.port, + allow_non_loopback=args.allow_non_loopback, + shared_secret=args.shared_secret, + ) + _sleep_forever() + return 0 + + +def _cmd_ui(_args: argparse.Namespace) -> int: + from automation_file.ui.launcher import launch_ui + + return launch_ui() + + +def _cmd_drive_upload(args: argparse.Namespace) -> int: + from automation_file.remote.google_drive.client import driver_instance + from automation_file.remote.google_drive.upload_ops import ( + drive_upload_to_drive, + drive_upload_to_folder, + ) + + driver_instance.later_init(args.token, args.credentials) + if args.folder_id: + result = drive_upload_to_folder(args.folder_id, args.file, args.name) + else: + result = drive_upload_to_drive(args.file, args.name) + return 0 if result is not None else 1 + + +def _sleep_forever() -> None: + while True: + time.sleep(3600) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="automation_file") + parser.add_argument("-e", "--execute_file", help="path to an action JSON file") + parser.add_argument("-d", "--execute_dir", help="directory containing action JSON files") + parser.add_argument("-c", "--create_project", help="scaffold a project at this path") + parser.add_argument("--execute_str", help="JSON action list as a string") + + subparsers = parser.add_subparsers(dest="command") + + zip_parser = subparsers.add_parser("zip", help="zip a file or directory") + zip_parser.add_argument("source") + zip_parser.add_argument("target") + zip_parser.add_argument( + "--dir", + dest="source_is_dir", + action="store_true", + help="treat source as a directory (zips the tree instead of one file)", + ) + zip_parser.set_defaults(handler=_cmd_zip) + + unzip_parser = subparsers.add_parser("unzip", help="extract an archive") + unzip_parser.add_argument("archive") + unzip_parser.add_argument("target_dir") + unzip_parser.add_argument("--password", default=None) + unzip_parser.set_defaults(handler=_cmd_unzip) + + download_parser = subparsers.add_parser("download", help="SSRF-validated HTTP download") + download_parser.add_argument("url") + download_parser.add_argument("output") + download_parser.set_defaults(handler=_cmd_download) + + touch_parser = subparsers.add_parser("create-file", help="write a text file") + touch_parser.add_argument("path") + touch_parser.add_argument("--content", default="") + touch_parser.set_defaults(handler=_cmd_create_file) + + server_parser = subparsers.add_parser("server", help="run the TCP action server") + server_parser.add_argument("--host", default="localhost") + server_parser.add_argument("--port", type=int, default=9943) + server_parser.add_argument("--allow-non-loopback", action="store_true") + server_parser.add_argument("--shared-secret", default=None) + server_parser.set_defaults(handler=_cmd_server) + + http_parser = subparsers.add_parser("http-server", help="run the HTTP action server") + http_parser.add_argument("--host", default="127.0.0.1") + http_parser.add_argument("--port", type=int, default=9944) + http_parser.add_argument("--allow-non-loopback", action="store_true") + http_parser.add_argument("--shared-secret", default=None) + http_parser.set_defaults(handler=_cmd_http_server) + + ui_parser = subparsers.add_parser("ui", help="launch the PySide6 GUI") + ui_parser.set_defaults(handler=_cmd_ui) + + drive_parser = subparsers.add_parser("drive-upload", help="upload a file to Google Drive") + drive_parser.add_argument("file") + drive_parser.add_argument("--token", required=True) + drive_parser.add_argument("--credentials", required=True) + drive_parser.add_argument("--folder-id", default=None) + drive_parser.add_argument("--name", default=None) + drive_parser.set_defaults(handler=_cmd_drive_upload) + + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + + if getattr(args, "command", None): + return args.handler(args) + + ran = False + for key, handler in _LEGACY_DISPATCH.items(): + value = getattr(args, key, None) + if value is None: + continue + handler(value) + ran = True + if not ran: + raise ArgparseException("no argument supplied; try --help") + return 0 -from automation_file.utils.exception.exception_tags import \ - argparse_get_wrong_data -from automation_file.utils.exception.exceptions import \ - ArgparseException -from automation_file.utils.executor.action_executor import execute_action -from automation_file.utils.executor.action_executor import execute_files -from automation_file.utils.file_process.get_dir_file_list import \ - get_dir_files_as_list -from automation_file.utils.json.json_file import read_action_json -from automation_file.utils.project.create_project_structure import create_project_dir if __name__ == "__main__": try: - def preprocess_execute_action(file_path: str): - execute_action(read_action_json(file_path)) - - - def preprocess_execute_files(file_path: str): - execute_files(get_dir_files_as_list(file_path)) - - - def preprocess_read_str_execute_action(execute_str: str): - if sys.platform in ["win32", "cygwin", "msys"]: - json_data = json.loads(execute_str) - execute_str = json.loads(json_data) - else: - execute_str = json.loads(execute_str) - execute_action(execute_str) - - - argparse_event_dict = { - "execute_file": preprocess_execute_action, - "execute_dir": preprocess_execute_files, - "execute_str": preprocess_read_str_execute_action, - "create_project": create_project_dir - } - parser = argparse.ArgumentParser() - parser.add_argument( - "-e", "--execute_file", - type=str, help="choose action file to execute" - ) - parser.add_argument( - "-d", "--execute_dir", - type=str, help="choose dir include action file to execute" - ) - parser.add_argument( - "-c", "--create_project", - type=str, help="create project with template" - ) - parser.add_argument( - "--execute_str", - type=str, help="execute json str" - ) - args = parser.parse_args() - args = vars(args) - for key, value in args.items(): - if value is not None: - argparse_event_dict.get(key)(value) - if all(value is None for value in args.values()): - raise ArgparseException(argparse_get_wrong_data) - except Exception as error: + sys.exit(main()) + except ArgparseException as error: + print(repr(error), file=sys.stderr) + sys.exit(1) + except Exception as error: # pylint: disable=broad-except print(repr(error), file=sys.stderr) sys.exit(1) diff --git a/automation_file/local/dir/__init__.py b/automation_file/core/__init__.py similarity index 100% rename from automation_file/local/dir/__init__.py rename to automation_file/core/__init__.py diff --git a/automation_file/core/action_executor.py b/automation_file/core/action_executor.py new file mode 100644 index 0000000..b7c7972 --- /dev/null +++ b/automation_file/core/action_executor.py @@ -0,0 +1,223 @@ +"""Action executor (Facade + Template Method over :class:`ActionRegistry`). + +An *action* is one of three shapes inside a JSON list: + +* ``[name]`` — call the registered command with no arguments +* ``[name, {kwargs}]`` — call ``command(**kwargs)`` +* ``[name, [args]]`` — call ``command(*args)`` + +``ActionExecutor.execute_action`` iterates a list of actions and returns a +dict mapping each action's string form to either its return value or the +``repr`` of the exception it raised. This keeps one bad action from aborting +the batch, which is important when running against Google Drive where +transient errors are common. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from concurrent.futures import ThreadPoolExecutor +from typing import Any + +from automation_file.core.action_registry import ActionRegistry, build_default_registry +from automation_file.core.json_store import read_action_json +from automation_file.exceptions import ExecuteActionException, ValidationException +from automation_file.logging_config import file_automation_logger + + +class ActionExecutor: + """Execute named actions resolved through an :class:`ActionRegistry`.""" + + def __init__(self, registry: ActionRegistry | None = None) -> None: + self.registry: ActionRegistry = registry or build_default_registry() + self.registry.register_many( + { + "FA_execute_action": self.execute_action, + "FA_execute_files": self.execute_files, + "FA_execute_action_parallel": self.execute_action_parallel, + "FA_validate": self.validate, + } + ) + + # Template-method: single action ------------------------------------ + def _execute_event(self, action: list) -> Any: + name, payload_kind, payload = self._parse_action(action) + command = self.registry.resolve(name) + if command is None: + raise ExecuteActionException(f"unknown action: {name!r}") + if payload_kind == "none": + return command() + if payload_kind == "kwargs": + return command(**payload) + return command(*payload) + + @staticmethod + def _parse_action(action: list) -> tuple[str, str, Any]: + if not isinstance(action, list) or not action: + raise ExecuteActionException(f"malformed action: {action!r}") + name = action[0] + if not isinstance(name, str): + raise ExecuteActionException(f"action name must be str: {action!r}") + if len(action) == 1: + return name, "none", None + if len(action) == 2: + payload = action[1] + if isinstance(payload, dict): + return name, "kwargs", payload + if isinstance(payload, list): + return name, "args", payload + raise ExecuteActionException( + f"action {name!r} payload must be dict or list, got {type(payload).__name__}" + ) + raise ExecuteActionException(f"action has too many elements: {action!r}") + + # Public API -------------------------------------------------------- + def validate(self, action_list: list | Mapping[str, Any]) -> list[str]: + """Validate shape and resolve every name; return the list of action names. + + Raises :class:`ValidationException` on the first problem. Useful for + fail-fast checks before executing an entire batch. + """ + actions = self._coerce(action_list) + names: list[str] = [] + for action in actions: + try: + name, _, _ = self._parse_action(action) + except ExecuteActionException as error: + raise ValidationException(str(error)) from error + if self.registry.resolve(name) is None: + raise ValidationException(f"unknown action: {name!r}") + names.append(name) + return names + + def execute_action( + self, + action_list: list | Mapping[str, Any], + dry_run: bool = False, + validate_first: bool = False, + ) -> dict[str, Any]: + """Execute every action; return ``{"execute: ": result|repr(error)}``. + + ``dry_run=True`` logs and records the resolved name without invoking the + command. ``validate_first=True`` runs :meth:`validate` before touching + any action so a typo aborts the whole batch up-front. + """ + actions = self._coerce(action_list) + if validate_first: + self.validate(actions) + results: dict[str, Any] = {} + for action in actions: + key = f"execute: {action}" + results[key] = self._run_one(action, dry_run=dry_run) + return results + + def execute_action_parallel( + self, + action_list: list | Mapping[str, Any], + max_workers: int = 4, + dry_run: bool = False, + ) -> dict[str, Any]: + """Execute actions concurrently with a ``ThreadPoolExecutor``. + + Callers are responsible for ensuring the chosen actions are independent + (no shared file target, no ordering dependency). + """ + actions = self._coerce(action_list) + results: dict[str, Any] = {} + with ThreadPoolExecutor(max_workers=max_workers) as pool: + futures = [ + (index, action, pool.submit(self._run_one, action, dry_run)) + for index, action in enumerate(actions) + ] + for index, action, future in futures: + results[f"execute[{index}]: {action}"] = future.result() + return results + + def execute_files(self, execute_files_list: list[str]) -> list[dict[str, Any]]: + """Execute every JSON file's action list and return their results.""" + return [self.execute_action(read_action_json(path)) for path in execute_files_list] + + def add_command_to_executor(self, command_dict: Mapping[str, Any]) -> None: + """Register every ``name -> callable`` pair (Registry facade).""" + file_automation_logger.info("add_command_to_executor: %s", list(command_dict.keys())) + self.registry.register_many(command_dict) + + # Internals --------------------------------------------------------- + def _run_one(self, action: list, dry_run: bool) -> Any: + try: + if dry_run: + name, kind, payload = self._parse_action(action) + if self.registry.resolve(name) is None: + raise ExecuteActionException(f"unknown action: {name!r}") + file_automation_logger.info( + "dry_run: %s kind=%s payload=%r", + name, + kind, + payload, + ) + return f"dry_run:{name}" + value = self._execute_event(action) + file_automation_logger.info("execute_action: %s", action) + return value + except ExecuteActionException as error: + file_automation_logger.error("execute_action malformed: %r", error) + return repr(error) + except Exception as error: # pylint: disable=broad-except + file_automation_logger.error("execute_action runtime error: %r", error) + return repr(error) + + @staticmethod + def _coerce(action_list: list | Mapping[str, Any]) -> list: + if isinstance(action_list, Mapping): + nested = action_list.get("auto_control") + if nested is None: + raise ExecuteActionException("dict action list missing 'auto_control'") + action_list = nested + if not isinstance(action_list, list): + raise ExecuteActionException( + f"action_list must be list, got {type(action_list).__name__}" + ) + if not action_list: + raise ExecuteActionException("action_list is empty") + return action_list + + +# Default shared executor — built once, mutated in place by plugins. +executor: ActionExecutor = ActionExecutor() + + +def execute_action( + action_list: list | Mapping[str, Any], + dry_run: bool = False, + validate_first: bool = False, +) -> dict[str, Any]: + """Module-level shim that delegates to the shared executor.""" + return executor.execute_action( + action_list, + dry_run=dry_run, + validate_first=validate_first, + ) + + +def execute_action_parallel( + action_list: list | Mapping[str, Any], + max_workers: int = 4, + dry_run: bool = False, +) -> dict[str, Any]: + """Module-level shim that delegates to the shared executor.""" + return executor.execute_action_parallel(action_list, max_workers, dry_run) + + +def validate_action(action_list: list | Mapping[str, Any]) -> list[str]: + """Module-level shim that delegates to :meth:`ActionExecutor.validate`.""" + return executor.validate(action_list) + + +def execute_files(execute_files_list: list[str]) -> list[dict[str, Any]]: + """Module-level shim that delegates to the shared executor.""" + return executor.execute_files(execute_files_list) + + +def add_command_to_executor(command_dict: Mapping[str, Any]) -> None: + """Module-level shim that delegates to the shared executor.""" + executor.add_command_to_executor(command_dict) diff --git a/automation_file/core/action_registry.py b/automation_file/core/action_registry.py new file mode 100644 index 0000000..26d882a --- /dev/null +++ b/automation_file/core/action_registry.py @@ -0,0 +1,154 @@ +"""Registry of named callables (Registry + Command pattern). + +The registry decouples "what to run" (a string name inside a JSON action list) +from "how to run it" (a Python callable). Executors delegate name resolution +to an :class:`ActionRegistry`, which keeps look-up O(1) and lets plugins add +commands at runtime without touching the executor class. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable, Iterator, Mapping +from typing import Any + +from automation_file.exceptions import AddCommandException +from automation_file.logging_config import file_automation_logger + +Command = Callable[..., Any] + + +class ActionRegistry: + """Mapping of action name -> callable.""" + + def __init__(self, initial: Mapping[str, Command] | None = None) -> None: + self._commands: dict[str, Command] = {} + if initial: + for name, command in initial.items(): + self.register(name, command) + + def register(self, name: str, command: Command) -> None: + """Add or overwrite a command. Raises if ``command`` is not callable.""" + if not callable(command): + raise AddCommandException(f"{name!r} is not callable") + self._commands[name] = command + + def register_many(self, mapping: Mapping[str, Command]) -> None: + """Register every ``name -> command`` pair in ``mapping``.""" + for name, command in mapping.items(): + self.register(name, command) + + def update(self, mapping: Mapping[str, Command]) -> None: + """Alias for :meth:`register_many` (dict-compatible).""" + self.register_many(mapping) + + def unregister(self, name: str) -> None: + self._commands.pop(name, None) + + def resolve(self, name: str) -> Command | None: + return self._commands.get(name) + + def __contains__(self, name: object) -> bool: + return isinstance(name, str) and name in self._commands + + def __len__(self) -> int: + return len(self._commands) + + def __iter__(self) -> Iterator[str]: + return iter(self._commands) + + def names(self) -> Iterable[str]: + return self._commands.keys() + + @property + def event_dict(self) -> dict[str, Command]: + """Backwards-compatible view used by older ``package_manager`` style code.""" + return self._commands + + +def _local_commands() -> dict[str, Command]: + from automation_file.local import dir_ops, file_ops, zip_ops + + return { + # Files + "FA_create_file": file_ops.create_file, + "FA_copy_file": file_ops.copy_file, + "FA_rename_file": file_ops.rename_file, + "FA_remove_file": file_ops.remove_file, + "FA_copy_all_file_to_dir": file_ops.copy_all_file_to_dir, + "FA_copy_specify_extension_file": file_ops.copy_specify_extension_file, + # Directories + "FA_copy_dir": dir_ops.copy_dir, + "FA_create_dir": dir_ops.create_dir, + "FA_remove_dir_tree": dir_ops.remove_dir_tree, + "FA_rename_dir": dir_ops.rename_dir, + # Zip + "FA_zip_dir": zip_ops.zip_dir, + "FA_zip_file": zip_ops.zip_file, + "FA_zip_info": zip_ops.zip_info, + "FA_zip_file_info": zip_ops.zip_file_info, + "FA_set_zip_password": zip_ops.set_zip_password, + "FA_unzip_file": zip_ops.unzip_file, + "FA_read_zip_file": zip_ops.read_zip_file, + "FA_unzip_all": zip_ops.unzip_all, + } + + +def _drive_commands() -> dict[str, Command]: + from automation_file.remote.google_drive import ( + client, + delete_ops, + download_ops, + folder_ops, + search_ops, + share_ops, + upload_ops, + ) + + return { + "FA_drive_later_init": client.driver_instance.later_init, + "FA_drive_search_all_file": search_ops.drive_search_all_file, + "FA_drive_search_field": search_ops.drive_search_field, + "FA_drive_search_file_mimetype": search_ops.drive_search_file_mimetype, + "FA_drive_upload_dir_to_folder": upload_ops.drive_upload_dir_to_folder, + "FA_drive_upload_to_folder": upload_ops.drive_upload_to_folder, + "FA_drive_upload_dir_to_drive": upload_ops.drive_upload_dir_to_drive, + "FA_drive_upload_to_drive": upload_ops.drive_upload_to_drive, + "FA_drive_add_folder": folder_ops.drive_add_folder, + "FA_drive_share_file_to_anyone": share_ops.drive_share_file_to_anyone, + "FA_drive_share_file_to_domain": share_ops.drive_share_file_to_domain, + "FA_drive_share_file_to_user": share_ops.drive_share_file_to_user, + "FA_drive_delete_file": delete_ops.drive_delete_file, + "FA_drive_download_file": download_ops.drive_download_file, + "FA_drive_download_file_from_folder": download_ops.drive_download_file_from_folder, + } + + +def _http_commands() -> dict[str, Command]: + from automation_file.remote import http_download + + return {"FA_download_file": http_download.download_file} + + +def _register_cloud_backends(registry: ActionRegistry) -> None: + from automation_file.remote.azure_blob import register_azure_blob_ops + from automation_file.remote.dropbox_api import register_dropbox_ops + from automation_file.remote.s3 import register_s3_ops + from automation_file.remote.sftp import register_sftp_ops + + register_s3_ops(registry) + register_azure_blob_ops(registry) + register_dropbox_ops(registry) + register_sftp_ops(registry) + + +def build_default_registry() -> ActionRegistry: + """Return a registry pre-populated with every built-in ``FA_*`` action.""" + registry = ActionRegistry() + registry.register_many(_local_commands()) + registry.register_many(_http_commands()) + registry.register_many(_drive_commands()) + _register_cloud_backends(registry) + file_automation_logger.info( + "action_registry: built default registry with %d commands", len(registry) + ) + return registry diff --git a/automation_file/core/callback_executor.py b/automation_file/core/callback_executor.py new file mode 100644 index 0000000..459a6a6 --- /dev/null +++ b/automation_file/core/callback_executor.py @@ -0,0 +1,63 @@ +"""Callback executor — runs a trigger, then a callback. + +Implements the "do X then do Y" flow many automation JSON files want. The +registry is shared with :class:`ActionExecutor`, so adding a command to one +adds it to the other. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from typing import Any + +from automation_file.core.action_registry import ActionRegistry +from automation_file.exceptions import CallbackExecutorException +from automation_file.logging_config import file_automation_logger + +_VALID_METHODS = frozenset({"kwargs", "args"}) + + +class CallbackExecutor: + """Invoke ``trigger(**kwargs)`` then ``callback(*args | **kwargs)``.""" + + def __init__(self, registry: ActionRegistry) -> None: + self.registry: ActionRegistry = registry + + def callback_function( + self, + trigger_function_name: str, + callback_function: Callable[..., Any], + callback_function_param: Mapping[str, Any] | list[Any] | None = None, + callback_param_method: str = "kwargs", + **kwargs: Any, + ) -> Any: + trigger = self.registry.resolve(trigger_function_name) + if trigger is None: + raise CallbackExecutorException(f"unknown trigger: {trigger_function_name!r}") + if callback_param_method not in _VALID_METHODS: + raise CallbackExecutorException( + f"callback_param_method must be 'kwargs' or 'args', got {callback_param_method!r}" + ) + + file_automation_logger.info("callback: trigger=%s kwargs=%s", trigger_function_name, kwargs) + return_value = trigger(**kwargs) + + if callback_function_param is None: + callback_function() + elif callback_param_method == "kwargs": + if not isinstance(callback_function_param, Mapping): + raise CallbackExecutorException( + "callback_param_method='kwargs' requires a mapping payload" + ) + callback_function(**callback_function_param) + else: + if not isinstance(callback_function_param, (list, tuple)): + raise CallbackExecutorException( + "callback_param_method='args' requires a list/tuple payload" + ) + callback_function(*callback_function_param) + + file_automation_logger.info( + "callback: done trigger=%s callback=%r", trigger_function_name, callback_function + ) + return return_value diff --git a/automation_file/core/json_store.py b/automation_file/core/json_store.py new file mode 100644 index 0000000..ac887df --- /dev/null +++ b/automation_file/core/json_store.py @@ -0,0 +1,43 @@ +"""JSON persistence for action lists. + +Reads/writes are serialised through a module-level lock so concurrent callers +cannot interleave writes against the same file. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from threading import Lock +from typing import Any + +from automation_file.exceptions import JsonActionException +from automation_file.logging_config import file_automation_logger + +_lock = Lock() + + +def read_action_json(json_file_path: str) -> Any: + """Return the parsed JSON content at ``json_file_path``.""" + with _lock: + path = Path(json_file_path) + if not path.is_file(): + raise JsonActionException(f"can't read JSON file: {json_file_path}") + try: + with path.open(encoding="utf-8") as read_file: + data = json.load(read_file) + except (OSError, json.JSONDecodeError) as error: + raise JsonActionException(f"can't read JSON file: {json_file_path}") from error + file_automation_logger.info("read_action_json: %s", json_file_path) + return data + + +def write_action_json(json_save_path: str, action_json: Any) -> None: + """Write ``action_json`` to ``json_save_path`` as pretty UTF-8 JSON.""" + with _lock: + try: + with open(json_save_path, "w", encoding="utf-8") as file_to_write: + json.dump(action_json, file_to_write, indent=4, ensure_ascii=False) + except (OSError, TypeError) as error: + raise JsonActionException(f"can't write JSON file: {json_save_path}") from error + file_automation_logger.info("write_action_json: %s", json_save_path) diff --git a/automation_file/core/package_loader.py b/automation_file/core/package_loader.py new file mode 100644 index 0000000..46d4564 --- /dev/null +++ b/automation_file/core/package_loader.py @@ -0,0 +1,59 @@ +"""Dynamic plugin registration into an :class:`ActionRegistry`. + +``PackageLoader`` imports an external package by name and registers every +top-level function / class / builtin under the key ``"_"``. +""" + +from __future__ import annotations + +from importlib import import_module +from importlib.util import find_spec +from inspect import getmembers, isbuiltin, isclass, isfunction +from types import ModuleType + +from automation_file.core.action_registry import ActionRegistry +from automation_file.logging_config import file_automation_logger + + +class PackageLoader: + """Load packages lazily and register their public callables.""" + + def __init__(self, registry: ActionRegistry) -> None: + self.registry: ActionRegistry = registry + self._cache: dict[str, ModuleType] = {} + + def load(self, package: str) -> ModuleType | None: + """Import ``package`` once and return the module (cached).""" + cached = self._cache.get(package) + if cached is not None: + return cached + spec = find_spec(package) + if spec is None: + file_automation_logger.error("PackageLoader: cannot find %s", package) + return None + try: + # nosemgrep: python.lang.security.audit.non-literal-import.non-literal-import + # `package` is a trusted caller-supplied name (see PackageLoader docstring and + # the CLAUDE.md security note on plugin loading); it is not untrusted input. + module = import_module(spec.name) + except (ImportError, ModuleNotFoundError) as error: + file_automation_logger.error("PackageLoader import error: %r", error) + return None + self._cache[package] = module + return module + + def add_package_to_executor(self, package: str) -> int: + """Register every function / class / builtin from ``package``. + + Returns the number of commands that were registered. + """ + module = self.load(package) + if module is None: + return 0 + count = 0 + for predicate in (isfunction, isbuiltin, isclass): + for member_name, member in getmembers(module, predicate): + self.registry.register(f"{package}_{member_name}", member) + count += 1 + file_automation_logger.info("PackageLoader: registered %d members from %s", count, package) + return count diff --git a/automation_file/core/quota.py b/automation_file/core/quota.py new file mode 100644 index 0000000..dd002b3 --- /dev/null +++ b/automation_file/core/quota.py @@ -0,0 +1,72 @@ +"""Per-action quota enforcement. + +``Quota`` bundles a maximum byte size and maximum duration. Callers use +``Quota.check_size(bytes)`` before an I/O-heavy action and wrap the action in +``with quota.time_budget(label):`` to bound wall-clock time. +""" + +from __future__ import annotations + +import time +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass + +from automation_file.exceptions import QuotaExceededException +from automation_file.logging_config import file_automation_logger + + +@dataclass(frozen=True) +class Quota: + """Bundle of per-action limits. + + ``max_bytes`` <= 0 means no size cap; ``max_seconds`` <= 0 means no time + cap. Defaults allow callers to share one ``Quota`` instance across many + actions with fine-grained overrides at each call site. + """ + + max_bytes: int = 0 + max_seconds: float = 0.0 + + def check_size(self, nbytes: int, label: str = "action") -> None: + """Raise :class:`QuotaExceededException` if ``nbytes`` exceeds the cap.""" + if self.max_bytes > 0 and nbytes > self.max_bytes: + raise QuotaExceededException(f"{label} size {nbytes} exceeds quota {self.max_bytes}") + + @contextmanager + def time_budget(self, label: str = "action") -> Iterator[None]: + """Context manager that raises if the enclosed block runs past the cap.""" + start = time.monotonic() + try: + yield + finally: + elapsed = time.monotonic() - start + if self.max_seconds > 0 and elapsed > self.max_seconds: + file_automation_logger.warning( + "quota: %s took %.2fs > %.2fs", + label, + elapsed, + self.max_seconds, + ) + raise QuotaExceededException( + f"{label} took {elapsed:.2f}s exceeding quota {self.max_seconds:.2f}s" + ) + + def wraps(self, label: str, size_fn=None): + """Return a decorator that enforces the time budget around ``func``. + + If ``size_fn`` is provided it is called with the function's return + value to derive a byte count for :meth:`check_size`. + """ + + def decorator(func): + def wrapper(*args, **kwargs): + with self.time_budget(label): + result = func(*args, **kwargs) + if size_fn is not None: + self.check_size(int(size_fn(result)), label=label) + return result + + return wrapper + + return decorator diff --git a/automation_file/core/retry.py b/automation_file/core/retry.py new file mode 100644 index 0000000..5bf39c2 --- /dev/null +++ b/automation_file/core/retry.py @@ -0,0 +1,62 @@ +"""Retry helper for transient network failures. + +``retry_on_transient`` is a small wrapper around exponential back-off. It is +intentionally dependency-free so that modules which do not actually use +``requests`` or ``googleapiclient`` can import it without pulling those in. +""" + +from __future__ import annotations + +import time +from collections.abc import Callable +from functools import wraps +from typing import Any, TypeVar + +from automation_file.exceptions import RetryExhaustedException +from automation_file.logging_config import file_automation_logger + +F = TypeVar("F", bound=Callable[..., Any]) + + +def retry_on_transient( + max_attempts: int = 3, + backoff_base: float = 0.5, + backoff_cap: float = 8.0, + retriable: tuple[type[BaseException], ...] = (ConnectionError, TimeoutError, OSError), +) -> Callable[[F], F]: + """Return a decorator that retries ``retriable`` exceptions with back-off. + + On the final failure raises :class:`RetryExhaustedException` chained to the + underlying error so callers can still inspect the cause. + """ + if max_attempts < 1: + raise ValueError("max_attempts must be >= 1") + + def decorator(func: F) -> F: + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + last_error: BaseException | None = None + for attempt in range(1, max_attempts + 1): + try: + return func(*args, **kwargs) + except retriable as error: + last_error = error + if attempt >= max_attempts: + break + delay = min(backoff_cap, backoff_base * (2 ** (attempt - 1))) + file_automation_logger.warning( + "retry_on_transient: %s attempt %d/%d failed (%r); sleeping %.2fs", + func.__name__, + attempt, + max_attempts, + error, + delay, + ) + time.sleep(delay) + raise RetryExhaustedException( + f"{func.__name__} failed after {max_attempts} attempts" + ) from last_error + + return wrapper # type: ignore[return-value] + + return decorator diff --git a/automation_file/exceptions.py b/automation_file/exceptions.py new file mode 100644 index 0000000..7908413 --- /dev/null +++ b/automation_file/exceptions.py @@ -0,0 +1,77 @@ +"""Exception hierarchy for automation_file. + +All custom exceptions inherit from ``FileAutomationException`` so callers can +filter with a single ``except`` and still distinguish specific failures. +""" + +from __future__ import annotations + + +class FileAutomationException(Exception): + """Root of the automation_file exception tree.""" + + +class FileNotExistsException(FileAutomationException): + """Raised when a required source file is missing.""" + + +class DirNotExistsException(FileAutomationException): + """Raised when a required directory is missing.""" + + +class ZipInputException(FileAutomationException): + """Raised when a zip helper receives an unsupported input type.""" + + +class CallbackExecutorException(FileAutomationException): + """Raised by ``CallbackExecutor`` for registration / dispatch failures.""" + + +class ExecuteActionException(FileAutomationException): + """Raised by ``ActionExecutor`` when an action list cannot be run.""" + + +class AddCommandException(FileAutomationException): + """Raised when a command registered into the executor is not callable.""" + + +class JsonActionException(FileAutomationException): + """Raised when JSON action files cannot be read or written.""" + + +class ArgparseException(FileAutomationException): + """Raised when the CLI receives no actionable argument.""" + + +class UrlValidationException(FileAutomationException): + """Raised when a URL fails scheme / host validation (SSRF guard).""" + + +class ValidationException(FileAutomationException): + """Raised when an action list fails pre-execution validation.""" + + +class RetryExhaustedException(FileAutomationException): + """Raised when a ``@retry_on_transient`` wrapped call runs out of attempts.""" + + +class QuotaExceededException(FileAutomationException): + """Raised when an action exceeds a configured size or duration quota.""" + + +class PathTraversalException(FileAutomationException): + """Raised when a user-supplied path escapes the allowed root.""" + + +class TCPAuthException(FileAutomationException): + """Raised when a TCP client fails shared-secret authentication.""" + + +_ARGPARSE_EMPTY_MESSAGE = "argparse received no actionable argument" +_BAD_TRIGGER_FUNCTION = "trigger name is not registered in the executor" +_BAD_CALLBACK_METHOD = "callback_param_method must be 'kwargs' or 'args'" +_ADD_COMMAND_NOT_CALLABLE = "command value must be a callable" +_ACTION_LIST_EMPTY = "action list is empty or wrong type" +_ACTION_LIST_MISSING_KEY = "action dict missing 'auto_control' key" +_CANT_FIND_JSON = "can't read JSON file" +_CANT_SAVE_JSON = "can't write JSON file" diff --git a/automation_file/local/dir/dir_process.py b/automation_file/local/dir/dir_process.py deleted file mode 100644 index dcd088c..0000000 --- a/automation_file/local/dir/dir_process.py +++ /dev/null @@ -1,113 +0,0 @@ -import shutil -from pathlib import Path - -# 匯入自訂例外與日誌工具 -# Import custom exception and logging utility -from automation_file.utils.exception.exceptions import DirNotExistsException -from automation_file.utils.logging.loggin_instance import file_automation_logger - - -def copy_dir(dir_path: str, target_dir_path: str) -> bool: - """ - 複製資料夾到目標路徑 - Copy directory to target path - :param dir_path: 要複製的資料夾路徑 (str) - Directory path to copy (str) - :param target_dir_path: 複製到的目標資料夾路徑 (str) - Target directory path (str) - :return: 成功回傳 True，失敗回傳 False - Return True if success, else False - """ - dir_path = Path(dir_path) # 轉換為 Path 物件 / Convert to Path object - target_dir_path = Path(target_dir_path) - if dir_path.is_dir(): # 確認來源是否為資料夾 / Check if source is a directory - try: - # 複製整個資料夾，若目標已存在則允許覆蓋 - # Copy entire directory, allow overwrite if target exists - shutil.copytree(dir_path, target_dir_path, dirs_exist_ok=True) - file_automation_logger.info(f"Copy dir {dir_path}") - return True - except shutil.Error as error: - # 複製失敗時記錄錯誤 - # Log error if copy fails - file_automation_logger.error(f"Copy dir {dir_path} failed: {repr(error)}") - else: - # 若來源資料夾不存在，記錄錯誤 - # Log error if source directory does not exist - file_automation_logger.error(f"Copy dir {dir_path} failed: {repr(DirNotExistsException)}") - return False - return False - - -def remove_dir_tree(dir_path: str) -> bool: - """ - 刪除整個資料夾樹 - Remove entire directory tree - :param dir_path: 要刪除的資料夾路徑 (str) - Directory path to remove (str) - :return: 成功回傳 True，失敗回傳 False - Return True if success, else False - """ - dir_path = Path(dir_path) - if dir_path.is_dir(): # 確認是否為資料夾 / Check if directory exists - try: - shutil.rmtree(dir_path) # 遞迴刪除資料夾 / Recursively delete directory - file_automation_logger.info(f"Remove dir tree {dir_path}") - return True - except shutil.Error as error: - file_automation_logger.error(f"Remove dir tree {dir_path} error: {repr(error)}") - return False - return False - - -def rename_dir(origin_dir_path, target_dir: str) -> bool: - """ - 重新命名資料夾 - Rename directory - :param origin_dir_path: 原始資料夾路徑 (str) - Original directory path (str) - :param target_dir: 新的完整路徑 (str) - Target directory path (str) - :return: 成功回傳 True，失敗回傳 False - Return True if success, else False - """ - origin_dir_path = Path(origin_dir_path) - if origin_dir_path.exists() and origin_dir_path.is_dir(): - try: - # 使用 Path.rename 重新命名資料夾 - # Rename directory using Path.rename - Path.rename(origin_dir_path, target_dir) - file_automation_logger.info( - f"Rename dir origin dir path: {origin_dir_path}, target dir path: {target_dir}") - return True - except Exception as error: - # 捕捉所有例外並記錄 - # Catch all exceptions and log - file_automation_logger.error( - f"Rename dir error: {repr(error)}, " - f"Rename dir origin dir path: {origin_dir_path}, " - f"target dir path: {target_dir}") - else: - # 若來源資料夾不存在，記錄錯誤 - # Log error if source directory does not exist - file_automation_logger.error( - f"Rename dir error: {repr(DirNotExistsException)}, " - f"Rename dir origin dir path: {origin_dir_path}, " - f"target dir path: {target_dir}") - return False - return False - - -def create_dir(dir_path: str) -> None: - """ - 建立資料夾 - Create directory - :param dir_path: 要建立的資料夾路徑 (str) - Directory path to create (str) - :return: None - """ - dir_path = Path(dir_path) - # 若資料夾已存在則不會報錯 - # Create directory, no error if already exists - dir_path.mkdir(exist_ok=True) - file_automation_logger.info(f"Create dir {dir_path}") \ No newline at end of file diff --git a/automation_file/local/dir_ops.py b/automation_file/local/dir_ops.py new file mode 100644 index 0000000..7648106 --- /dev/null +++ b/automation_file/local/dir_ops.py @@ -0,0 +1,59 @@ +"""Directory-level operations (Strategy module for the executor).""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +from automation_file.exceptions import DirNotExistsException +from automation_file.logging_config import file_automation_logger + + +def copy_dir(dir_path: str, target_dir_path: str) -> bool: + """Recursively copy a directory tree. Return True on success.""" + source = Path(dir_path) + if not source.is_dir(): + raise DirNotExistsException(str(source)) + try: + shutil.copytree(source, Path(target_dir_path), dirs_exist_ok=True) + file_automation_logger.info("copy_dir: %s -> %s", source, target_dir_path) + return True + except (OSError, shutil.Error) as error: + file_automation_logger.error("copy_dir failed: %r", error) + return False + + +def remove_dir_tree(dir_path: str) -> bool: + """Recursively delete a directory tree.""" + path = Path(dir_path) + if not path.is_dir(): + return False + try: + shutil.rmtree(path) + file_automation_logger.info("remove_dir_tree: %s", path) + return True + except (OSError, shutil.Error) as error: + file_automation_logger.error("remove_dir_tree failed: %r", error) + return False + + +def rename_dir(origin_dir_path: str, target_dir: str) -> bool: + """Rename (move) a directory.""" + source = Path(origin_dir_path) + if not source.is_dir(): + raise DirNotExistsException(str(source)) + try: + source.rename(target_dir) + file_automation_logger.info("rename_dir: %s -> %s", source, target_dir) + return True + except OSError as error: + file_automation_logger.error("rename_dir failed: %r", error) + return False + + +def create_dir(dir_path: str) -> bool: + """Create a directory (no error if it already exists).""" + path = Path(dir_path) + path.mkdir(parents=True, exist_ok=True) + file_automation_logger.info("create_dir: %s", path) + return True diff --git a/automation_file/local/file/file_process.py b/automation_file/local/file/file_process.py deleted file mode 100644 index 21c1257..0000000 --- a/automation_file/local/file/file_process.py +++ /dev/null @@ -1,174 +0,0 @@ -import shutil -import sys -from pathlib import Path - -# 匯入自訂例外與日誌工具 -# Import custom exceptions and logging utility -from automation_file.utils.exception.exceptions import FileNotExistsException, DirNotExistsException -from automation_file.utils.logging.loggin_instance import file_automation_logger - - -def copy_file(file_path: str, target_path: str, copy_metadata: bool = True) -> bool: - """ - 複製單一檔案 - Copy a single file - :param file_path: 要複製的檔案路徑 (str) - File path to copy (str) - :param target_path: 複製到的目標路徑 (str) - Target path (str) - :param copy_metadata: 是否複製檔案的中繼資料 (預設 True) - Whether to copy file metadata (default True) - :return: 成功回傳 True，失敗回傳 False - Return True if success, else False - """ - file_path = Path(file_path) - if file_path.is_file() and file_path.exists(): - try: - if copy_metadata: - shutil.copy2(file_path, target_path) # 複製檔案與中繼資料 / Copy file with metadata - else: - shutil.copy(file_path, target_path) # 只複製檔案內容 / Copy file only - file_automation_logger.info(f"Copy file origin path: {file_path}, target path : {target_path}") - return True - except shutil.Error as error: - file_automation_logger.error(f"Copy file failed: {repr(error)}") - else: - file_automation_logger.error(f"Copy file failed: {repr(FileNotExistsException)}") - return False - return False - - -def copy_specify_extension_file( - file_dir_path: str, target_extension: str, target_path: str, copy_metadata: bool = True) -> bool: - """ - 複製指定副檔名的檔案 - Copy files with a specific extension - :param file_dir_path: 要搜尋的資料夾路徑 (str) - Directory path to search (str) - :param target_extension: 要搜尋的副檔名 (str) - File extension to search (str) - :param target_path: 複製到的目標路徑 (str) - Target path (str) - :param copy_metadata: 是否複製檔案中繼資料 (bool) - Whether to copy metadata (bool) - :return: 成功回傳 True，失敗回傳 False - Return True if success, else False - """ - file_dir_path = Path(file_dir_path) - if file_dir_path.exists() and file_dir_path.is_dir(): - for file in file_dir_path.glob(f"**/*.{target_extension}"): # 遞迴搜尋指定副檔名 / Recursively search files - copy_file(str(file), target_path, copy_metadata=copy_metadata) - file_automation_logger.info( - f"Copy specify extension file on dir" - f"origin dir path: {file_dir_path}, target extension: {target_extension}, " - f"to target path {target_path}") - return True - else: - file_automation_logger.error( - f"Copy specify extension file failed: {repr(FileNotExistsException)}") - return False - - -def copy_all_file_to_dir(dir_path: str, target_dir_path: str) -> bool: - """ - 將整個資料夾移動到目標資料夾 - Move entire directory into target directory - :param dir_path: 要移動的資料夾路徑 (str) - Directory path to move (str) - :param target_dir_path: 目標資料夾路徑 (str) - Target directory path (str) - :return: 成功回傳 True，失敗回傳 False - Return True if success, else False - """ - dir_path = Path(dir_path) - target_dir_path = Path(target_dir_path) - if dir_path.is_dir() and target_dir_path.is_dir(): - try: - shutil.move(str(dir_path), str(target_dir_path)) # 移動資料夾 / Move directory - file_automation_logger.info( - f"Copy all file to dir, " - f"origin dir: {dir_path}, " - f"target dir: {target_dir_path}" - ) - return True - except shutil.Error as error: - file_automation_logger.error( - f"Copy all file to dir failed, " - f"origin dir: {dir_path}, " - f"target dir: {target_dir_path}, " - f"error: {repr(error)}" - ) - else: - print(repr(DirNotExistsException), file=sys.stderr) - return False - return False - - -def rename_file(origin_file_path, target_name: str, file_extension=None) -> bool: - """ - 重新命名資料夾內的檔案 - Rename files inside a directory - :param origin_file_path: 要搜尋檔案的資料夾路徑 (str) - Directory path to search (str) - :param target_name: 新的檔案名稱 (str) - New file name (str) - :param file_extension: 指定副檔名 (可選) (str) - File extension filter (optional) (str) - :return: 成功回傳 True，失敗回傳 False - Return True if success, else False - """ - origin_file_path = Path(origin_file_path) - if origin_file_path.exists() and origin_file_path.is_dir(): - if file_extension is None: - file_list = list(origin_file_path.glob("**/*")) # 全部檔案 / All files - else: - file_list = list(origin_file_path.glob(f"**/*.{file_extension}")) # 指定副檔名 / Specific extension - try: - file_index = 0 - for file in file_list: - file.rename(Path(origin_file_path, target_name)) # 重新命名檔案 / Rename file - file_index = file_index + 1 - file_automation_logger.info( - f"Renamed file: origin file path:{origin_file_path}, with new name: {target_name}") - return True - except Exception as error: - file_automation_logger.error( - f"Rename file failed, " - f"origin file path: {origin_file_path}, " - f"target name: {target_name}, " - f"file_extension: {file_extension}, " - f"error: {repr(error)}" - ) - else: - file_automation_logger.error( - f"Rename file failed, error: {repr(DirNotExistsException)}") - return False - return False - - -def remove_file(file_path: str) -> None: - """ - 刪除檔案 - Remove a file - :param file_path: 要刪除的檔案路徑 (str) - File path to remove (str) - :return: None - """ - file_path = Path(file_path) - if file_path.exists() and file_path.is_file(): - file_path.unlink(missing_ok=True) # 刪除檔案，若不存在則忽略 / Delete file, ignore if missing - file_automation_logger.info(f"Remove file, file path: {file_path}") - - -def create_file(file_path: str, content: str) -> None: - """ - 建立檔案並寫入內容 - Create a file and write content - :param file_path: 檔案路徑 (str) - File path (str) - :param content: 要寫入的內容 (str) - Content to write (str) - :return: None - """ - with open(file_path, "w+") as file: # "w+" 表示寫入模式，若不存在則建立 / "w+" means write mode, create if not exists - file.write(content) \ No newline at end of file diff --git a/automation_file/local/file_ops.py b/automation_file/local/file_ops.py new file mode 100644 index 0000000..9e3287f --- /dev/null +++ b/automation_file/local/file_ops.py @@ -0,0 +1,121 @@ +"""File-level operations (Strategy module for the executor).""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +from automation_file.exceptions import DirNotExistsException, FileNotExistsException +from automation_file.logging_config import file_automation_logger + + +def copy_file(file_path: str, target_path: str, copy_metadata: bool = True) -> bool: + """Copy a single file. Return True on success.""" + source = Path(file_path) + if not source.is_file(): + file_automation_logger.error("copy_file: source not found: %s", source) + raise FileNotExistsException(str(source)) + try: + if copy_metadata: + shutil.copy2(source, target_path) + else: + shutil.copy(source, target_path) + file_automation_logger.info("copy_file: %s -> %s", source, target_path) + return True + except (OSError, shutil.Error) as error: + file_automation_logger.error("copy_file failed: %r", error) + return False + + +def copy_specify_extension_file( + file_dir_path: str, + target_extension: str, + target_path: str, + copy_metadata: bool = True, +) -> bool: + """Copy every file under ``file_dir_path`` whose extension matches.""" + source_dir = Path(file_dir_path) + if not source_dir.is_dir(): + file_automation_logger.error("copy_specify_extension_file: dir not found: %s", source_dir) + raise DirNotExistsException(str(source_dir)) + extension = target_extension.lstrip(".") + copied = 0 + for file in source_dir.glob(f"**/*.{extension}"): + if copy_file(str(file), target_path, copy_metadata=copy_metadata): + copied += 1 + file_automation_logger.info( + "copy_specify_extension_file: copied %d *.%s from %s to %s", + copied, + extension, + source_dir, + target_path, + ) + return True + + +def copy_all_file_to_dir(dir_path: str, target_dir_path: str) -> bool: + """Move a directory into another directory.""" + source = Path(dir_path) + destination = Path(target_dir_path) + if not source.is_dir(): + raise DirNotExistsException(str(source)) + if not destination.is_dir(): + raise DirNotExistsException(str(destination)) + try: + shutil.move(str(source), str(destination)) + file_automation_logger.info("copy_all_file_to_dir: %s -> %s", source, destination) + return True + except (OSError, shutil.Error) as error: + file_automation_logger.error("copy_all_file_to_dir failed: %r", error) + return False + + +def rename_file( + origin_file_path: str, + target_name: str, + file_extension: str | None = None, +) -> bool: + """Rename every matching file under ``origin_file_path`` to ``target_name_{i}``. + + The original implementation renamed every match to the same name, which + silently overwrote previous renames. Each file now gets a unique suffix. + """ + source_dir = Path(origin_file_path) + if not source_dir.is_dir(): + raise DirNotExistsException(str(source_dir)) + + pattern = "**/*" if file_extension is None else f"**/*.{file_extension.lstrip('.')}" + matches = [p for p in source_dir.glob(pattern) if p.is_file()] + + try: + for index, file in enumerate(matches): + new_path = file.with_name(f"{target_name}_{index}{file.suffix}") + file.rename(new_path) + file_automation_logger.info("rename_file: %s -> %s", file, new_path) + return True + except OSError as error: + file_automation_logger.error( + "rename_file failed: source=%s target=%s ext=%s error=%r", + source_dir, + target_name, + file_extension, + error, + ) + return False + + +def remove_file(file_path: str) -> bool: + """Delete a file if it exists. Return True when a file was removed.""" + path = Path(file_path) + if not path.is_file(): + return False + path.unlink(missing_ok=True) + file_automation_logger.info("remove_file: %s", path) + return True + + +def create_file(file_path: str, content: str = "", encoding: str = "utf-8") -> None: + """Create a file with the given text content (overwrites existing file).""" + with open(file_path, "w", encoding=encoding) as file: + file.write(content) + file_automation_logger.info("create_file: %s (%d bytes)", file_path, len(content)) diff --git a/automation_file/local/safe_paths.py b/automation_file/local/safe_paths.py new file mode 100644 index 0000000..f4c92dc --- /dev/null +++ b/automation_file/local/safe_paths.py @@ -0,0 +1,45 @@ +"""Path-traversal guard for local file operations. + +``safe_join(root, user_path)`` returns the resolved absolute path only if it +lies under ``root`` once symlinks are followed; otherwise it raises +:class:`PathTraversalException`. The helper is intentionally independent of +any configuration module so callers can wrap individual operations instead of +opting in globally. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +from automation_file.exceptions import PathTraversalException + + +def safe_join(root: str | os.PathLike[str], user_path: str | os.PathLike[str]) -> Path: + """Resolve ``user_path`` under ``root`` and guarantee containment. + + Raises :class:`PathTraversalException` if the resolved target would escape + ``root`` through ``..`` components, an absolute path, or a symlink. + """ + root_resolved = Path(root).resolve() + candidate = Path(user_path) + if candidate.is_absolute(): + resolved = candidate.resolve() + else: + resolved = (root_resolved / candidate).resolve() + try: + resolved.relative_to(root_resolved) + except ValueError as error: + raise PathTraversalException( + f"path {user_path!r} escapes root {str(root_resolved)!r}" + ) from error + return resolved + + +def is_within(root: str | os.PathLike[str], user_path: str | os.PathLike[str]) -> bool: + """Return True if ``user_path`` resolves inside ``root``. Never raises.""" + try: + safe_join(root, user_path) + except PathTraversalException: + return False + return True diff --git a/automation_file/local/zip/zip_process.py b/automation_file/local/zip/zip_process.py deleted file mode 100644 index 0510ea1..0000000 --- a/automation_file/local/zip/zip_process.py +++ /dev/null @@ -1,163 +0,0 @@ -import zipfile -from pathlib import Path -from shutil import make_archive -from typing import List, Dict, Union -from zipfile import ZipInfo - -# 匯入自訂例外與日誌工具 -# Import custom exception and logging utility -from automation_file.utils.exception.exceptions import ZIPGetWrongFileException -from automation_file.utils.logging.loggin_instance import file_automation_logger - - -def zip_dir(dir_we_want_to_zip: str, zip_name: str) -> None: - """ - 壓縮整個資料夾成 zip 檔 - Zip an entire directory - :param dir_we_want_to_zip: 要壓縮的資料夾路徑 (str) - Directory path to zip (str) - :param zip_name: 壓縮檔名稱 (str) - Zip file name (str) - :return: None - """ - make_archive(root_dir=dir_we_want_to_zip, base_name=zip_name, format="zip") - file_automation_logger.info(f"Dir to zip: {dir_we_want_to_zip}, zip file name: {zip_name}") - - -def zip_file(zip_file_path: str, file: Union[str, List[str]]) -> None: - """ - 將單一檔案或多個檔案加入 zip - Add single or multiple files into a zip - :param zip_file_path: zip 檔路徑 (str) - Zip file path (str) - :param file: 檔案路徑或檔案路徑清單 (str 或 List[str]) - File path or list of file paths - :return: None - """ - current_zip = zipfile.ZipFile(zip_file_path, mode="w") - if isinstance(file, str): - file_name = Path(file) - current_zip.write(file, file_name.name) # 寫入單一檔案 / Write single file - file_automation_logger.info(f"Write file: {file_name} to zip: {current_zip}") - else: - if isinstance(file, list): - for writeable in file: - file_name = Path(writeable) - current_zip.write(writeable, file_name.name) # 寫入多個檔案 / Write multiple files - file_automation_logger.info(f"Write file: {writeable} to zip: {current_zip}") - else: - file_automation_logger.error(repr(ZIPGetWrongFileException)) - current_zip.close() - - -def read_zip_file(zip_file_path: str, file_name: str, password: Union[str, None] = None) -> bytes: - """ - 讀取 zip 檔中的指定檔案 - Read a specific file inside a zip - :param zip_file_path: zip 檔路徑 (str) - Zip file path (str) - :param file_name: zip 中的檔案名稱 (str) - File name inside zip (str) - :param password: 若 zip 有密碼，需提供 (str 或 None) - Password if zip is protected - :return: 檔案內容 (bytes) - File content (bytes) - """ - current_zip = zipfile.ZipFile(zip_file_path, mode="r") - with current_zip.open(name=file_name, mode="r", pwd=password, force_zip64=True) as read_file: - data = read_file.read() - current_zip.close() - file_automation_logger.info(f"Read zip file: {zip_file_path}") - return data - - -def unzip_file(zip_file_path: str, extract_member, extract_path: Union[str, None] = None, - password: Union[str, None] = None) -> None: - """ - 解壓縮 zip 中的單一檔案 - Extract a single file from a zip - :param zip_file_path: zip 檔路徑 (str) - Zip file path (str) - :param extract_member: 要解壓縮的檔案名稱 (str) - File name to extract - :param extract_path: 解壓縮到的路徑 (str 或 None) - Path to extract to - :param password: 若 zip 有密碼，需提供 (str 或 None) - Password if zip is protected - :return: None - """ - current_zip = zipfile.ZipFile(zip_file_path, mode="r") - current_zip.extract(member=extract_member, path=extract_path, pwd=password) - file_automation_logger.info( - f"Unzip file: {zip_file_path}, extract member: {extract_member}, extract path: {extract_path}, password: {password}" - ) - current_zip.close() - - -def unzip_all(zip_file_path: str, extract_member: Union[str, None] = None, - extract_path: Union[str, None] = None, password: Union[str, None] = None) -> None: - """ - 解壓縮 zip 中的所有檔案 - Extract all files from a zip - :param zip_file_path: zip 檔路徑 (str) - Zip file path (str) - :param extract_member: 指定要解壓縮的檔案 (可選) (str 或 None) - Specific members to extract (optional) - :param extract_path: 解壓縮到的路徑 (str 或 None) - Path to extract to - :param password: 若 zip 有密碼，需提供 (str 或 None) - Password if zip is protected - :return: None - """ - current_zip = zipfile.ZipFile(zip_file_path, mode="r") - current_zip.extractall(members=extract_member, path=extract_path, pwd=password) - file_automation_logger.info( - f"Unzip file: {zip_file_path}, extract member: {extract_member}, extract path: {extract_path}, password: {password}" - ) - current_zip.close() - - -def zip_info(zip_file_path: str) -> List[ZipInfo]: - """ - 取得 zip 檔案的詳細資訊 (ZipInfo 物件) - Get detailed info of a zip file (ZipInfo objects) - :param zip_file_path: zip 檔路徑 (str) - Zip file path (str) - :return: List[ZipInfo] - """ - current_zip = zipfile.ZipFile(zip_file_path, mode="r") - info_list = current_zip.infolist() # 回傳 ZipInfo 物件清單 / Return list of ZipInfo objects - current_zip.close() - file_automation_logger.info(f"Show zip info: {zip_file_path}") - return info_list - - -def zip_file_info(zip_file_path: str) -> List[str]: - """ - 取得 zip 檔案內所有檔案名稱 - Get list of file names inside a zip - :param zip_file_path: zip 檔路徑 (str) - Zip file path (str) - :return: List[str] - """ - current_zip = zipfile.ZipFile(zip_file_path, mode="r") - name_list = current_zip.namelist() # 回傳檔案名稱清單 / Return list of file names - current_zip.close() - file_automation_logger.info(f"Show zip file info: {zip_file_path}") - return name_list - - -def set_zip_password(zip_file_path: str, password: bytes) -> None: - """ - 設定 zip 檔案的密碼 (注意：標準 zipfile 僅支援讀取密碼，不支援加密寫入) - Set password for a zip file (Note: standard zipfile only supports reading with password, not writing encrypted zips) - :param zip_file_path: zip 檔路徑 (str) - Zip file path (str) - :param password: 密碼 (bytes) - Password (bytes) - :return: None - """ - current_zip = zipfile.ZipFile(zip_file_path) - current_zip.setpassword(pwd=password) # 設定解壓縮時的密碼 / Set password for extraction - current_zip.close() - file_automation_logger.info(f"Set zip file password, zip file: {zip_file_path}, zip password: {password}") \ No newline at end of file diff --git a/automation_file/local/zip_ops.py b/automation_file/local/zip_ops.py new file mode 100644 index 0000000..782ccc5 --- /dev/null +++ b/automation_file/local/zip_ops.py @@ -0,0 +1,100 @@ +"""Zip archive operations. + +Note: Python's standard ``zipfile`` module does not write encrypted archives. +``set_zip_password`` only sets the read-side password used to extract an +already-encrypted archive. +""" + +from __future__ import annotations + +import zipfile +from pathlib import Path +from shutil import make_archive +from zipfile import ZipInfo + +from automation_file.exceptions import ZipInputException +from automation_file.logging_config import file_automation_logger + + +def zip_dir(dir_we_want_to_zip: str, zip_name: str) -> None: + """Create ``zip_name.zip`` from the contents of ``dir_we_want_to_zip``.""" + make_archive(root_dir=dir_we_want_to_zip, base_name=zip_name, format="zip") + file_automation_logger.info("zip_dir: %s -> %s.zip", dir_we_want_to_zip, zip_name) + + +def zip_file(zip_file_path: str, file: str | list[str]) -> None: + """Write one or many files into ``zip_file_path``.""" + if isinstance(file, str): + paths = [file] + elif isinstance(file, list): + paths = file + else: + raise ZipInputException(f"unsupported type: {type(file).__name__}") + with zipfile.ZipFile(zip_file_path, mode="w") as archive: + for path in paths: + name = Path(path).name + archive.write(path, name) + file_automation_logger.info("zip_file: %s -> %s", path, zip_file_path) + + +def read_zip_file(zip_file_path: str, file_name: str, password: bytes | None = None) -> bytes: + """Return the raw bytes of ``file_name`` inside the zip.""" + with ( + zipfile.ZipFile(zip_file_path, mode="r") as archive, + archive.open(name=file_name, mode="r", pwd=password, force_zip64=True) as member, + ): + data = member.read() + file_automation_logger.info("read_zip_file: %s/%s", zip_file_path, file_name) + return data + + +def unzip_file( + zip_file_path: str, + extract_member: str, + extract_path: str | None = None, + password: bytes | None = None, +) -> None: + """Extract a single member to ``extract_path``.""" + with zipfile.ZipFile(zip_file_path, mode="r") as archive: + archive.extract(member=extract_member, path=extract_path, pwd=password) + file_automation_logger.info( + "unzip_file: %s member=%s to=%s", + zip_file_path, + extract_member, + extract_path, + ) + + +def unzip_all( + zip_file_path: str, + extract_member: list[str] | None = None, + extract_path: str | None = None, + password: bytes | None = None, +) -> None: + """Extract every member (or a subset) to ``extract_path``.""" + with zipfile.ZipFile(zip_file_path, mode="r") as archive: + archive.extractall(members=extract_member, path=extract_path, pwd=password) + file_automation_logger.info("unzip_all: %s to=%s", zip_file_path, extract_path) + + +def zip_info(zip_file_path: str) -> list[ZipInfo]: + """Return the ``ZipInfo`` list for every member in the archive.""" + with zipfile.ZipFile(zip_file_path, mode="r") as archive: + info_list = archive.infolist() + file_automation_logger.info("zip_info: %s", zip_file_path) + return info_list + + +def zip_file_info(zip_file_path: str) -> list[str]: + """Return the member names inside the archive.""" + with zipfile.ZipFile(zip_file_path, mode="r") as archive: + name_list = archive.namelist() + file_automation_logger.info("zip_file_info: %s", zip_file_path) + return name_list + + +def set_zip_password(zip_file_path: str, password: bytes) -> None: + """Set the read-side password on an encrypted archive.""" + with zipfile.ZipFile(zip_file_path, mode="r") as archive: + archive.setpassword(pwd=password) + file_automation_logger.info("set_zip_password: %s", zip_file_path) diff --git a/automation_file/logging_config.py b/automation_file/logging_config.py new file mode 100644 index 0000000..2b30e0a --- /dev/null +++ b/automation_file/logging_config.py @@ -0,0 +1,52 @@ +"""Module-level logger for automation_file. + +A single :data:`file_automation_logger` is exposed. It writes to +``FileAutomation.log`` in append mode and mirrors every record to stderr via a +custom handler. The handler list is rebuilt only once, even if the module is +reloaded, so tests can import this safely. +""" + +from __future__ import annotations + +import logging +import sys + +_LOG_FORMAT = "%(asctime)s | %(name)s | %(levelname)s | %(message)s" +_LOG_FILENAME = "FileAutomation.log" +_LOGGER_NAME = "automation_file" + + +class _StderrHandler(logging.Handler): + """Mirror log records to stderr so scripts see progress without enabling root.""" + + def emit(self, record: logging.LogRecord) -> None: + try: + print(self.format(record), file=sys.stderr) + except (OSError, ValueError): + self.handleError(record) + + +def _build_logger() -> logging.Logger: + logger = logging.getLogger(_LOGGER_NAME) + if getattr(logger, "_file_automation_initialised", False): + return logger + logger.setLevel(logging.DEBUG) + logger.propagate = False + + formatter = logging.Formatter(_LOG_FORMAT) + + file_handler = logging.FileHandler(filename=_LOG_FILENAME, mode="a", encoding="utf-8") + file_handler.setFormatter(formatter) + file_handler.setLevel(logging.DEBUG) + logger.addHandler(file_handler) + + stream_handler = _StderrHandler() + stream_handler.setFormatter(formatter) + stream_handler.setLevel(logging.INFO) + logger.addHandler(stream_handler) + + logger._file_automation_initialised = True # type: ignore[attr-defined] # pylint: disable=protected-access # stamp our own init marker on the shared logger + return logger + + +file_automation_logger: logging.Logger = _build_logger() diff --git a/automation_file/local/file/__init__.py b/automation_file/project/__init__.py similarity index 100% rename from automation_file/local/file/__init__.py rename to automation_file/project/__init__.py diff --git a/automation_file/project/project_builder.py b/automation_file/project/project_builder.py new file mode 100644 index 0000000..04ac98d --- /dev/null +++ b/automation_file/project/project_builder.py @@ -0,0 +1,66 @@ +"""Project skeleton builder (Builder pattern).""" + +from __future__ import annotations + +from os import getcwd +from pathlib import Path + +from automation_file.core.json_store import write_action_json +from automation_file.logging_config import file_automation_logger +from automation_file.project.templates import ( + EXECUTOR_FOLDER_TEMPLATE, + EXECUTOR_ONE_FILE_TEMPLATE, + KEYWORD_CREATE_TEMPLATE, + KEYWORD_TEARDOWN_TEMPLATE, +) + +_KEYWORD_DIR = "keyword" +_EXECUTOR_DIR = "executor" + + +class ProjectBuilder: + """Create a ``keyword/`` + ``executor/`` skeleton under ``project_root``.""" + + def __init__( + self, project_root: str | None = None, parent_name: str = "FileAutomation" + ) -> None: + self.project_root: Path = Path(project_root or getcwd()) + self.parent: Path = self.project_root / parent_name + self.keyword_dir: Path = self.parent / _KEYWORD_DIR + self.executor_dir: Path = self.parent / _EXECUTOR_DIR + + def build(self) -> None: + self.keyword_dir.mkdir(parents=True, exist_ok=True) + self.executor_dir.mkdir(parents=True, exist_ok=True) + self._write_keyword_files() + self._write_executor_files() + file_automation_logger.info("ProjectBuilder: built %s", self.parent) + + def _write_keyword_files(self) -> None: + write_action_json( + str(self.keyword_dir / "keyword_create.json"), + KEYWORD_CREATE_TEMPLATE, + ) + write_action_json( + str(self.keyword_dir / "keyword_teardown.json"), + KEYWORD_TEARDOWN_TEMPLATE, + ) + + def _write_executor_files(self) -> None: + (self.executor_dir / "executor_one_file.py").write_text( + EXECUTOR_ONE_FILE_TEMPLATE.format( + keyword_json=str(self.keyword_dir / "keyword_create.json") + ), + encoding="utf-8", + ) + (self.executor_dir / "executor_folder.py").write_text( + EXECUTOR_FOLDER_TEMPLATE.format(keyword_dir=str(self.keyword_dir)), + encoding="utf-8", + ) + + +def create_project_dir( + project_path: str | None = None, parent_name: str = "FileAutomation" +) -> None: + """Create a project skeleton (module-level shim).""" + ProjectBuilder(project_root=project_path, parent_name=parent_name).build() diff --git a/automation_file/project/templates.py b/automation_file/project/templates.py new file mode 100644 index 0000000..b92e317 --- /dev/null +++ b/automation_file/project/templates.py @@ -0,0 +1,33 @@ +"""Project scaffolding templates (keyword JSON + Python entry points).""" + +from __future__ import annotations + +EXECUTOR_ONE_FILE_TEMPLATE: str = """\ +from automation_file import execute_action, read_action_json + +execute_action( + read_action_json( + r"{keyword_json}" + ) +) +""" + +EXECUTOR_FOLDER_TEMPLATE: str = """\ +from automation_file import execute_files, get_dir_files_as_list + +execute_files( + get_dir_files_as_list( + r"{keyword_dir}" + ) +) +""" + +KEYWORD_CREATE_TEMPLATE: list = [ + ["FA_create_dir", {"dir_path": "test_dir"}], + ["FA_create_file", {"file_path": "test.txt", "content": "test"}], +] + +KEYWORD_TEARDOWN_TEMPLATE: list = [ + ["FA_remove_file", {"file_path": "test.txt"}], + ["FA_remove_dir_tree", {"dir_path": "test_dir"}], +] diff --git a/automation_file/remote/_upload_tree.py b/automation_file/remote/_upload_tree.py new file mode 100644 index 0000000..af9d841 --- /dev/null +++ b/automation_file/remote/_upload_tree.py @@ -0,0 +1,63 @@ +"""Shared directory-walker for cloud/SFTP ``*_upload_dir`` operations. + +Every backend implements the same pattern: validate that ``dir_path`` +exists, normalise the remote prefix, iterate ``Path.rglob('*')``, skip +non-files, compute a POSIX-relative remote identifier, call +``upload_file`` for each, and collect the successful remote keys. This +module factors that walk out so each backend only supplies the two +parts that actually differ — how to assemble the remote identifier and +which per-file upload function to call. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import NamedTuple + +from automation_file.exceptions import DirNotExistsException + + +class UploadDirResult(NamedTuple): + """Return value of :func:`walk_and_upload`. + + Carries the resolved ``source`` ``Path``, the normalised prefix + (trailing ``/`` stripped), and the list of remote identifiers that + uploaded successfully — so each backend can feed its own log line + without re-doing the Path / prefix work. + """ + + source: Path + prefix: str + uploaded: list[str] + + +def walk_and_upload( + dir_path: str, + prefix: str, + make_remote: Callable[[str, str], str], + upload_one: Callable[[Path, str], bool], +) -> UploadDirResult: + """Walk ``dir_path`` and upload every file via ``upload_one``. + + Raises :class:`DirNotExistsException` if ``dir_path`` is not a + directory. ``prefix`` is ``rstrip("/")``-ed before being passed to + ``make_remote(normalised_prefix, rel_posix)``, and ``upload_one`` + receives ``(local_path, remote_key)`` returning True on success. + Per-file failures are not raised — they are simply omitted from + :attr:`UploadDirResult.uploaded`, matching the existing backend + contract. + """ + source = Path(dir_path) + if not source.is_dir(): + raise DirNotExistsException(str(source)) + normalised = prefix.rstrip("/") + uploaded: list[str] = [] + for entry in source.rglob("*"): + if not entry.is_file(): + continue + rel = entry.relative_to(source).as_posix() + remote = make_remote(normalised, rel) + if upload_one(entry, remote): + uploaded.append(remote) + return UploadDirResult(source=source, prefix=normalised, uploaded=uploaded) diff --git a/automation_file/remote/azure_blob/__init__.py b/automation_file/remote/azure_blob/__init__.py new file mode 100644 index 0000000..189fe71 --- /dev/null +++ b/automation_file/remote/azure_blob/__init__.py @@ -0,0 +1,28 @@ +"""Azure Blob Storage strategy module. + +Actions (``FA_azure_blob_*``) are registered on the shared default registry +automatically. +""" + +from __future__ import annotations + +from automation_file.core.action_registry import ActionRegistry +from automation_file.remote.azure_blob import delete_ops, download_ops, list_ops, upload_ops +from automation_file.remote.azure_blob.client import AzureBlobClient, azure_blob_instance + + +def register_azure_blob_ops(registry: ActionRegistry) -> None: + """Register every ``FA_azure_blob_*`` command into ``registry``.""" + registry.register_many( + { + "FA_azure_blob_later_init": azure_blob_instance.later_init, + "FA_azure_blob_upload_file": upload_ops.azure_blob_upload_file, + "FA_azure_blob_upload_dir": upload_ops.azure_blob_upload_dir, + "FA_azure_blob_download_file": download_ops.azure_blob_download_file, + "FA_azure_blob_delete_blob": delete_ops.azure_blob_delete_blob, + "FA_azure_blob_list_container": list_ops.azure_blob_list_container, + } + ) + + +__all__ = ["AzureBlobClient", "azure_blob_instance", "register_azure_blob_ops"] diff --git a/automation_file/remote/azure_blob/client.py b/automation_file/remote/azure_blob/client.py new file mode 100644 index 0000000..3cab75a --- /dev/null +++ b/automation_file/remote/azure_blob/client.py @@ -0,0 +1,50 @@ +"""Azure Blob Storage client (Singleton Facade).""" + +from __future__ import annotations + +from typing import Any + +from automation_file.logging_config import file_automation_logger + + +def _import_blob_service_client() -> Any: + try: + from azure.storage.blob import BlobServiceClient + except ImportError as error: + raise RuntimeError( + "azure-storage-blob import failed — reinstall `automation_file` to restore" + " the Azure Blob backend" + ) from error + return BlobServiceClient + + +class AzureBlobClient: + """Lazy wrapper around :class:`azure.storage.blob.BlobServiceClient`.""" + + def __init__(self) -> None: + self.service: Any = None + + def later_init( + self, + connection_string: str | None = None, + account_url: str | None = None, + credential: Any = None, + ) -> Any: + """Build a BlobServiceClient. Prefer ``connection_string`` when set.""" + service_cls = _import_blob_service_client() + if connection_string: + self.service = service_cls.from_connection_string(connection_string) + elif account_url: + self.service = service_cls(account_url=account_url, credential=credential) + else: + raise ValueError("provide connection_string or account_url") + file_automation_logger.info("AzureBlobClient: service ready") + return self.service + + def require_service(self) -> Any: + if self.service is None: + raise RuntimeError("AzureBlobClient not initialised; call later_init() first") + return self.service + + +azure_blob_instance: AzureBlobClient = AzureBlobClient() diff --git a/automation_file/remote/azure_blob/delete_ops.py b/automation_file/remote/azure_blob/delete_ops.py new file mode 100644 index 0000000..46d145f --- /dev/null +++ b/automation_file/remote/azure_blob/delete_ops.py @@ -0,0 +1,22 @@ +"""Azure Blob delete operations.""" + +from __future__ import annotations + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.azure_blob.client import azure_blob_instance + + +def azure_blob_delete_blob(container: str, blob_name: str) -> bool: + """Delete a blob. Returns True on success.""" + service = azure_blob_instance.require_service() + try: + service.get_blob_client(container=container, blob=blob_name).delete_blob() + file_automation_logger.info( + "azure_blob_delete_blob: %s/%s", + container, + blob_name, + ) + return True + except Exception as error: # pylint: disable=broad-except + file_automation_logger.error("azure_blob_delete_blob failed: %r", error) + return False diff --git a/automation_file/remote/azure_blob/download_ops.py b/automation_file/remote/azure_blob/download_ops.py new file mode 100644 index 0000000..204f577 --- /dev/null +++ b/automation_file/remote/azure_blob/download_ops.py @@ -0,0 +1,28 @@ +"""Azure Blob download operations.""" + +from __future__ import annotations + +from pathlib import Path + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.azure_blob.client import azure_blob_instance + + +def azure_blob_download_file(container: str, blob_name: str, target_path: str) -> bool: + """Download a blob to ``target_path``.""" + service = azure_blob_instance.require_service() + Path(target_path).parent.mkdir(parents=True, exist_ok=True) + try: + blob = service.get_blob_client(container=container, blob=blob_name) + with open(target_path, "wb") as fp: + fp.write(blob.download_blob().readall()) + file_automation_logger.info( + "azure_blob_download_file: %s/%s -> %s", + container, + blob_name, + target_path, + ) + return True + except Exception as error: # pylint: disable=broad-except + file_automation_logger.error("azure_blob_download_file failed: %r", error) + return False diff --git a/automation_file/remote/azure_blob/list_ops.py b/automation_file/remote/azure_blob/list_ops.py new file mode 100644 index 0000000..bef63b1 --- /dev/null +++ b/automation_file/remote/azure_blob/list_ops.py @@ -0,0 +1,26 @@ +"""Azure Blob listing operations.""" + +from __future__ import annotations + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.azure_blob.client import azure_blob_instance + + +def azure_blob_list_container(container: str, name_prefix: str = "") -> list[str]: + """Return every blob name under ``container``/``name_prefix``.""" + service = azure_blob_instance.require_service() + names: list[str] = [] + try: + container_client = service.get_container_client(container) + iterator = container_client.list_blobs(name_starts_with=name_prefix or None) + names = [blob.name for blob in iterator] + except Exception as error: # pylint: disable=broad-except + file_automation_logger.error("azure_blob_list_container failed: %r", error) + return [] + file_automation_logger.info( + "azure_blob_list_container: %s/%s (%d blobs)", + container, + name_prefix, + len(names), + ) + return names diff --git a/automation_file/remote/azure_blob/upload_ops.py b/automation_file/remote/azure_blob/upload_ops.py new file mode 100644 index 0000000..5f80eec --- /dev/null +++ b/automation_file/remote/azure_blob/upload_ops.py @@ -0,0 +1,59 @@ +"""Azure Blob upload operations.""" + +from __future__ import annotations + +from pathlib import Path + +from automation_file.exceptions import FileNotExistsException +from automation_file.logging_config import file_automation_logger +from automation_file.remote._upload_tree import walk_and_upload +from automation_file.remote.azure_blob.client import azure_blob_instance + + +def azure_blob_upload_file( + file_path: str, + container: str, + blob_name: str, + overwrite: bool = True, +) -> bool: + """Upload a single file to ``container/blob_name``.""" + path = Path(file_path) + if not path.is_file(): + raise FileNotExistsException(str(path)) + service = azure_blob_instance.require_service() + try: + blob = service.get_blob_client(container=container, blob=blob_name) + with open(path, "rb") as fp: + blob.upload_blob(fp, overwrite=overwrite) + file_automation_logger.info( + "azure_blob_upload_file: %s -> %s/%s", + path, + container, + blob_name, + ) + return True + except Exception as error: # pylint: disable=broad-except + file_automation_logger.error("azure_blob_upload_file failed: %r", error) + return False + + +def azure_blob_upload_dir( + dir_path: str, + container: str, + name_prefix: str = "", +) -> list[str]: + """Upload every file under ``dir_path`` to ``container`` under ``name_prefix``.""" + result = walk_and_upload( + dir_path, + name_prefix, + lambda prefix, rel: f"{prefix}/{rel}" if prefix else rel, + lambda local, blob_name: azure_blob_upload_file(str(local), container, blob_name), + ) + file_automation_logger.info( + "azure_blob_upload_dir: %s -> %s/%s (%d files)", + result.source, + container, + result.prefix, + len(result.uploaded), + ) + return result.uploaded diff --git a/automation_file/remote/download/file.py b/automation_file/remote/download/file.py deleted file mode 100644 index c9d1239..0000000 --- a/automation_file/remote/download/file.py +++ /dev/null @@ -1,61 +0,0 @@ -import requests -from tqdm import tqdm - -# 匯入自訂的日誌工具 -# Import custom logging utility -from automation_file.utils.logging.loggin_instance import file_automation_logger - - -def download_file(file_url: str, file_name: str, chunk_size: int = 1024, timeout: int = 10): - """ - 下載檔案並顯示進度條 - Download a file with progress bar - :param file_url: 檔案下載網址 (str) - File download URL (str) - :param file_name: 儲存檔案名稱 (str) - File name to save as (str) - :param chunk_size: 每次下載的資料塊大小，預設 1024 bytes - Size of each download chunk, default 1024 bytes - :param timeout: 請求逾時時間 (秒)，預設 10 - Request timeout in seconds, default 10 - :return: None - """ - try: - # 發送 HTTP GET 請求，使用串流模式避免一次載入大檔案 - # Send HTTP GET request with streaming to avoid loading large file at once - response = requests.get(file_url, stream=True, timeout=timeout) - response.raise_for_status() # 若狀態碼非 200，則拋出例外 / Raise exception if status code is not 200 - - # 從回應標頭取得檔案大小 (若伺服器有提供) - # Get total file size from response headers (if available) - total_size = int(response.headers.get('content-length', 0)) - - # 以二進位寫入模式開啟檔案 - # Open file in binary write mode - with open(file_name, 'wb') as file: - if total_size > 0: - # 使用 tqdm 顯示下載進度條 - # Use tqdm to show download progress bar - with tqdm(total=total_size, unit='B', unit_scale=True, desc=file_name) as progress: - for chunk in response.iter_content(chunk_size=chunk_size): - if chunk: # 避免空資料塊 / Avoid empty chunks - file.write(chunk) - progress.update(len(chunk)) # 更新進度條 / Update progress bar - else: - # 若無法取得檔案大小，仍逐塊下載 - # If file size is unknown, still download in chunks - for chunk in response.iter_content(chunk_size=chunk_size): - if chunk: - file.write(chunk) - - file_automation_logger.info(f"File download is complete. Saved as: {file_name}") - - # 錯誤處理區塊 / Error handling - except requests.exceptions.HTTPError as http_err: - file_automation_logger.error(f"HTTP error：{http_err}") - except requests.exceptions.ConnectionError: - file_automation_logger.error("Connection error. Please check your internet connection.") - except requests.exceptions.Timeout: - file_automation_logger.error("Request timed out. The server did not respond.") - except Exception as err: - file_automation_logger.error(f"Error：{err}") \ No newline at end of file diff --git a/automation_file/remote/dropbox_api/__init__.py b/automation_file/remote/dropbox_api/__init__.py new file mode 100644 index 0000000..7cdb8b5 --- /dev/null +++ b/automation_file/remote/dropbox_api/__init__.py @@ -0,0 +1,29 @@ +"""Dropbox strategy module. + +Named ``dropbox_api`` to avoid shadowing the ``dropbox`` PyPI package inside +``automation_file.remote``. Actions (``FA_dropbox_*``) are registered on the +shared default registry automatically. +""" + +from __future__ import annotations + +from automation_file.core.action_registry import ActionRegistry +from automation_file.remote.dropbox_api import delete_ops, download_ops, list_ops, upload_ops +from automation_file.remote.dropbox_api.client import DropboxClient, dropbox_instance + + +def register_dropbox_ops(registry: ActionRegistry) -> None: + """Register every ``FA_dropbox_*`` command into ``registry``.""" + registry.register_many( + { + "FA_dropbox_later_init": dropbox_instance.later_init, + "FA_dropbox_upload_file": upload_ops.dropbox_upload_file, + "FA_dropbox_upload_dir": upload_ops.dropbox_upload_dir, + "FA_dropbox_download_file": download_ops.dropbox_download_file, + "FA_dropbox_delete_path": delete_ops.dropbox_delete_path, + "FA_dropbox_list_folder": list_ops.dropbox_list_folder, + } + ) + + +__all__ = ["DropboxClient", "dropbox_instance", "register_dropbox_ops"] diff --git a/automation_file/remote/dropbox_api/client.py b/automation_file/remote/dropbox_api/client.py new file mode 100644 index 0000000..13a1115 --- /dev/null +++ b/automation_file/remote/dropbox_api/client.py @@ -0,0 +1,39 @@ +"""Dropbox client (Singleton Facade).""" + +from __future__ import annotations + +from typing import Any + +from automation_file.logging_config import file_automation_logger + + +def _import_dropbox() -> Any: + try: + import dropbox + except ImportError as error: + raise RuntimeError( + "dropbox import failed — reinstall `automation_file` to restore the Dropbox backend" + ) from error + return dropbox + + +class DropboxClient: + """Lazy wrapper around :class:`dropbox.Dropbox`.""" + + def __init__(self) -> None: + self.client: Any = None + + def later_init(self, oauth2_access_token: str) -> Any: + """Build a Dropbox client from a user-supplied OAuth2 access token.""" + dropbox = _import_dropbox() + self.client = dropbox.Dropbox(oauth2_access_token) + file_automation_logger.info("DropboxClient: client ready") + return self.client + + def require_client(self) -> Any: + if self.client is None: + raise RuntimeError("DropboxClient not initialised; call later_init() first") + return self.client + + +dropbox_instance: DropboxClient = DropboxClient() diff --git a/automation_file/remote/dropbox_api/delete_ops.py b/automation_file/remote/dropbox_api/delete_ops.py new file mode 100644 index 0000000..2c361e8 --- /dev/null +++ b/automation_file/remote/dropbox_api/delete_ops.py @@ -0,0 +1,18 @@ +"""Dropbox delete operations.""" + +from __future__ import annotations + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.dropbox_api.client import dropbox_instance + + +def dropbox_delete_path(remote_path: str) -> bool: + """Delete a file or folder at ``remote_path``.""" + client = dropbox_instance.require_client() + try: + client.files_delete_v2(remote_path) + file_automation_logger.info("dropbox_delete_path: %s", remote_path) + return True + except Exception as error: # pylint: disable=broad-except + file_automation_logger.error("dropbox_delete_path failed: %r", error) + return False diff --git a/automation_file/remote/dropbox_api/download_ops.py b/automation_file/remote/dropbox_api/download_ops.py new file mode 100644 index 0000000..3564550 --- /dev/null +++ b/automation_file/remote/dropbox_api/download_ops.py @@ -0,0 +1,25 @@ +"""Dropbox download operations.""" + +from __future__ import annotations + +from pathlib import Path + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.dropbox_api.client import dropbox_instance + + +def dropbox_download_file(remote_path: str, target_path: str) -> bool: + """Download ``remote_path`` to ``target_path``.""" + client = dropbox_instance.require_client() + Path(target_path).parent.mkdir(parents=True, exist_ok=True) + try: + client.files_download_to_file(target_path, remote_path) + file_automation_logger.info( + "dropbox_download_file: %s -> %s", + remote_path, + target_path, + ) + return True + except Exception as error: # pylint: disable=broad-except + file_automation_logger.error("dropbox_download_file failed: %r", error) + return False diff --git a/automation_file/remote/dropbox_api/list_ops.py b/automation_file/remote/dropbox_api/list_ops.py new file mode 100644 index 0000000..25ae254 --- /dev/null +++ b/automation_file/remote/dropbox_api/list_ops.py @@ -0,0 +1,27 @@ +"""Dropbox listing operations.""" + +from __future__ import annotations + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.dropbox_api.client import dropbox_instance + + +def dropbox_list_folder(remote_path: str = "", recursive: bool = False) -> list[str]: + """Return every path under ``remote_path``.""" + client = dropbox_instance.require_client() + names: list[str] = [] + try: + result = client.files_list_folder(remote_path, recursive=recursive) + names.extend(entry.path_display for entry in result.entries) + while getattr(result, "has_more", False): + result = client.files_list_folder_continue(result.cursor) + names.extend(entry.path_display for entry in result.entries) + except Exception as error: # pylint: disable=broad-except + file_automation_logger.error("dropbox_list_folder failed: %r", error) + return [] + file_automation_logger.info( + "dropbox_list_folder: %s (%d entries)", + remote_path, + len(names), + ) + return names diff --git a/automation_file/remote/dropbox_api/upload_ops.py b/automation_file/remote/dropbox_api/upload_ops.py new file mode 100644 index 0000000..2bc3a65 --- /dev/null +++ b/automation_file/remote/dropbox_api/upload_ops.py @@ -0,0 +1,61 @@ +"""Dropbox upload operations.""" + +from __future__ import annotations + +from pathlib import Path + +from automation_file.exceptions import FileNotExistsException +from automation_file.logging_config import file_automation_logger +from automation_file.remote._upload_tree import walk_and_upload +from automation_file.remote.dropbox_api.client import dropbox_instance + + +def _normalise_path(remote_path: str) -> str: + return remote_path if remote_path.startswith("/") else f"/{remote_path}" + + +def dropbox_upload_file(file_path: str, remote_path: str) -> bool: + """Upload a single file to ``remote_path`` (overwrites).""" + path = Path(file_path) + if not path.is_file(): + raise FileNotExistsException(str(path)) + client = dropbox_instance.require_client() + try: + from dropbox import files as dropbox_files + except ImportError as error: + raise RuntimeError( + "dropbox import failed — reinstall `automation_file` to restore the Dropbox backend" + ) from error + try: + with open(path, "rb") as fp: + client.files_upload( + fp.read(), + _normalise_path(remote_path), + mode=dropbox_files.WriteMode.overwrite, + ) + file_automation_logger.info( + "dropbox_upload_file: %s -> %s", + path, + remote_path, + ) + return True + except Exception as error: # pylint: disable=broad-except + file_automation_logger.error("dropbox_upload_file failed: %r", error) + return False + + +def dropbox_upload_dir(dir_path: str, remote_prefix: str = "/") -> list[str]: + """Upload every file under ``dir_path`` to Dropbox under ``remote_prefix``.""" + result = walk_and_upload( + dir_path, + remote_prefix, + lambda prefix, rel: f"{prefix}/{rel}" if prefix else f"/{rel}", + lambda local, remote: dropbox_upload_file(str(local), remote), + ) + file_automation_logger.info( + "dropbox_upload_dir: %s -> %s (%d files)", + result.source, + result.prefix, + len(result.uploaded), + ) + return result.uploaded diff --git a/automation_file/remote/google_drive/client.py b/automation_file/remote/google_drive/client.py new file mode 100644 index 0000000..b361c32 --- /dev/null +++ b/automation_file/remote/google_drive/client.py @@ -0,0 +1,75 @@ +"""Google Drive client (Singleton Facade). + +Wraps OAuth2 credential loading and exposes a lazily-built ``service`` attribute +that every operation module calls through. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from google.auth.transport.requests import Request +from google.oauth2.credentials import Credentials +from google_auth_oauthlib.flow import InstalledAppFlow +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError + +from automation_file.logging_config import file_automation_logger + +_DEFAULT_SCOPES = ("https://www.googleapis.com/auth/drive",) + + +class GoogleDriveClient: + """Holds credentials and the Drive API service handle.""" + + def __init__(self, scopes: tuple[str, ...] = _DEFAULT_SCOPES) -> None: + self.scopes: tuple[str, ...] = scopes + self.creds: Credentials | None = None + self.service: Any = None + + def later_init(self, token_path: str, credentials_path: str) -> Any: + """Load / refresh credentials and build the Drive service. + + Writes the refreshed token back to ``token_path`` with UTF-8 encoding. + """ + token_file = Path(token_path) + credentials_file = Path(credentials_path) + creds: Credentials | None = None + + if token_file.exists(): + file_automation_logger.info("GoogleDriveClient: loading token from %s", token_file) + creds = Credentials.from_authorized_user_file(str(token_file), list(self.scopes)) + + if creds is None or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + flow = InstalledAppFlow.from_client_secrets_file( + str(credentials_file), + list(self.scopes), + ) + creds = flow.run_local_server(port=0) + with open(token_file, "w", encoding="utf-8") as token_fp: + token_fp.write(creds.to_json()) + + try: + self.creds = creds + self.service = build("drive", "v3", credentials=creds) + file_automation_logger.info("GoogleDriveClient: service ready") + return self.service + except HttpError as error: + file_automation_logger.error("GoogleDriveClient init failed: %r", error) + self.service = None + raise + + def require_service(self) -> Any: + """Return ``self.service`` or raise if the client has not been initialised.""" + if self.service is None: + raise RuntimeError( + "GoogleDriveClient not initialised; call later_init(token, credentials) first" + ) + return self.service + + +driver_instance: GoogleDriveClient = GoogleDriveClient() diff --git a/automation_file/remote/google_drive/delete/delete_manager.py b/automation_file/remote/google_drive/delete/delete_manager.py deleted file mode 100644 index 5f46657..0000000 --- a/automation_file/remote/google_drive/delete/delete_manager.py +++ /dev/null @@ -1,36 +0,0 @@ -from typing import Union, Dict - -from googleapiclient.errors import HttpError - -# 匯入 Google Drive 驅動實例與日誌工具 -# Import Google Drive driver instance and logging utility -from automation_file.remote.google_drive.driver_instance import driver_instance -from automation_file.utils.logging.loggin_instance import file_automation_logger - - -def drive_delete_file(file_id: str) -> Union[Dict[str, str], None]: - """ - 刪除 Google Drive 上的檔案 - Delete a file from Google Drive - :param file_id: Google Drive 檔案 ID (str) - Google Drive file ID (str) - :return: 若成功，回傳刪除結果 (Dict)，否則回傳 None - Return deletion result (Dict) if success, else None - """ - try: - # 呼叫 Google Drive API 刪除檔案 - # Call Google Drive API to delete file - file = driver_instance.service.files().delete(fileId=file_id).execute() - - # 記錄刪除成功的訊息 - # Log successful deletion - file_automation_logger.info(f"Delete drive file: {file_id}") - return file - - except HttpError as error: - # 捕捉 Google API 錯誤並記錄 - # Catch Google API error and log it - file_automation_logger.error( - f"Delete file failed, error: {error}" - ) - return None \ No newline at end of file diff --git a/automation_file/remote/google_drive/delete_ops.py b/automation_file/remote/google_drive/delete_ops.py new file mode 100644 index 0000000..6225a24 --- /dev/null +++ b/automation_file/remote/google_drive/delete_ops.py @@ -0,0 +1,21 @@ +"""Delete-side Google Drive operations.""" + +from __future__ import annotations + +from typing import Any + +from googleapiclient.errors import HttpError + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.google_drive.client import driver_instance + + +def drive_delete_file(file_id: str) -> Any | None: + """Delete a file by Drive ID. Returns the API response or None.""" + try: + result = driver_instance.require_service().files().delete(fileId=file_id).execute() + file_automation_logger.info("drive_delete_file: %s", file_id) + return result + except HttpError as error: + file_automation_logger.error("drive_delete_file failed: %r", error) + return None diff --git a/automation_file/remote/google_drive/dir/folder_manager.py b/automation_file/remote/google_drive/dir/folder_manager.py deleted file mode 100644 index dd6073d..0000000 --- a/automation_file/remote/google_drive/dir/folder_manager.py +++ /dev/null @@ -1,49 +0,0 @@ -from typing import Union - -from googleapiclient.errors import HttpError - -# 匯入 Google Drive 驅動實例與日誌工具 -# Import Google Drive driver instance and logging utility -from automation_file.remote.google_drive.driver_instance import driver_instance -from automation_file.utils.logging.loggin_instance import file_automation_logger - - -def drive_add_folder(folder_name: str) -> Union[dict, None]: - """ - 在 Google Drive 建立資料夾 - Create a folder on Google Drive - :param folder_name: 要建立的資料夾名稱 (str) - Folder name to create (str) - :return: 若成功，回傳資料夾 ID (dict)，否則回傳 None - Return folder ID (dict) if success, else None - """ - try: - # 設定資料夾的中繼資料 (名稱與 MIME 類型) - # Define folder metadata (name and MIME type) - file_metadata = { - "name": folder_name, - "mimeType": "application/vnd.google-apps.folder" - } - - # 呼叫 Google Drive API 建立資料夾，並只回傳 id 欄位 - # Call Google Drive API to create folder, return only "id" - file = driver_instance.service.files().create( - body=file_metadata, - fields="id" - ).execute() - - # 記錄建立成功的訊息 - # Log successful folder creation - file_automation_logger.info(f"Add drive folder: {folder_name}") - - # 回傳資料夾 ID - # Return folder ID - return file.get("id") - - except HttpError as error: - # 捕捉 Google API 錯誤並記錄 - # Catch Google API error and log it - file_automation_logger.error( - f"Add folder failed, error: {error}" - ) - return None \ No newline at end of file diff --git a/automation_file/remote/google_drive/download/__init__.py b/automation_file/remote/google_drive/download/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/automation_file/remote/google_drive/download/download_file.py b/automation_file/remote/google_drive/download/download_file.py deleted file mode 100644 index 54c2c7d..0000000 --- a/automation_file/remote/google_drive/download/download_file.py +++ /dev/null @@ -1,106 +0,0 @@ -import io -from io import BytesIO -from typing import Union - -from googleapiclient.errors import HttpError -from googleapiclient.http import MediaIoBaseDownload - -# 匯入 Google Drive 驅動實例與日誌工具 -# Import Google Drive driver instance and logging utility -from automation_file.remote.google_drive.driver_instance import driver_instance -from automation_file.utils.logging.loggin_instance import file_automation_logger - - -def drive_download_file(file_id: str, file_name: str) -> Union[BytesIO, None]: - """ - 從 Google Drive 下載單一檔案 - Download a single file from Google Drive - :param file_id: Google Drive 檔案 ID (str) - Google Drive file ID (str) - :param file_name: 本地端儲存檔案名稱 (str) - Local file name to save as (str) - :return: BytesIO 物件 (檔案內容) 或 None - BytesIO object (file content) or None - """ - try: - # 建立下載請求 - # Create download request - request = driver_instance.service.files().get_media(fileId=file_id) - - # 使用 BytesIO 暫存檔案內容 - # Use BytesIO to temporarily store file content - file = io.BytesIO() - - # 建立下載器 - # Create downloader - downloader = MediaIoBaseDownload(file, request) - done = False - - # 逐區塊下載檔案，直到完成 - # Download file in chunks until done - while done is False: - status, done = downloader.next_chunk() - file_automation_logger.info( - f"Download {file_name} {int(status.progress() * 100)}%." - ) - - except HttpError as error: - file_automation_logger.error( - f"Download file failed, error: {error}" - ) - return None - - # 將下載完成的檔案寫入本地端 - # Save downloaded file to local storage - with open(file_name, "wb") as output_file: - output_file.write(file.getbuffer()) - - file_automation_logger.info( - f"Download file: {file_id} with name: {file_name}" - ) - return file - - -def drive_download_file_from_folder(folder_name: str) -> Union[dict, None]: - """ - 從 Google Drive 指定資料夾下載所有檔案 - Download all files from a specific Google Drive folder - :param folder_name: 資料夾名稱 (str) - Folder name (str) - :return: 檔案名稱與 ID 的字典，或 None - Dictionary of file names and IDs, or None - """ - try: - files = dict() - - # 先找到指定名稱的資料夾 - # Find the folder by name - response = driver_instance.service.files().list( - q=f"mimeType = 'application/vnd.google-apps.folder' and name = '{folder_name}'" - ).execute() - - folder = response.get("files", [])[0] - folder_id = folder.get("id") - - # 列出該資料夾下的所有檔案 - # List all files inside the folder - response = driver_instance.service.files().list( - q=f"'{folder_id}' in parents" - ).execute() - - # 逐一下載檔案 - # Download each file - for file in response.get("files", []): - drive_download_file(file.get("id"), file.get("name")) - files.update({file.get("name"): file.get("id")}) - - file_automation_logger.info( - f"Download all file on {folder_name} done." - ) - return files - - except HttpError as error: - file_automation_logger.error( - f"Download file failed, error: {error}" - ) - return None \ No newline at end of file diff --git a/automation_file/remote/google_drive/download_ops.py b/automation_file/remote/google_drive/download_ops.py new file mode 100644 index 0000000..a99ee0a --- /dev/null +++ b/automation_file/remote/google_drive/download_ops.py @@ -0,0 +1,76 @@ +"""Download-side Google Drive operations.""" + +from __future__ import annotations + +import io + +from googleapiclient.errors import HttpError +from googleapiclient.http import MediaIoBaseDownload + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.google_drive.client import driver_instance + + +def drive_download_file(file_id: str, file_name: str) -> io.BytesIO | None: + """Download a single file by ID to ``file_name`` on disk. + + Returns the in-memory buffer on success, or ``None`` on failure. The file + is **only** written after the download completes cleanly, so a failed + request cannot leave an empty file behind. + """ + service = driver_instance.require_service() + buffer = io.BytesIO() + try: + request = service.files().get_media(fileId=file_id) + downloader = MediaIoBaseDownload(buffer, request) + done = False + while not done: + status, done = downloader.next_chunk() + if status is not None: + file_automation_logger.info( + "drive_download_file: %s %d%%", + file_name, + int(status.progress() * 100), + ) + except HttpError as error: + file_automation_logger.error("drive_download_file failed: %r", error) + return None + + with open(file_name, "wb") as output_file: + output_file.write(buffer.getbuffer()) + file_automation_logger.info("drive_download_file: %s -> %s", file_id, file_name) + return buffer + + +def drive_download_file_from_folder(folder_name: str) -> dict[str, str] | None: + """Download every file inside the Drive folder named ``folder_name``.""" + service = driver_instance.require_service() + try: + folders = ( + service.files() + .list(q=(f"mimeType = 'application/vnd.google-apps.folder' and name = '{folder_name}'")) + .execute() + ) + folder_list = folders.get("files", []) + if not folder_list: + file_automation_logger.error( + "drive_download_file_from_folder: folder not found: %s", + folder_name, + ) + return None + folder_id = folder_list[0].get("id") + response = service.files().list(q=f"'{folder_id}' in parents").execute() + except HttpError as error: + file_automation_logger.error("drive_download_file_from_folder failed: %r", error) + return None + + result: dict[str, str] = {} + for file in response.get("files", []): + drive_download_file(file.get("id"), file.get("name")) + result[file.get("name")] = file.get("id") + file_automation_logger.info( + "drive_download_file_from_folder: %s (%d files)", + folder_name, + len(result), + ) + return result diff --git a/automation_file/remote/google_drive/driver_instance.py b/automation_file/remote/google_drive/driver_instance.py deleted file mode 100644 index cdfa84c..0000000 --- a/automation_file/remote/google_drive/driver_instance.py +++ /dev/null @@ -1,77 +0,0 @@ -from pathlib import Path - -from google.auth.transport.requests import Request -from google.oauth2.credentials import Credentials -from google_auth_oauthlib.flow import InstalledAppFlow -from googleapiclient.discovery import build -from googleapiclient.errors import HttpError - -from automation_file.utils.logging.loggin_instance import file_automation_logger - - -class GoogleDrive(object): - - def __init__(self): - # Google Drive 實例相關屬性 - # Attributes for Google Drive instance - self.google_drive_instance = None - self.creds = None - self.service = None - # 權限範圍：完整存取 Google Drive - # Scope: full access to Google Drive - self.scopes = ["https://www.googleapis.com/auth/drive"] - - def later_init(self, token_path: str, credentials_path: str): - """ - 初始化 Google Drive API 驅動 - Initialize Google Drive API driver - :param token_path: Google Drive token 檔案路徑 (str) - Path to token.json file - :param credentials_path: Google Drive credentials 憑證檔案路徑 (str) - Path to credentials.json file - :return: None - """ - token_path = Path(token_path) - credentials_path = Path(credentials_path) - creds = None - - # token.json 儲存使用者的 access 與 refresh token - # token.json stores user's access and refresh tokens - if token_path.exists(): - file_automation_logger.info("Token exists, try to load.") - creds = Credentials.from_authorized_user_file(str(token_path), self.scopes) - - # 如果沒有有效的憑證，則重新登入 - # If no valid credentials, perform login - if not creds or not creds.valid: - if creds and creds.expired and creds.refresh_token: - # 如果憑證過期但有 refresh token，則刷新 - # Refresh credentials if expired but refresh token exists - creds.refresh(Request()) - else: - # 使用 OAuth2 流程重新登入 - # Use OAuth2 flow for login - flow = InstalledAppFlow.from_client_secrets_file( - str(credentials_path), self.scopes - ) - creds = flow.run_local_server(port=0) - - # 儲存憑證到 token.json，供下次使用 - # Save credentials to token.json for future use - with open(str(token_path), 'w') as token: - token.write(creds.to_json()) - - try: - # 建立 Google Drive API service - # Build Google Drive API service - self.service = build('drive', 'v3', credentials=creds) - file_automation_logger.info("Loading service successfully.") - except HttpError as error: - file_automation_logger.error( - f"Init service failed, error: {error}" - ) - - -# 建立單例，供其他模組使用 -# Create a singleton instance for other modules to use -driver_instance = GoogleDrive() \ No newline at end of file diff --git a/automation_file/remote/google_drive/folder_ops.py b/automation_file/remote/google_drive/folder_ops.py new file mode 100644 index 0000000..a10f026 --- /dev/null +++ b/automation_file/remote/google_drive/folder_ops.py @@ -0,0 +1,24 @@ +"""Folder (mkdir-equivalent) operations on Google Drive.""" + +from __future__ import annotations + +from googleapiclient.errors import HttpError + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.google_drive.client import driver_instance + +_FOLDER_MIME = "application/vnd.google-apps.folder" + + +def drive_add_folder(folder_name: str) -> str | None: + """Create a folder on Drive. Returns the new folder's ID or None.""" + metadata = {"name": folder_name, "mimeType": _FOLDER_MIME} + try: + response = ( + driver_instance.require_service().files().create(body=metadata, fields="id").execute() + ) + file_automation_logger.info("drive_add_folder: %s", folder_name) + return response.get("id") + except HttpError as error: + file_automation_logger.error("drive_add_folder failed: %r", error) + return None diff --git a/automation_file/remote/google_drive/search/__init__.py b/automation_file/remote/google_drive/search/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/automation_file/remote/google_drive/search/search_drive.py b/automation_file/remote/google_drive/search/search_drive.py deleted file mode 100644 index 3cccf40..0000000 --- a/automation_file/remote/google_drive/search/search_drive.py +++ /dev/null @@ -1,101 +0,0 @@ -from typing import Union - -from googleapiclient.errors import HttpError - -# 匯入 Google Drive 驅動實例與日誌工具 -# Import Google Drive driver instance and logging utility -from automation_file.remote.google_drive.driver_instance import driver_instance -from automation_file.utils.logging.loggin_instance import file_automation_logger - - -def drive_search_all_file() -> Union[dict, None]: - """ - 搜尋 Google Drive 上的所有檔案 - Search all files on Google Drive - :return: 檔案名稱與 ID 的字典，或 None - Dictionary of file names and IDs, or None - """ - try: - item = dict() - # 呼叫 Google Drive API 取得所有檔案 - # Call Google Drive API to list all files - response = driver_instance.service.files().list().execute() - for file in response.get("files", []): - item.update({file.get("name"): file.get("id")}) - - file_automation_logger.info("Search all file on drive") - return item - - except HttpError as error: - file_automation_logger.error( - f"Search file failed, error: {error}" - ) - return None - - -def drive_search_file_mimetype(mime_type: str) -> Union[dict, None]: - """ - 搜尋 Google Drive 上指定 MIME 類型的檔案 - Search all files with a specific MIME type on Google Drive - :param mime_type: MIME 類型 (str) - MIME type (str) - :return: 檔案名稱與 ID 的字典，或 None - Dictionary of file names and IDs, or None - """ - try: - files = dict() - page_token = None - while True: - # 呼叫 Google Drive API，依 MIME 類型搜尋檔案 - # Call Google Drive API to search files by MIME type - response = driver_instance.service.files().list( - q=f"mimeType='{mime_type}'", - fields="nextPageToken, files(id, name)", - pageToken=page_token - ).execute() - - for file in response.get("files", []): - files.update({file.get("name"): file.get("id")}) - - # 處理分頁結果 - # Handle pagination - page_token = response.get('nextPageToken', None) - if page_token is None: - break - - file_automation_logger.info(f"Search all {mime_type} file on drive") - return files - - except HttpError as error: - file_automation_logger.error( - f"Search file failed, error: {error}" - ) - return None - - -def drive_search_field(field_pattern: str) -> Union[dict, None]: - """ - 使用自訂欄位模式搜尋檔案 - Search files with a custom field pattern - :param field_pattern: 欄位模式 (str) - Field pattern (str) - :return: 檔案名稱與 ID 的字典，或 None - Dictionary of file names and IDs, or None - """ - try: - files = dict() - # 呼叫 Google Drive API，依指定欄位模式搜尋 - # Call Google Drive API with custom field pattern - response = driver_instance.service.files().list(fields=field_pattern).execute() - - for file in response.get("files", []): - files.update({file.get("name"): file.get("id")}) - - file_automation_logger.info(f"Search all {field_pattern}") - return files - - except HttpError as error: - file_automation_logger.error( - f"Search file failed, error: {error}" - ) - return None \ No newline at end of file diff --git a/automation_file/remote/google_drive/search_ops.py b/automation_file/remote/google_drive/search_ops.py new file mode 100644 index 0000000..cb3adb0 --- /dev/null +++ b/automation_file/remote/google_drive/search_ops.py @@ -0,0 +1,62 @@ +"""Search-side Google Drive operations.""" + +from __future__ import annotations + +from googleapiclient.errors import HttpError + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.google_drive.client import driver_instance + + +def drive_search_all_file() -> dict[str, str] | None: + """Return ``{name: id}`` for every file visible to the current token.""" + try: + response = driver_instance.require_service().files().list().execute() + except HttpError as error: + file_automation_logger.error("drive_search_all_file failed: %r", error) + return None + result = {file.get("name"): file.get("id") for file in response.get("files", [])} + file_automation_logger.info("drive_search_all_file: %d results", len(result)) + return result + + +def drive_search_file_mimetype(mime_type: str) -> dict[str, str] | None: + """Return ``{name: id}`` for files matching ``mime_type`` (all pages).""" + results: dict[str, str] = {} + page_token: str | None = None + service = driver_instance.require_service() + try: + while True: + response = ( + service.files() + .list( + q=f"mimeType='{mime_type}'", + fields="nextPageToken, files(id, name)", + pageToken=page_token, + ) + .execute() + ) + for file in response.get("files", []): + results[file.get("name")] = file.get("id") + page_token = response.get("nextPageToken") + if page_token is None: + break + except HttpError as error: + file_automation_logger.error("drive_search_file_mimetype failed: %r", error) + return None + file_automation_logger.info( + "drive_search_file_mimetype: mime=%s %d results", mime_type, len(results) + ) + return results + + +def drive_search_field(field_pattern: str) -> dict[str, str] | None: + """Return ``{name: id}`` for a list call with a custom ``fields=`` pattern.""" + try: + response = driver_instance.require_service().files().list(fields=field_pattern).execute() + except HttpError as error: + file_automation_logger.error("drive_search_field failed: %r", error) + return None + result = {file.get("name"): file.get("id") for file in response.get("files", [])} + file_automation_logger.info("drive_search_field: %d results", len(result)) + return result diff --git a/automation_file/remote/google_drive/share/__init__.py b/automation_file/remote/google_drive/share/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/automation_file/remote/google_drive/share/share_file.py b/automation_file/remote/google_drive/share/share_file.py deleted file mode 100644 index bf1d020..0000000 --- a/automation_file/remote/google_drive/share/share_file.py +++ /dev/null @@ -1,113 +0,0 @@ -from typing import Union - -from googleapiclient.errors import HttpError - -# 匯入 Google Drive 驅動實例與日誌工具 -# Import Google Drive driver instance and logging utility -from automation_file.remote.google_drive.driver_instance import driver_instance -from automation_file.utils.logging.loggin_instance import file_automation_logger - - -def drive_share_file_to_user( - file_id: str, user: str, user_role: str = "writer") -> Union[dict, None]: - """ - 分享檔案給指定使用者 - Share a file with a specific user - :param file_id: 要分享的檔案 ID (str) - File ID to share (str) - :param user: 使用者的 email (str) - User email address (str) - :param user_role: 權限角色 (預設 writer) - Permission role (default writer) - :return: 成功回傳 dict，失敗回傳 None - Return dict if success, else None - """ - try: - service = driver_instance.service - user_permission = { - "type": "user", - "role": user_role, - "emailAddress": user - } - file_automation_logger.info( - f"Share file: {file_id}, to user: {user}, with user role: {user_role}" - ) - return service.permissions().create( - fileId=file_id, - body=user_permission, - fields='id', - ).execute() - except HttpError as error: - file_automation_logger.error( - f"Share file failed, error: {error}" - ) - return None - - -def drive_share_file_to_anyone(file_id: str, share_role: str = "reader") -> Union[dict, None]: - """ - 分享檔案給任何人（公開連結） - Share a file with anyone (public link) - :param file_id: 要分享的檔案 ID (str) - File ID to share (str) - :param share_role: 權限角色 (預設 reader) - Permission role (default reader) - :return: 成功回傳 dict，失敗回傳 None - Return dict if success, else None - """ - try: - service = driver_instance.service - user_permission = { - "type": "anyone", - "value": "anyone", - "role": share_role - } - file_automation_logger.info( - f"Share file to anyone, file: {file_id} with role: {share_role}" - ) - return service.permissions().create( - fileId=file_id, - body=user_permission, - fields='id', - ).execute() - except HttpError as error: - file_automation_logger.error( - f"Share file failed, error: {error}" - ) - return None - - -def drive_share_file_to_domain( - file_id: str, domain: str, domain_role: str = "reader") -> Union[dict, None]: - """ - 分享檔案給指定網域的所有使用者 - Share a file with all users in a specific domain - :param file_id: 要分享的檔案 ID (str) - File ID to share (str) - :param domain: 網域名稱 (str)，例如 "example.com" - Domain name (str), e.g., "example.com" - :param domain_role: 權限角色 (預設 reader) - Permission role (default reader) - :return: 成功回傳 dict，失敗回傳 None - Return dict if success, else None - """ - try: - service = driver_instance.service - domain_permission = { - "type": "domain", - "role": domain_role, - "domain": domain - } - file_automation_logger.info( - f"Share file to domain: {domain}, with domain role: {domain_role}" - ) - return service.permissions().create( - fileId=file_id, - body=domain_permission, - fields='id', - ).execute() - except HttpError as error: - file_automation_logger.error( - f"Share file failed, error: {error}" - ) - return None \ No newline at end of file diff --git a/automation_file/remote/google_drive/share_ops.py b/automation_file/remote/google_drive/share_ops.py new file mode 100644 index 0000000..921e964 --- /dev/null +++ b/automation_file/remote/google_drive/share_ops.py @@ -0,0 +1,40 @@ +"""Permission / share operations on Google Drive.""" + +from __future__ import annotations + +from googleapiclient.errors import HttpError + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.google_drive.client import driver_instance + + +def _create_permission(file_id: str, body: dict, description: str) -> dict | None: + try: + response = ( + driver_instance.require_service() + .permissions() + .create(fileId=file_id, body=body, fields="id") + .execute() + ) + file_automation_logger.info("drive_share (%s): file=%s", description, file_id) + return response + except HttpError as error: + file_automation_logger.error("drive_share (%s) failed: %r", description, error) + return None + + +def drive_share_file_to_user(file_id: str, user: str, user_role: str = "writer") -> dict | None: + body = {"type": "user", "role": user_role, "emailAddress": user} + return _create_permission(file_id, body, f"user={user},role={user_role}") + + +def drive_share_file_to_anyone(file_id: str, share_role: str = "reader") -> dict | None: + body = {"type": "anyone", "role": share_role} + return _create_permission(file_id, body, f"anyone,role={share_role}") + + +def drive_share_file_to_domain( + file_id: str, domain: str, domain_role: str = "reader" +) -> dict | None: + body = {"type": "domain", "role": domain_role, "domain": domain} + return _create_permission(file_id, body, f"domain={domain},role={domain_role}") diff --git a/automation_file/remote/google_drive/upload/__init__.py b/automation_file/remote/google_drive/upload/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/automation_file/remote/google_drive/upload/upload_to_driver.py b/automation_file/remote/google_drive/upload/upload_to_driver.py deleted file mode 100644 index a700fc8..0000000 --- a/automation_file/remote/google_drive/upload/upload_to_driver.py +++ /dev/null @@ -1,159 +0,0 @@ -from pathlib import Path -from typing import List, Union, Optional - -from googleapiclient.errors import HttpError -from googleapiclient.http import MediaFileUpload - -# 匯入 Google Drive 驅動實例與日誌工具 -# Import Google Drive driver instance and logging utility -from automation_file.remote.google_drive.driver_instance import driver_instance -from automation_file.utils.logging.loggin_instance import file_automation_logger - - -def drive_upload_to_drive(file_path: str, file_name: str = None) -> Union[dict, None]: - """ - 上傳單一檔案到 Google Drive 根目錄 - Upload a single file to Google Drive root - :param file_path: 要上傳的檔案路徑 (str) - File path to upload (str) - :param file_name: 在 Google Drive 上的檔案名稱 (可選) - File name on Google Drive (optional) - :return: 成功回傳 dict (包含檔案 ID)，失敗回傳 None - Return dict (with file ID) if success, else None - """ - try: - file_path = Path(file_path) - if file_path.is_file(): - file_metadata = { - "name": file_path.name if file_name is None else file_name, - "mimeType": "*/*" - } - media = MediaFileUpload( - file_path, - mimetype="*/*", - resumable=True - ) - file_id = driver_instance.service.files().create( - body=file_metadata, - media_body=media, - fields="id" - ).execute() - file_automation_logger.info( - f"Upload file to drive file: {file_path}, with name: {file_name}" - ) - return file_id - else: - # 若檔案不存在，記錄錯誤 - # Log error if file does not exist - file_automation_logger.error(FileNotFoundError) - except HttpError as error: - # ⚠️ 原本寫成 Delete file failed，應改為 Upload file failed - file_automation_logger.error( - f"Upload file failed, error: {error}" - ) - return None - - -def drive_upload_to_folder(folder_id: str, file_path: str, file_name: str = None) -> Union[dict, None]: - """ - 上傳單一檔案到 Google Drive 指定資料夾 - Upload a single file into a specific Google Drive folder - :param folder_id: 目標資料夾 ID (str) - Target folder ID (str) - :param file_path: 要上傳的檔案路徑 (str) - File path to upload (str) - :param file_name: 在 Google Drive 上的檔案名稱 (可選) - File name on Google Drive (optional) - :return: 成功回傳 dict (包含檔案 ID)，失敗回傳 None - Return dict (with file ID) if success, else None - """ - try: - file_path = Path(file_path) - if file_path.is_file(): - file_metadata = { - "name": file_path.name if file_name is None else file_name, - "mimeType": "*/*", - "parents": [f"{folder_id}"] - } - media = MediaFileUpload( - file_path, - mimetype="*/*", - resumable=True - ) - file_id = driver_instance.service.files().create( - body=file_metadata, - media_body=media, - fields="id" - ).execute() - file_automation_logger.info( - f"Upload file to folder: {folder_id}, file_path: {file_path}, with name: {file_name}" - ) - return file_id - else: - file_automation_logger.error(FileNotFoundError) - except HttpError as error: - file_automation_logger.error( - f"Upload file failed, error: {error}" - ) - return None - - -def drive_upload_dir_to_drive(dir_path: str) -> List[Optional[dict]] | None: - """ - 上傳整個資料夾中的所有檔案到 Google Drive 根目錄 - Upload all files from a local directory to Google Drive root - :param dir_path: 要上傳的資料夾路徑 (str) - Directory path to upload (str) - :return: 檔案 ID 清單 (List[dict])，或空清單 - List of file IDs (List[dict]) or empty list - """ - dir_path = Path(dir_path) - ids = list() - if dir_path.is_dir(): - path_list = dir_path.iterdir() - for path in path_list: - if path.is_file(): - ids.append(drive_upload_to_drive(str(path.absolute()), path.name)) - file_automation_logger.info( - f"Upload all file on dir: {dir_path} to drive" - ) - return ids - else: - file_automation_logger.error(FileNotFoundError) - return None - - -def drive_upload_dir_to_folder(folder_id: str, dir_path: str) -> List[Optional[dict]] | None: - """ - 上傳整個資料夾中的所有檔案到 Google Drive 指定資料夾 - Upload all files from a local directory into a specific Google Drive folder - - :param folder_id: 目標 Google Drive 資料夾 ID (str) - Target Google Drive folder ID (str) - :param dir_path: 本地端要上傳的資料夾路徑 (str) - Local directory path to upload (str) - :return: 檔案 ID 清單 (List[dict])，或 None - List of file IDs (List[dict]) or None - """ - dir_path = Path(dir_path) - ids: List[Optional[dict]] = [] - - if dir_path.is_dir(): - path_list = dir_path.iterdir() - for path in path_list: - if path.is_file(): - # 呼叫單檔上傳函式 (drive_upload_to_folder)，並收集回傳的檔案 ID - # Call single-file upload function and collect returned file ID - ids.append(drive_upload_to_folder(folder_id, str(path.absolute()), path.name)) - - file_automation_logger.info( - f"Upload all files in dir: {dir_path} to folder: {folder_id}" - ) - return ids - else: - # 若資料夾不存在，記錄錯誤 - # Log error if directory does not exist - file_automation_logger.error(FileNotFoundError) - - return None - diff --git a/automation_file/remote/google_drive/upload_ops.py b/automation_file/remote/google_drive/upload_ops.py new file mode 100644 index 0000000..530fc90 --- /dev/null +++ b/automation_file/remote/google_drive/upload_ops.py @@ -0,0 +1,89 @@ +"""Upload-side Google Drive operations.""" + +from __future__ import annotations + +import mimetypes +from pathlib import Path + +from googleapiclient.errors import HttpError +from googleapiclient.http import MediaFileUpload + +from automation_file.exceptions import FileNotExistsException +from automation_file.logging_config import file_automation_logger +from automation_file.remote.google_drive.client import driver_instance + + +def _guess_mime(path: Path) -> str: + mime, _ = mimetypes.guess_type(path.name) + return mime or "application/octet-stream" + + +def _upload(path: Path, metadata: dict, description: str) -> dict | None: + try: + media = MediaFileUpload(str(path), mimetype=_guess_mime(path), resumable=True) + response = ( + driver_instance.require_service() + .files() + .create(body=metadata, media_body=media, fields="id") + .execute() + ) + file_automation_logger.info("drive_upload (%s): %s", description, path) + return response + except HttpError as error: + file_automation_logger.error("drive_upload (%s) failed: %r", description, error) + return None + + +def drive_upload_to_drive(file_path: str, file_name: str | None = None) -> dict | None: + """Upload a single file to the Drive root.""" + path = Path(file_path) + if not path.is_file(): + raise FileNotExistsException(str(path)) + metadata = {"name": file_name or path.name, "mimeType": _guess_mime(path)} + return _upload(path, metadata, f"root,name={metadata['name']}") + + +def drive_upload_to_folder( + folder_id: str, file_path: str, file_name: str | None = None +) -> dict | None: + """Upload a single file into a specific Drive folder.""" + path = Path(file_path) + if not path.is_file(): + raise FileNotExistsException(str(path)) + metadata = { + "name": file_name or path.name, + "mimeType": _guess_mime(path), + "parents": [folder_id], + } + return _upload(path, metadata, f"folder={folder_id},name={metadata['name']}") + + +def drive_upload_dir_to_drive(dir_path: str) -> list[dict | None]: + """Upload every file in ``dir_path`` (non-recursive) to the Drive root.""" + source = Path(dir_path) + if not source.is_dir(): + return [] + results: list[dict | None] = [] + for entry in source.iterdir(): + if entry.is_file(): + results.append(drive_upload_to_drive(str(entry.absolute()), entry.name)) + file_automation_logger.info("drive_upload_dir_to_drive: %s (%d files)", source, len(results)) + return results + + +def drive_upload_dir_to_folder(folder_id: str, dir_path: str) -> list[dict | None]: + """Upload every file in ``dir_path`` (non-recursive) to a Drive folder.""" + source = Path(dir_path) + if not source.is_dir(): + return [] + results: list[dict | None] = [] + for entry in source.iterdir(): + if entry.is_file(): + results.append(drive_upload_to_folder(folder_id, str(entry.absolute()), entry.name)) + file_automation_logger.info( + "drive_upload_dir_to_folder: %s -> %s (%d files)", + source, + folder_id, + len(results), + ) + return results diff --git a/automation_file/remote/http_download.py b/automation_file/remote/http_download.py new file mode 100644 index 0000000..1b83382 --- /dev/null +++ b/automation_file/remote/http_download.py @@ -0,0 +1,114 @@ +"""SSRF-guarded HTTP downloader.""" + +from __future__ import annotations + +import requests +from tqdm import tqdm + +from automation_file.core.retry import retry_on_transient +from automation_file.exceptions import RetryExhaustedException, UrlValidationException +from automation_file.logging_config import file_automation_logger +from automation_file.remote.url_validator import validate_http_url + +_DEFAULT_TIMEOUT_SECONDS = 15 +_DEFAULT_CHUNK_SIZE = 1024 * 64 +_MAX_RESPONSE_BYTES = 20 * 1024 * 1024 + +_RETRIABLE_EXCEPTIONS = ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + requests.exceptions.ChunkedEncodingError, +) + + +@retry_on_transient(max_attempts=3, backoff_base=0.5, retriable=_RETRIABLE_EXCEPTIONS) +def _open_stream( + file_url: str, + timeout: int, +) -> requests.Response: + response = requests.get( + file_url, + stream=True, + timeout=timeout, + allow_redirects=False, + ) + response.raise_for_status() + return response + + +def download_file( + file_url: str, + file_name: str, + chunk_size: int = _DEFAULT_CHUNK_SIZE, + timeout: int = _DEFAULT_TIMEOUT_SECONDS, + max_bytes: int = _MAX_RESPONSE_BYTES, +) -> bool: + """Download ``file_url`` to ``file_name`` with progress display. + + Validates the URL against SSRF rules, disables redirects, enforces a size + cap, retries transient network errors up to three times, and uses default + TLS verification. Returns True on success. + """ + try: + validate_http_url(file_url) + except UrlValidationException as error: + file_automation_logger.error("download_file rejected URL: %r", error) + return False + + try: + response = _open_stream(file_url, timeout) + except RetryExhaustedException as error: + file_automation_logger.error("download_file retries exhausted: %r", error) + return False + except requests.exceptions.HTTPError as error: + file_automation_logger.error("download_file HTTP error: %r", error) + return False + except requests.exceptions.RequestException as error: + file_automation_logger.error("download_file request error: %r", error) + return False + + total_size = int(response.headers.get("content-length", 0)) + if total_size > max_bytes: + file_automation_logger.error( + "download_file rejected: content-length %d > %d", + total_size, + max_bytes, + ) + return False + + written = 0 + try: + with open(file_name, "wb") as output, _progress(total_size, file_name) as progress: + for chunk in response.iter_content(chunk_size=chunk_size): + if not chunk: + continue + written += len(chunk) + if written > max_bytes: + file_automation_logger.error( + "download_file aborted: stream exceeded %d bytes", + max_bytes, + ) + return False + output.write(chunk) + progress.update(len(chunk)) + except OSError as error: + file_automation_logger.error("download_file write error: %r", error) + return False + + file_automation_logger.info("download_file: %s -> %s (%d bytes)", file_url, file_name, written) + return True + + +class _NullBar: + def update(self, _n: int) -> None: ... + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + +def _progress(total: int, label: str): + if total > 0: + return tqdm(total=total, unit="B", unit_scale=True, desc=label) + return _NullBar() diff --git a/automation_file/remote/s3/__init__.py b/automation_file/remote/s3/__init__.py new file mode 100644 index 0000000..0d0d1d3 --- /dev/null +++ b/automation_file/remote/s3/__init__.py @@ -0,0 +1,29 @@ +"""S3 strategy module. + +S3 actions (``FA_s3_*``) are registered on the shared default registry +automatically. :func:`register_s3_ops` is kept public for callers that build +their own :class:`ActionRegistry` instances. +""" + +from __future__ import annotations + +from automation_file.core.action_registry import ActionRegistry +from automation_file.remote.s3 import delete_ops, download_ops, list_ops, upload_ops +from automation_file.remote.s3.client import S3Client, s3_instance + + +def register_s3_ops(registry: ActionRegistry) -> None: + """Register every ``FA_s3_*`` command into ``registry``.""" + registry.register_many( + { + "FA_s3_later_init": s3_instance.later_init, + "FA_s3_upload_file": upload_ops.s3_upload_file, + "FA_s3_upload_dir": upload_ops.s3_upload_dir, + "FA_s3_download_file": download_ops.s3_download_file, + "FA_s3_delete_object": delete_ops.s3_delete_object, + "FA_s3_list_bucket": list_ops.s3_list_bucket, + } + ) + + +__all__ = ["S3Client", "register_s3_ops", "s3_instance"] diff --git a/automation_file/remote/s3/client.py b/automation_file/remote/s3/client.py new file mode 100644 index 0000000..05b6cfc --- /dev/null +++ b/automation_file/remote/s3/client.py @@ -0,0 +1,51 @@ +"""S3 client (Singleton Facade around ``boto3``).""" + +from __future__ import annotations + +from typing import Any + +from automation_file.logging_config import file_automation_logger + + +def _import_boto3() -> Any: + try: + import boto3 + except ImportError as error: + raise RuntimeError( + "boto3 import failed — reinstall `automation_file` to restore the S3 backend" + ) from error + return boto3 + + +class S3Client: + """Lazy wrapper around ``boto3.client('s3', ...)``.""" + + def __init__(self) -> None: + self.client: Any = None + + def later_init( + self, + aws_access_key_id: str | None = None, + aws_secret_access_key: str | None = None, + region_name: str | None = None, + endpoint_url: str | None = None, + ) -> Any: + """Build a boto3 S3 client. Arguments default to the standard AWS chain.""" + boto3 = _import_boto3() + self.client = boto3.client( + "s3", + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + region_name=region_name, + endpoint_url=endpoint_url, + ) + file_automation_logger.info("S3Client: client ready (region=%s)", region_name) + return self.client + + def require_client(self) -> Any: + if self.client is None: + raise RuntimeError("S3Client not initialised; call later_init() first") + return self.client + + +s3_instance: S3Client = S3Client() diff --git a/automation_file/remote/s3/delete_ops.py b/automation_file/remote/s3/delete_ops.py new file mode 100644 index 0000000..1fd1964 --- /dev/null +++ b/automation_file/remote/s3/delete_ops.py @@ -0,0 +1,18 @@ +"""S3 delete operations.""" + +from __future__ import annotations + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.s3.client import s3_instance + + +def s3_delete_object(bucket: str, key: str) -> bool: + """Delete ``s3://bucket/key``. Returns True on success.""" + client = s3_instance.require_client() + try: + client.delete_object(Bucket=bucket, Key=key) + file_automation_logger.info("s3_delete_object: s3://%s/%s", bucket, key) + return True + except Exception as error: # pylint: disable=broad-except + file_automation_logger.error("s3_delete_object failed: %r", error) + return False diff --git a/automation_file/remote/s3/download_ops.py b/automation_file/remote/s3/download_ops.py new file mode 100644 index 0000000..3e6a85f --- /dev/null +++ b/automation_file/remote/s3/download_ops.py @@ -0,0 +1,26 @@ +"""S3 download operations.""" + +from __future__ import annotations + +from pathlib import Path + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.s3.client import s3_instance + + +def s3_download_file(bucket: str, key: str, target_path: str) -> bool: + """Download ``s3://bucket/key`` to ``target_path``.""" + client = s3_instance.require_client() + Path(target_path).parent.mkdir(parents=True, exist_ok=True) + try: + client.download_file(bucket, key, target_path) + file_automation_logger.info( + "s3_download_file: s3://%s/%s -> %s", + bucket, + key, + target_path, + ) + return True + except Exception as error: # pylint: disable=broad-except + file_automation_logger.error("s3_download_file failed: %r", error) + return False diff --git a/automation_file/remote/s3/list_ops.py b/automation_file/remote/s3/list_ops.py new file mode 100644 index 0000000..12adfee --- /dev/null +++ b/automation_file/remote/s3/list_ops.py @@ -0,0 +1,27 @@ +"""S3 listing operations.""" + +from __future__ import annotations + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.s3.client import s3_instance + + +def s3_list_bucket(bucket: str, prefix: str = "") -> list[str]: + """Return every key under ``bucket``/``prefix`` (paginated).""" + client = s3_instance.require_client() + keys: list[str] = [] + try: + paginator = client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): + for entry in page.get("Contents", []): + keys.append(entry["Key"]) + except Exception as error: # pylint: disable=broad-except + file_automation_logger.error("s3_list_bucket failed: %r", error) + return [] + file_automation_logger.info( + "s3_list_bucket: s3://%s/%s (%d keys)", + bucket, + prefix, + len(keys), + ) + return keys diff --git a/automation_file/remote/s3/upload_ops.py b/automation_file/remote/s3/upload_ops.py new file mode 100644 index 0000000..27e1030 --- /dev/null +++ b/automation_file/remote/s3/upload_ops.py @@ -0,0 +1,43 @@ +"""S3 upload operations.""" + +from __future__ import annotations + +from pathlib import Path + +from automation_file.exceptions import FileNotExistsException +from automation_file.logging_config import file_automation_logger +from automation_file.remote._upload_tree import walk_and_upload +from automation_file.remote.s3.client import s3_instance + + +def s3_upload_file(file_path: str, bucket: str, key: str) -> bool: + """Upload a single file to ``s3://bucket/key``.""" + path = Path(file_path) + if not path.is_file(): + raise FileNotExistsException(str(path)) + client = s3_instance.require_client() + try: + client.upload_file(str(path), bucket, key) + file_automation_logger.info("s3_upload_file: %s -> s3://%s/%s", path, bucket, key) + return True + except Exception as error: # pylint: disable=broad-except + file_automation_logger.error("s3_upload_file failed: %r", error) + return False + + +def s3_upload_dir(dir_path: str, bucket: str, key_prefix: str = "") -> list[str]: + """Upload every file under ``dir_path`` to ``bucket`` under ``key_prefix``.""" + result = walk_and_upload( + dir_path, + key_prefix, + lambda prefix, rel: f"{prefix}/{rel}" if prefix else rel, + lambda local, key: s3_upload_file(str(local), bucket, key), + ) + file_automation_logger.info( + "s3_upload_dir: %s -> s3://%s/%s (%d files)", + result.source, + bucket, + result.prefix, + len(result.uploaded), + ) + return result.uploaded diff --git a/automation_file/remote/sftp/__init__.py b/automation_file/remote/sftp/__init__.py new file mode 100644 index 0000000..0dacc5d --- /dev/null +++ b/automation_file/remote/sftp/__init__.py @@ -0,0 +1,29 @@ +"""SFTP strategy module. + +Actions (``FA_sftp_*``) are registered on the shared default registry +automatically. +""" + +from __future__ import annotations + +from automation_file.core.action_registry import ActionRegistry +from automation_file.remote.sftp import delete_ops, download_ops, list_ops, upload_ops +from automation_file.remote.sftp.client import SFTPClient, sftp_instance + + +def register_sftp_ops(registry: ActionRegistry) -> None: + """Register every ``FA_sftp_*`` command into ``registry``.""" + registry.register_many( + { + "FA_sftp_later_init": sftp_instance.later_init, + "FA_sftp_close": sftp_instance.close, + "FA_sftp_upload_file": upload_ops.sftp_upload_file, + "FA_sftp_upload_dir": upload_ops.sftp_upload_dir, + "FA_sftp_download_file": download_ops.sftp_download_file, + "FA_sftp_delete_path": delete_ops.sftp_delete_path, + "FA_sftp_list_dir": list_ops.sftp_list_dir, + } + ) + + +__all__ = ["SFTPClient", "register_sftp_ops", "sftp_instance"] diff --git a/automation_file/remote/sftp/client.py b/automation_file/remote/sftp/client.py new file mode 100644 index 0000000..0c4861e --- /dev/null +++ b/automation_file/remote/sftp/client.py @@ -0,0 +1,105 @@ +"""SFTP client (Singleton Facade over ``paramiko``). + +Host key policy is strict: unknown hosts raise ``SSHException``. Callers must +supply a ``known_hosts`` path (defaults to the OpenSSH user file) so that +host identity is pinned. We never fall back to ``AutoAddPolicy`` — silently +trusting new hosts defeats the point of SSH host verification. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from automation_file.logging_config import file_automation_logger + + +def _import_paramiko() -> Any: + try: + import paramiko + except ImportError as error: + raise RuntimeError( + "paramiko import failed — reinstall `automation_file` to restore the SFTP backend" + ) from error + return paramiko + + +@dataclass(frozen=True) +class SFTPConnectOptions: + """Connection parameters for :meth:`SFTPClient.later_init`.""" + + host: str + username: str + password: str | None = None + key_filename: str | None = None + port: int = 22 + known_hosts: str | None = None + timeout: float = 15.0 + + +class SFTPClient: + """Paramiko SSH + SFTP facade with strict host-key verification.""" + + def __init__(self) -> None: + self._ssh: Any = None + self._sftp: Any = None + + def later_init(self, options: SFTPConnectOptions | None = None, **kwargs: Any) -> Any: + """Open the SSH + SFTP session. Raises if the host key is not pinned. + + Accepts either a prebuilt :class:`SFTPConnectOptions` instance or + the same field names as keyword arguments (so action-registry + callers can keep their existing kwargs form). + """ + opts = options if options is not None else SFTPConnectOptions(**kwargs) + paramiko = _import_paramiko() + ssh = paramiko.SSHClient() + resolved_known = opts.known_hosts or str(Path.home() / ".ssh" / "known_hosts") + if Path(resolved_known).exists(): + ssh.load_host_keys(resolved_known) + else: + file_automation_logger.warning( + "SFTPClient: known_hosts %s missing; unknown host will be rejected", + resolved_known, + ) + ssh.set_missing_host_key_policy(paramiko.RejectPolicy()) + ssh.connect( + hostname=opts.host, + port=opts.port, + username=opts.username, + password=opts.password, + key_filename=opts.key_filename, + timeout=opts.timeout, + allow_agent=False, + look_for_keys=opts.key_filename is None, + ) + self._ssh = ssh + self._sftp = ssh.open_sftp() + file_automation_logger.info( + "SFTPClient: connected to %s@%s:%d", opts.username, opts.host, opts.port + ) + return self._sftp + + def require_sftp(self) -> Any: + if self._sftp is None: + raise RuntimeError("SFTPClient not initialised; call later_init() first") + return self._sftp + + def close(self) -> bool: + """Close the underlying SFTP and SSH connections.""" + if self._sftp is not None: + try: + self._sftp.close() + finally: + self._sftp = None + if self._ssh is not None: + try: + self._ssh.close() + finally: + self._ssh = None + file_automation_logger.info("SFTPClient: closed") + return True + + +sftp_instance: SFTPClient = SFTPClient() diff --git a/automation_file/remote/sftp/delete_ops.py b/automation_file/remote/sftp/delete_ops.py new file mode 100644 index 0000000..e209e67 --- /dev/null +++ b/automation_file/remote/sftp/delete_ops.py @@ -0,0 +1,18 @@ +"""SFTP delete operations.""" + +from __future__ import annotations + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.sftp.client import sftp_instance + + +def sftp_delete_path(remote_path: str) -> bool: + """Delete a remote file. (Directories require a recursive helper.)""" + sftp = sftp_instance.require_sftp() + try: + sftp.remove(remote_path) + file_automation_logger.info("sftp_delete_path: %s", remote_path) + return True + except OSError as error: + file_automation_logger.error("sftp_delete_path failed: %r", error) + return False diff --git a/automation_file/remote/sftp/download_ops.py b/automation_file/remote/sftp/download_ops.py new file mode 100644 index 0000000..da83d7b --- /dev/null +++ b/automation_file/remote/sftp/download_ops.py @@ -0,0 +1,25 @@ +"""SFTP download operations.""" + +from __future__ import annotations + +from pathlib import Path + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.sftp.client import sftp_instance + + +def sftp_download_file(remote_path: str, target_path: str) -> bool: + """Download ``remote_path`` to ``target_path``.""" + sftp = sftp_instance.require_sftp() + Path(target_path).parent.mkdir(parents=True, exist_ok=True) + try: + sftp.get(remote_path, target_path) + file_automation_logger.info( + "sftp_download_file: %s -> %s", + remote_path, + target_path, + ) + return True + except OSError as error: + file_automation_logger.error("sftp_download_file failed: %r", error) + return False diff --git a/automation_file/remote/sftp/list_ops.py b/automation_file/remote/sftp/list_ops.py new file mode 100644 index 0000000..9e64887 --- /dev/null +++ b/automation_file/remote/sftp/list_ops.py @@ -0,0 +1,22 @@ +"""SFTP listing operations.""" + +from __future__ import annotations + +from automation_file.logging_config import file_automation_logger +from automation_file.remote.sftp.client import sftp_instance + + +def sftp_list_dir(remote_path: str = ".") -> list[str]: + """Return the non-recursive file listing of ``remote_path``.""" + sftp = sftp_instance.require_sftp() + try: + names = sftp.listdir(remote_path) + except OSError as error: + file_automation_logger.error("sftp_list_dir failed: %r", error) + return [] + file_automation_logger.info( + "sftp_list_dir: %s (%d entries)", + remote_path, + len(names), + ) + return list(names) diff --git a/automation_file/remote/sftp/upload_ops.py b/automation_file/remote/sftp/upload_ops.py new file mode 100644 index 0000000..7cee92c --- /dev/null +++ b/automation_file/remote/sftp/upload_ops.py @@ -0,0 +1,59 @@ +"""SFTP upload operations.""" + +from __future__ import annotations + +import posixpath +from pathlib import Path + +from automation_file.exceptions import FileNotExistsException +from automation_file.logging_config import file_automation_logger +from automation_file.remote._upload_tree import walk_and_upload +from automation_file.remote.sftp.client import sftp_instance + + +def _ensure_remote_dir(sftp, remote_dir: str) -> None: + if not remote_dir or remote_dir == "/": + return + parts: list[str] = [] + current = remote_dir + while current not in ("", "/"): + parts.append(current) + current = posixpath.dirname(current) + for part in reversed(parts): + try: + sftp.stat(part) + except FileNotFoundError: + sftp.mkdir(part) + + +def sftp_upload_file(file_path: str, remote_path: str) -> bool: + """Upload ``file_path`` to ``remote_path`` over SFTP.""" + path = Path(file_path) + if not path.is_file(): + raise FileNotExistsException(str(path)) + sftp = sftp_instance.require_sftp() + try: + _ensure_remote_dir(sftp, posixpath.dirname(remote_path)) + sftp.put(str(path), remote_path) + file_automation_logger.info("sftp_upload_file: %s -> %s", path, remote_path) + return True + except OSError as error: + file_automation_logger.error("sftp_upload_file failed: %r", error) + return False + + +def sftp_upload_dir(dir_path: str, remote_prefix: str) -> list[str]: + """Upload every file under ``dir_path`` to ``remote_prefix``.""" + result = walk_and_upload( + dir_path, + remote_prefix, + lambda prefix, rel: f"{prefix}/{rel}" if prefix else rel, + lambda local, remote: sftp_upload_file(str(local), remote), + ) + file_automation_logger.info( + "sftp_upload_dir: %s -> %s (%d files)", + result.source, + result.prefix, + len(result.uploaded), + ) + return result.uploaded diff --git a/automation_file/remote/url_validator.py b/automation_file/remote/url_validator.py new file mode 100644 index 0000000..ca1c21a --- /dev/null +++ b/automation_file/remote/url_validator.py @@ -0,0 +1,60 @@ +"""SSRF guard for outbound HTTP requests. + +``validate_http_url`` rejects non-http(s) schemes, resolves the host, and +rejects private / loopback / link-local / reserved IP ranges. Every remote +function that accepts a user-supplied URL must pass it through here first. +""" + +from __future__ import annotations + +import ipaddress +import socket +from urllib.parse import urlparse + +from automation_file.exceptions import UrlValidationException + +_ALLOWED_SCHEMES = frozenset({"http", "https"}) + + +def _require_host(url: str) -> str: + if not isinstance(url, str) or not url: + raise UrlValidationException("url must be a non-empty string") + parsed = urlparse(url) + if parsed.scheme not in _ALLOWED_SCHEMES: + raise UrlValidationException(f"disallowed scheme: {parsed.scheme!r}") + host = parsed.hostname + if not host: + raise UrlValidationException("url must contain a host") + return host + + +def _resolve_ips(host: str) -> list[str]: + try: + infos = socket.getaddrinfo(host, None) + except socket.gaierror as error: + raise UrlValidationException(f"cannot resolve host: {host}") from error + return [str(info[4][0]) for info in infos] + + +def _is_disallowed_ip(ip_obj: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: + return ( + ip_obj.is_private + or ip_obj.is_loopback + or ip_obj.is_link_local + or ip_obj.is_reserved + or ip_obj.is_multicast + or ip_obj.is_unspecified + ) + + +def validate_http_url(url: str) -> str: + """Return ``url`` if safe; raise :class:`UrlValidationException` otherwise.""" + host = _require_host(url) + for ip_str in _resolve_ips(host): + try: + ip_obj = ipaddress.ip_address(ip_str) + except ValueError as error: + raise UrlValidationException(f"cannot parse resolved ip: {ip_str}") from error + if _is_disallowed_ip(ip_obj): + raise UrlValidationException(f"disallowed ip: {ip_str}") + return url diff --git a/automation_file/local/zip/__init__.py b/automation_file/server/__init__.py similarity index 100% rename from automation_file/local/zip/__init__.py rename to automation_file/server/__init__.py diff --git a/automation_file/server/http_server.py b/automation_file/server/http_server.py new file mode 100644 index 0000000..97e1dfc --- /dev/null +++ b/automation_file/server/http_server.py @@ -0,0 +1,126 @@ +"""HTTP action server (stdlib only). + +Listens for ``POST /actions`` requests whose body is a JSON action list; the +response body is a JSON object mirroring :func:`execute_action`'s return +value. Bound to loopback by default with the same opt-in semantics as +:mod:`tcp_server`. When ``shared_secret`` is supplied clients must send +``Authorization: Bearer `` — useful when placing the server behind a +reverse proxy. +""" + +from __future__ import annotations + +import hmac +import json +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from automation_file.core.action_executor import execute_action +from automation_file.exceptions import TCPAuthException +from automation_file.logging_config import file_automation_logger +from automation_file.server.network_guards import ensure_loopback + +_DEFAULT_HOST = "127.0.0.1" +_DEFAULT_PORT = 9944 +_MAX_CONTENT_BYTES = 1 * 1024 * 1024 + + +class _HTTPActionHandler(BaseHTTPRequestHandler): + """POST /actions -> JSON results.""" + + def log_message( # pylint: disable=arguments-differ + self, format_str: str, *args: object + ) -> None: + file_automation_logger.info("http_server: " + format_str, *args) + + def do_POST(self) -> None: # pylint: disable=invalid-name — BaseHTTPRequestHandler API + if self.path != "/actions": + self._send_json(HTTPStatus.NOT_FOUND, {"error": "not found"}) + return + try: + payload = self._read_payload() + except TCPAuthException as error: + self._send_json(HTTPStatus.UNAUTHORIZED, {"error": str(error)}) + return + except ValueError as error: + self._send_json(HTTPStatus.BAD_REQUEST, {"error": str(error)}) + return + + try: + results = execute_action(payload) + except Exception as error: # pylint: disable=broad-except + file_automation_logger.error("http_server handler: %r", error) + self._send_json(HTTPStatus.INTERNAL_SERVER_ERROR, {"error": repr(error)}) + return + self._send_json(HTTPStatus.OK, results) + + def _read_payload(self) -> list: + secret: str | None = getattr(self.server, "shared_secret", None) + if secret: + header = self.headers.get("Authorization", "") + if not header.startswith("Bearer "): + raise TCPAuthException("missing bearer token") + if not hmac.compare_digest(header[len("Bearer ") :], secret): + raise TCPAuthException("bad shared secret") + + try: + length = int(self.headers.get("Content-Length", "0")) + except ValueError as error: + raise ValueError("invalid Content-Length") from error + if length <= 0: + raise ValueError("empty body") + if length > _MAX_CONTENT_BYTES: + raise ValueError(f"body {length} exceeds cap {_MAX_CONTENT_BYTES}") + + body = self.rfile.read(length) + try: + return json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError(f"bad JSON: {error!r}") from error + + def _send_json(self, status: HTTPStatus, data: object) -> None: + payload = json.dumps(data, default=repr).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + +class HTTPActionServer(ThreadingHTTPServer): + """Threaded HTTP server carrying an optional shared secret.""" + + def __init__( + self, + server_address: tuple[str, int], + handler_class: type = _HTTPActionHandler, + shared_secret: str | None = None, + ) -> None: + super().__init__(server_address, handler_class) + self.shared_secret: str | None = shared_secret + + +def start_http_action_server( + host: str = _DEFAULT_HOST, + port: int = _DEFAULT_PORT, + allow_non_loopback: bool = False, + shared_secret: str | None = None, +) -> HTTPActionServer: + """Start the HTTP action server on a background thread.""" + if not allow_non_loopback: + ensure_loopback(host) + if allow_non_loopback and not shared_secret: + file_automation_logger.warning( + "http_server: non-loopback bind without shared_secret is insecure", + ) + server = HTTPActionServer((host, port), shared_secret=shared_secret) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + file_automation_logger.info( + "http_server: listening on %s:%d (auth=%s)", + host, + port, + "on" if shared_secret else "off", + ) + return server diff --git a/automation_file/server/network_guards.py b/automation_file/server/network_guards.py new file mode 100644 index 0000000..9731ec9 --- /dev/null +++ b/automation_file/server/network_guards.py @@ -0,0 +1,26 @@ +"""Network-binding guards shared by every embedded action server.""" + +from __future__ import annotations + +import ipaddress +import socket + + +def ensure_loopback(host: str) -> None: + """Raise ``ValueError`` if ``host`` resolves to a non-loopback address. + + Every resolved A / AAAA record must be loopback. The explicit error message + names the opt-out flag so callers are reminded that exposing a server + dispatching arbitrary registry commands is equivalent to a remote REPL. + """ + try: + infos = socket.getaddrinfo(host, None) + except socket.gaierror as error: + raise ValueError(f"cannot resolve host: {host}") from error + for info in infos: + ip_obj = ipaddress.ip_address(info[4][0]) + if not ip_obj.is_loopback: + raise ValueError( + f"host {host} resolves to non-loopback {ip_obj}; pass allow_non_loopback=True " + "if exposure is intentional" + ) diff --git a/automation_file/server/tcp_server.py b/automation_file/server/tcp_server.py new file mode 100644 index 0000000..625d8f5 --- /dev/null +++ b/automation_file/server/tcp_server.py @@ -0,0 +1,157 @@ +"""TCP socket server that executes JSON action payloads. + +Binds to localhost by default. Explicitly rejects non-loopback binds unless +``allow_non_loopback`` is True because the server accepts arbitrary action +names from clients and should not be exposed to the network by accident. + +When a ``shared_secret`` is supplied the server requires each connection to +begin with ``AUTH \\n`` before the JSON payload. This is the minimum +bar for exposing the server beyond loopback; use a TLS-terminating proxy for +anything resembling production. +""" + +from __future__ import annotations + +import hmac +import json +import socketserver +import sys +import threading +from typing import Any + +from automation_file.core.action_executor import execute_action +from automation_file.exceptions import TCPAuthException +from automation_file.logging_config import file_automation_logger +from automation_file.server.network_guards import ensure_loopback + +_DEFAULT_HOST = "localhost" +_DEFAULT_PORT = 9943 +_RECV_BYTES = 8192 +_END_MARKER = b"Return_Data_Over_JE\n" +_QUIT_COMMAND = "quit_server" +_AUTH_PREFIX = "AUTH " + + +class _TCPServerHandler(socketserver.StreamRequestHandler): + """One instance per connection; dispatches a single JSON payload.""" + + def handle(self) -> None: + raw = self.request.recv(_RECV_BYTES) + if not raw: + return + try: + command_string = raw.strip().decode("utf-8") + except UnicodeDecodeError as error: + self._send_line(f"decode error: {error!r}") + self._send_bytes(_END_MARKER) + return + + try: + command_string = self._enforce_auth(command_string) + except TCPAuthException as error: + file_automation_logger.warning("tcp_server auth: %r", error) + self._send_line("auth error") + self._send_bytes(_END_MARKER) + return + + file_automation_logger.info("tcp_server: recv %s", command_string) + if command_string == _QUIT_COMMAND: + self.server.close_flag = True # type: ignore[attr-defined] + threading.Thread(target=self.server.shutdown, daemon=True).start() + self._send_line("server shutting down") + return + + try: + payload = json.loads(command_string) + results = execute_action(payload) + for key, value in results.items(): + self._send_line(f"{key} -> {value}") + except json.JSONDecodeError as error: + self._send_line(f"json error: {error!r}") + except Exception as error: # pylint: disable=broad-except + file_automation_logger.error("tcp_server handler: %r", error) + self._send_line(f"execution error: {error!r}") + finally: + self._send_bytes(_END_MARKER) + + def _enforce_auth(self, command_string: str) -> str: + secret: str | None = getattr(self.server, "shared_secret", None) + if not secret: + return command_string + head, _, rest = command_string.partition("\n") + if not head.startswith(_AUTH_PREFIX): + raise TCPAuthException("missing AUTH header") + supplied = head[len(_AUTH_PREFIX) :].strip() + if not hmac.compare_digest(supplied, secret): + raise TCPAuthException("bad shared secret") + if not rest: + raise TCPAuthException("empty payload after AUTH") + return rest + + def _send_line(self, text: str) -> None: + self._send_bytes(text.encode("utf-8") + b"\n") + + def _send_bytes(self, data: bytes) -> None: + try: + self.request.sendall(data) + except OSError as error: + file_automation_logger.error("tcp_server sendall: %r", error) + + +class TCPActionServer(socketserver.ThreadingMixIn, socketserver.TCPServer): + """Threaded TCP server with an explicit close flag.""" + + daemon_threads = True + allow_reuse_address = True + + def __init__( + self, + server_address: tuple[str, int], + request_handler_class: type, + shared_secret: str | None = None, + ) -> None: + super().__init__(server_address, request_handler_class) + self.close_flag: bool = False + self.shared_secret: str | None = shared_secret + + +def start_autocontrol_socket_server( + host: str = _DEFAULT_HOST, + port: int = _DEFAULT_PORT, + allow_non_loopback: bool = False, + shared_secret: str | None = None, +) -> TCPActionServer: + """Start the action-dispatching TCP server on a background thread. + + ``shared_secret`` turns on per-connection authentication: clients must send + ``AUTH \\n`` followed by the JSON payload. Binding to a non-loopback + address without a shared secret is strongly discouraged. + """ + if not allow_non_loopback: + ensure_loopback(host) + if allow_non_loopback and not shared_secret: + file_automation_logger.warning( + "tcp_server: non-loopback bind without shared_secret is insecure", + ) + server = TCPActionServer((host, port), _TCPServerHandler, shared_secret=shared_secret) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + file_automation_logger.info( + "tcp_server: listening on %s:%d (auth=%s)", + host, + port, + "on" if shared_secret else "off", + ) + return server + + +def main(argv: list[str] | None = None) -> Any: + """Entry point for ``python -m automation_file.server.tcp_server``.""" + args = argv if argv is not None else sys.argv[1:] + host = args[0] if len(args) >= 1 else _DEFAULT_HOST + port = int(args[1]) if len(args) >= 2 else _DEFAULT_PORT + return start_autocontrol_socket_server(host=host, port=port) + + +if __name__ == "__main__": + main() diff --git a/automation_file/ui/__init__.py b/automation_file/ui/__init__.py new file mode 100644 index 0000000..33fee27 --- /dev/null +++ b/automation_file/ui/__init__.py @@ -0,0 +1,16 @@ +"""PySide6 GUI for automation_file. + +Exposes every registered ``FA_*`` action through a tabbed main window so users +can drive local file ops, HTTP downloads, Google Drive, S3, Azure Blob, +Dropbox, SFTP, JSON action lists, and the TCP / HTTP action servers without +writing any code. + +The entry point is :func:`launch_ui` (also mirrored as the ``ui`` subcommand +of ``python -m automation_file``). +""" + +from __future__ import annotations + +from automation_file.ui.launcher import launch_ui + +__all__ = ["launch_ui"] diff --git a/automation_file/ui/launcher.py b/automation_file/ui/launcher.py new file mode 100644 index 0000000..0166902 --- /dev/null +++ b/automation_file/ui/launcher.py @@ -0,0 +1,26 @@ +"""GUI launcher. + +Boots a :class:`QApplication` (reusing any existing instance so the window can +be launched from inside an IPython / Spyder REPL) and shows the main window. +""" + +from __future__ import annotations + +import sys +from collections.abc import Sequence + +from automation_file.logging_config import file_automation_logger + + +def launch_ui(argv: Sequence[str] | None = None) -> int: + """Launch the automation_file GUI. Blocks on the Qt event loop.""" + from PySide6.QtWidgets import QApplication + + from automation_file.ui.main_window import MainWindow + + args = list(argv) if argv is not None else sys.argv + app = QApplication.instance() or QApplication(args) + window = MainWindow() + window.show() + file_automation_logger.info("ui: launched main window") + return int(app.exec()) diff --git a/automation_file/ui/log_widget.py b/automation_file/ui/log_widget.py new file mode 100644 index 0000000..0ffa277 --- /dev/null +++ b/automation_file/ui/log_widget.py @@ -0,0 +1,27 @@ +"""Append-only activity log rendered in the main window footer.""" + +from __future__ import annotations + +import time + +from PySide6.QtCore import Signal +from PySide6.QtGui import QTextCursor +from PySide6.QtWidgets import QPlainTextEdit + + +class LogPanel(QPlainTextEdit): + """Read-only text panel that timestamps and appends log lines.""" + + message_appended = Signal(str) + + def __init__(self) -> None: + super().__init__() + self.setReadOnly(True) + self.setMaximumBlockCount(2000) + self.setPlaceholderText("Activity log — run an action to see output here.") + + def append_line(self, message: str) -> None: + stamp = time.strftime("%H:%M:%S") + self.appendPlainText(f"[{stamp}] {message}") + self.moveCursor(QTextCursor.MoveOperation.End) + self.message_appended.emit(message) diff --git a/automation_file/ui/main_window.py b/automation_file/ui/main_window.py new file mode 100644 index 0000000..e54bb4c --- /dev/null +++ b/automation_file/ui/main_window.py @@ -0,0 +1,79 @@ +"""Main window — tabbed interface over every built-in feature.""" + +from __future__ import annotations + +from PySide6.QtCore import Qt, QThreadPool +from PySide6.QtGui import QKeySequence, QShortcut +from PySide6.QtWidgets import QMainWindow, QSplitter, QTabWidget, QVBoxLayout, QWidget + +from automation_file.logging_config import file_automation_logger +from automation_file.ui.log_widget import LogPanel +from automation_file.ui.tabs import ( + HomeTab, + JSONEditorTab, + LocalOpsTab, + ServerTab, + TransferTab, +) + +_WINDOW_TITLE = "automation_file" +_DEFAULT_SIZE = (1100, 780) +_STATUS_DEFAULT = "Ready" + + +class MainWindow(QMainWindow): + """Tab-based control surface for every registered FA_* feature.""" + + def __init__(self) -> None: + super().__init__() + self.setWindowTitle(_WINDOW_TITLE) + self.resize(*_DEFAULT_SIZE) + + self._pool = QThreadPool.globalInstance() + self._log = LogPanel() + self._log.message_appended.connect(self._on_log_message) + + self._tabs = QTabWidget() + self._home_tab = HomeTab(self._log, self._pool) + self._home_tab.navigate_to_tab.connect(self._focus_tab_by_name) + self._tabs.addTab(self._home_tab, "Home") + self._tabs.addTab(LocalOpsTab(self._log, self._pool), "Local") + self._tabs.addTab(TransferTab(self._log, self._pool), "Transfer") + self._tabs.addTab(JSONEditorTab(self._log, self._pool), "JSON actions") + self._server_tab = ServerTab(self._log, self._pool) + self._tabs.addTab(self._server_tab, "Servers") + + splitter = QSplitter() + splitter.setOrientation(Qt.Orientation.Vertical) + splitter.addWidget(self._tabs) + splitter.addWidget(self._log) + splitter.setStretchFactor(0, 4) + splitter.setStretchFactor(1, 1) + + container = QWidget() + layout = QVBoxLayout(container) + layout.setContentsMargins(8, 8, 8, 8) + layout.addWidget(splitter) + self.setCentralWidget(container) + + self._register_shortcuts() + self.statusBar().showMessage(_STATUS_DEFAULT) + file_automation_logger.info("ui: main window constructed") + + def _register_shortcuts(self) -> None: + for index in range(self._tabs.count()): + shortcut = QShortcut(QKeySequence(f"Ctrl+{index + 1}"), self) + shortcut.activated.connect(lambda i=index: self._tabs.setCurrentIndex(i)) + + def _focus_tab_by_name(self, name: str) -> None: + for index in range(self._tabs.count()): + if self._tabs.tabText(index) == name: + self._tabs.setCurrentIndex(index) + return + + def _on_log_message(self, message: str) -> None: + self.statusBar().showMessage(message, 5000) + + def closeEvent(self, event) -> None: # noqa: N802 # pylint: disable=invalid-name — Qt override + self._server_tab.closeEvent(event) + super().closeEvent(event) diff --git a/automation_file/ui/tabs/__init__.py b/automation_file/ui/tabs/__init__.py new file mode 100644 index 0000000..57a85a2 --- /dev/null +++ b/automation_file/ui/tabs/__init__.py @@ -0,0 +1,29 @@ +"""Tab widgets assembled by :class:`automation_file.ui.main_window.MainWindow`.""" + +from __future__ import annotations + +from automation_file.ui.tabs.azure_tab import AzureBlobTab +from automation_file.ui.tabs.drive_tab import GoogleDriveTab +from automation_file.ui.tabs.dropbox_tab import DropboxTab +from automation_file.ui.tabs.home_tab import HomeTab +from automation_file.ui.tabs.http_tab import HTTPDownloadTab +from automation_file.ui.tabs.json_editor_tab import JSONEditorTab +from automation_file.ui.tabs.local_tab import LocalOpsTab +from automation_file.ui.tabs.s3_tab import S3Tab +from automation_file.ui.tabs.server_tab import ServerTab +from automation_file.ui.tabs.sftp_tab import SFTPTab +from automation_file.ui.tabs.transfer_tab import TransferTab + +__all__ = [ + "AzureBlobTab", + "DropboxTab", + "GoogleDriveTab", + "HTTPDownloadTab", + "HomeTab", + "JSONEditorTab", + "LocalOpsTab", + "S3Tab", + "SFTPTab", + "ServerTab", + "TransferTab", +] diff --git a/automation_file/ui/tabs/azure_tab.py b/automation_file/ui/tabs/azure_tab.py new file mode 100644 index 0000000..c5e47be --- /dev/null +++ b/automation_file/ui/tabs/azure_tab.py @@ -0,0 +1,115 @@ +"""Azure Blob Storage tab.""" + +from __future__ import annotations + +from PySide6.QtWidgets import ( + QFormLayout, + QGroupBox, + QLineEdit, + QPushButton, +) + +from automation_file.remote.azure_blob.client import azure_blob_instance +from automation_file.remote.azure_blob.delete_ops import azure_blob_delete_blob +from automation_file.remote.azure_blob.download_ops import azure_blob_download_file +from automation_file.remote.azure_blob.list_ops import azure_blob_list_container +from automation_file.remote.azure_blob.upload_ops import ( + azure_blob_upload_dir, + azure_blob_upload_file, +) +from automation_file.ui.tabs.base import RemoteBackendTab + + +class AzureBlobTab(RemoteBackendTab): + """Form-driven Azure Blob operations.""" + + def _init_group(self) -> QGroupBox: + box = QGroupBox("Client") + form = QFormLayout(box) + self._conn_string = QLineEdit() + self._conn_string.setEchoMode(QLineEdit.EchoMode.Password) + self._account_url = QLineEdit() + form.addRow("Connection string", self._conn_string) + form.addRow("Account URL (fallback)", self._account_url) + btn = QPushButton("Initialise Azure client") + btn.clicked.connect(self._on_init) + form.addRow(btn) + return box + + def _ops_group(self) -> QGroupBox: + box = QGroupBox("Operations") + form = QFormLayout(box) + self._local = QLineEdit() + self._container = QLineEdit() + self._blob = QLineEdit() + form.addRow("Local path", self._local) + form.addRow("Container", self._container) + form.addRow("Blob name / prefix", self._blob) + form.addRow(self.make_button("Upload file", self._on_upload_file)) + form.addRow(self.make_button("Upload dir", self._on_upload_dir)) + form.addRow(self.make_button("Download to local", self._on_download)) + form.addRow(self.make_button("Delete blob", self._on_delete)) + form.addRow(self.make_button("List container", self._on_list)) + return box + + def _on_init(self) -> None: + conn = self._conn_string.text().strip() + account = self._account_url.text().strip() + self.run_action( + azure_blob_instance.later_init, + "azure_blob.later_init", + kwargs={"connection_string": conn or None, "account_url": account or None}, + ) + + def _on_upload_file(self) -> None: + self.run_action( + azure_blob_upload_file, + f"azure_blob_upload_file {self._local.text().strip()}", + kwargs={ + "file_path": self._local.text().strip(), + "container": self._container.text().strip(), + "blob_name": self._blob.text().strip(), + }, + ) + + def _on_upload_dir(self) -> None: + self.run_action( + azure_blob_upload_dir, + f"azure_blob_upload_dir {self._local.text().strip()}", + kwargs={ + "dir_path": self._local.text().strip(), + "container": self._container.text().strip(), + "name_prefix": self._blob.text().strip(), + }, + ) + + def _on_download(self) -> None: + self.run_action( + azure_blob_download_file, + f"azure_blob_download_file {self._blob.text().strip()}", + kwargs={ + "container": self._container.text().strip(), + "blob_name": self._blob.text().strip(), + "target_path": self._local.text().strip(), + }, + ) + + def _on_delete(self) -> None: + self.run_action( + azure_blob_delete_blob, + f"azure_blob_delete_blob {self._blob.text().strip()}", + kwargs={ + "container": self._container.text().strip(), + "blob_name": self._blob.text().strip(), + }, + ) + + def _on_list(self) -> None: + self.run_action( + azure_blob_list_container, + f"azure_blob_list_container {self._container.text().strip()}", + kwargs={ + "container": self._container.text().strip(), + "name_prefix": self._blob.text().strip(), + }, + ) diff --git a/automation_file/ui/tabs/base.py b/automation_file/ui/tabs/base.py new file mode 100644 index 0000000..c9da9a3 --- /dev/null +++ b/automation_file/ui/tabs/base.py @@ -0,0 +1,111 @@ +"""Shared base class for UI tabs. + +Each tab gets a reference to the main window's :class:`LogPanel` plus a +:class:`QThreadPool` so long-running actions stay off the GUI thread. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from PySide6.QtCore import QThreadPool +from PySide6.QtWidgets import ( + QFileDialog, + QGroupBox, + QHBoxLayout, + QLineEdit, + QPushButton, + QVBoxLayout, + QWidget, +) + +from automation_file.ui.log_widget import LogPanel +from automation_file.ui.worker import ActionWorker + + +class BaseTab(QWidget): + """Common helpers for every feature tab.""" + + def __init__(self, log: LogPanel, pool: QThreadPool) -> None: + super().__init__() + self._log = log + self._pool = pool + + @staticmethod + def make_button(label: str, handler: Callable[[], Any]) -> QPushButton: + """Build a ``QPushButton`` wired to ``handler`` — the cloud tab idiom.""" + button = QPushButton(label) + button.clicked.connect(handler) + return button + + def run_action( + self, + target: Callable[..., Any], + label: str, + args: tuple[Any, ...] | None = None, + kwargs: dict[str, Any] | None = None, + ) -> None: + worker = ActionWorker(target, args=args, kwargs=kwargs, label=label) + worker.signals.log.connect(self._log.append_line) + worker.signals.finished.connect( + lambda result: self._log.append_line(f"result: {label} -> {result!r}") + ) + self._pool.start(worker) + + @staticmethod + def path_picker_row( + line_edit: QLineEdit, + button_text: str, + pick: Callable[[QWidget], str | None], + ) -> QHBoxLayout: + row = QHBoxLayout() + row.addWidget(line_edit) + button = QPushButton(button_text) + + def _on_click() -> None: + chosen = pick(line_edit) + if chosen: + line_edit.setText(chosen) + + button.clicked.connect(_on_click) + row.addWidget(button) + return row + + @staticmethod + def pick_existing_file(parent: QWidget) -> str | None: + path, _ = QFileDialog.getOpenFileName(parent, "Select file") + return path or None + + @staticmethod + def pick_save_file(parent: QWidget) -> str | None: + path, _ = QFileDialog.getSaveFileName(parent, "Save as") + return path or None + + @staticmethod + def pick_directory(parent: QWidget) -> str | None: + path = QFileDialog.getExistingDirectory(parent, "Select directory") + return path or None + + +class RemoteBackendTab(BaseTab): + """Shared layout template for cloud/SFTP tabs. + + Subclasses supply ``_init_group`` (credentials / session setup) and + ``_ops_group`` (file transfer actions). The base class stacks both + inside a ``QVBoxLayout`` with a trailing stretch so the groups pin + to the top of the tab. + """ + + def __init__(self, log: LogPanel, pool: QThreadPool) -> None: + super().__init__(log, pool) + root = QVBoxLayout(self) + root.addWidget(self._init_group()) + root.addWidget(self._ops_group()) + root.addStretch() + + def _init_group(self) -> QGroupBox: + raise NotImplementedError + + def _ops_group(self) -> QGroupBox: + raise NotImplementedError diff --git a/automation_file/ui/tabs/drive_tab.py b/automation_file/ui/tabs/drive_tab.py new file mode 100644 index 0000000..3ce1a42 --- /dev/null +++ b/automation_file/ui/tabs/drive_tab.py @@ -0,0 +1,108 @@ +"""Google Drive tab — init credentials, upload, list, delete.""" + +from __future__ import annotations + +from PySide6.QtWidgets import ( + QFormLayout, + QGroupBox, + QLineEdit, + QPushButton, + QVBoxLayout, +) + +from automation_file.remote.google_drive.client import driver_instance +from automation_file.remote.google_drive.delete_ops import drive_delete_file +from automation_file.remote.google_drive.download_ops import drive_download_file +from automation_file.remote.google_drive.search_ops import drive_search_all_file +from automation_file.remote.google_drive.upload_ops import drive_upload_to_drive +from automation_file.ui.tabs.base import BaseTab + + +class GoogleDriveTab(BaseTab): + """Initialise Drive credentials and dispatch a subset of FA_drive_* ops.""" + + def __init__(self, log, pool) -> None: + super().__init__(log, pool) + root = QVBoxLayout(self) + root.addWidget(self._init_group()) + root.addWidget(self._ops_group()) + root.addStretch() + + def _init_group(self) -> QGroupBox: + box = QGroupBox("Credentials") + form = QFormLayout(box) + self._token = QLineEdit() + self._token.setPlaceholderText("token.json") + self._creds = QLineEdit() + self._creds.setPlaceholderText("credentials.json") + form.addRow("Token path", self._token) + form.addRow("Credentials path", self._creds) + init_btn = QPushButton("Initialise Drive client") + init_btn.clicked.connect(self._on_init) + form.addRow(init_btn) + return box + + def _ops_group(self) -> QGroupBox: + box = QGroupBox("Operations") + form = QFormLayout(box) + self._upload_path = QLineEdit() + form.addRow("Upload local file", self._upload_path) + upload_btn = QPushButton("Upload") + upload_btn.clicked.connect(self._on_upload) + form.addRow(upload_btn) + + self._download_id = QLineEdit() + self._download_name = QLineEdit() + form.addRow("Download file_id", self._download_id) + form.addRow("Save as", self._download_name) + download_btn = QPushButton("Download") + download_btn.clicked.connect(self._on_download) + form.addRow(download_btn) + + self._delete_id = QLineEdit() + form.addRow("Delete file_id", self._delete_id) + delete_btn = QPushButton("Delete") + delete_btn.clicked.connect(self._on_delete) + form.addRow(delete_btn) + + list_btn = QPushButton("List all files") + list_btn.clicked.connect(self._on_list) + form.addRow(list_btn) + return box + + def _on_init(self) -> None: + token = self._token.text().strip() + creds = self._creds.text().strip() + self.run_action( + driver_instance.later_init, + "drive.later_init", + kwargs={"token_path": token, "credentials_path": creds}, + ) + + def _on_upload(self) -> None: + path = self._upload_path.text().strip() + self.run_action( + drive_upload_to_drive, + f"drive_upload {path}", + kwargs={"file_path": path}, + ) + + def _on_download(self) -> None: + file_id = self._download_id.text().strip() + name = self._download_name.text().strip() + self.run_action( + drive_download_file, + f"drive_download {file_id}", + kwargs={"file_id": file_id, "file_name": name}, + ) + + def _on_delete(self) -> None: + file_id = self._delete_id.text().strip() + self.run_action( + drive_delete_file, + f"drive_delete {file_id}", + kwargs={"file_id": file_id}, + ) + + def _on_list(self) -> None: + self.run_action(drive_search_all_file, "drive_search_all_file") diff --git a/automation_file/ui/tabs/dropbox_tab.py b/automation_file/ui/tabs/dropbox_tab.py new file mode 100644 index 0000000..1d52a47 --- /dev/null +++ b/automation_file/ui/tabs/dropbox_tab.py @@ -0,0 +1,108 @@ +"""Dropbox tab.""" + +from __future__ import annotations + +from PySide6.QtWidgets import ( + QCheckBox, + QFormLayout, + QGroupBox, + QLineEdit, + QPushButton, +) + +from automation_file.remote.dropbox_api.client import dropbox_instance +from automation_file.remote.dropbox_api.delete_ops import dropbox_delete_path +from automation_file.remote.dropbox_api.download_ops import dropbox_download_file +from automation_file.remote.dropbox_api.list_ops import dropbox_list_folder +from automation_file.remote.dropbox_api.upload_ops import ( + dropbox_upload_dir, + dropbox_upload_file, +) +from automation_file.ui.tabs.base import RemoteBackendTab + + +class DropboxTab(RemoteBackendTab): + """Form-driven Dropbox operations.""" + + def _init_group(self) -> QGroupBox: + box = QGroupBox("Client") + form = QFormLayout(box) + self._token = QLineEdit() + self._token.setEchoMode(QLineEdit.EchoMode.Password) + self._token.setPlaceholderText("OAuth2 access token") + form.addRow("Access token", self._token) + btn = QPushButton("Initialise Dropbox client") + btn.clicked.connect(self._on_init) + form.addRow(btn) + return box + + def _ops_group(self) -> QGroupBox: + box = QGroupBox("Operations") + form = QFormLayout(box) + self._local = QLineEdit() + self._remote = QLineEdit() + self._recursive = QCheckBox("Recursive list") + form.addRow("Local path", self._local) + form.addRow("Remote path", self._remote) + form.addRow(self._recursive) + form.addRow(self.make_button("Upload file", self._on_upload_file)) + form.addRow(self.make_button("Upload dir", self._on_upload_dir)) + form.addRow(self.make_button("Download", self._on_download)) + form.addRow(self.make_button("Delete path", self._on_delete)) + form.addRow(self.make_button("List folder", self._on_list)) + return box + + def _on_init(self) -> None: + token = self._token.text().strip() + self.run_action( + dropbox_instance.later_init, + "dropbox.later_init", + kwargs={"oauth2_access_token": token}, + ) + + def _on_upload_file(self) -> None: + self.run_action( + dropbox_upload_file, + f"dropbox_upload_file {self._local.text().strip()}", + kwargs={ + "file_path": self._local.text().strip(), + "remote_path": self._remote.text().strip(), + }, + ) + + def _on_upload_dir(self) -> None: + self.run_action( + dropbox_upload_dir, + f"dropbox_upload_dir {self._local.text().strip()}", + kwargs={ + "dir_path": self._local.text().strip(), + "remote_prefix": self._remote.text().strip() or "/", + }, + ) + + def _on_download(self) -> None: + self.run_action( + dropbox_download_file, + f"dropbox_download_file {self._remote.text().strip()}", + kwargs={ + "remote_path": self._remote.text().strip(), + "target_path": self._local.text().strip(), + }, + ) + + def _on_delete(self) -> None: + self.run_action( + dropbox_delete_path, + f"dropbox_delete_path {self._remote.text().strip()}", + kwargs={"remote_path": self._remote.text().strip()}, + ) + + def _on_list(self) -> None: + self.run_action( + dropbox_list_folder, + f"dropbox_list_folder {self._remote.text().strip()}", + kwargs={ + "remote_path": self._remote.text().strip(), + "recursive": self._recursive.isChecked(), + }, + ) diff --git a/automation_file/ui/tabs/home_tab.py b/automation_file/ui/tabs/home_tab.py new file mode 100644 index 0000000..7ef332a --- /dev/null +++ b/automation_file/ui/tabs/home_tab.py @@ -0,0 +1,116 @@ +"""Landing dashboard — overview, backend readiness, quick actions.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import NamedTuple + +from PySide6.QtCore import QThreadPool, QTimer, Signal +from PySide6.QtWidgets import ( + QFormLayout, + QGroupBox, + QHBoxLayout, + QLabel, + QVBoxLayout, +) + +from automation_file.remote.azure_blob.client import azure_blob_instance +from automation_file.remote.dropbox_api.client import dropbox_instance +from automation_file.remote.google_drive.client import driver_instance +from automation_file.remote.s3.client import s3_instance +from automation_file.remote.sftp.client import sftp_instance +from automation_file.ui.log_widget import LogPanel +from automation_file.ui.tabs.base import BaseTab + +_REFRESH_INTERVAL_MS = 2000 + + +class _BackendProbe(NamedTuple): + label: str + is_ready: Callable[[], bool] + + +_BACKENDS: tuple[_BackendProbe, ...] = ( + _BackendProbe("Google Drive", lambda: driver_instance.service is not None), + _BackendProbe("Amazon S3", lambda: s3_instance.client is not None), + _BackendProbe("Azure Blob", lambda: azure_blob_instance.service is not None), + _BackendProbe("Dropbox", lambda: dropbox_instance.client is not None), + _BackendProbe("SFTP", lambda: getattr(sftp_instance, "_sftp", None) is not None), +) + + +class HomeTab(BaseTab): + """Dashboard with overview text, backend status, and quick-nav buttons.""" + + navigate_to_tab = Signal(str) + + def __init__(self, log: LogPanel, pool: QThreadPool) -> None: + super().__init__(log, pool) + self._status_labels: dict[str, QLabel] = {} + + root = QVBoxLayout(self) + root.addWidget(self._overview_group()) + row = QHBoxLayout() + row.addWidget(self._status_group(), 1) + row.addWidget(self._actions_group(), 1) + root.addLayout(row) + root.addStretch() + + self._refresh_status() + self._timer = QTimer(self) + self._timer.setInterval(_REFRESH_INTERVAL_MS) + self._timer.timeout.connect(self._refresh_status) + self._timer.start() + + def _overview_group(self) -> QGroupBox: + box = QGroupBox("automation_file") + layout = QVBoxLayout(box) + headline = QLabel( + "Automate local and remote file work through a shared registry of " + "FA_* actions." + ) + headline.setWordWrap(True) + layout.addWidget(headline) + details = QLabel( + "Use Local for direct filesystem / ZIP operations, " + "Transfer to move bytes to cloud backends (HTTP, Drive, S3, " + "Azure, Dropbox, SFTP), and JSON actions for visual editing " + "of reusable action lists. Servers exposes the same registry " + "over localhost TCP or HTTP." + ) + details.setWordWrap(True) + layout.addWidget(details) + return box + + def _status_group(self) -> QGroupBox: + box = QGroupBox("Remote backends") + form = QFormLayout(box) + for probe in _BACKENDS: + label = QLabel("—") + self._status_labels[probe.label] = label + form.addRow(probe.label, label) + return box + + def _actions_group(self) -> QGroupBox: + box = QGroupBox("Jump to…") + layout = QVBoxLayout(box) + for tab_name in ("Local", "Transfer", "JSON actions", "Servers"): + button = self.make_button(tab_name, self._emit_nav(tab_name)) + layout.addWidget(button) + layout.addStretch() + return box + + def _emit_nav(self, tab_name: str) -> Callable[[], None]: + return lambda: self.navigate_to_tab.emit(tab_name) + + def _refresh_status(self) -> None: + for probe in _BACKENDS: + label = self._status_labels.get(probe.label) + if label is None: + continue + try: + ready = bool(probe.is_ready()) + except Exception: # pylint: disable=broad-except + ready = False + label.setText("Ready" if ready else "Not initialised") + label.setStyleSheet("color: #2f8f3f;" if ready else "color: #888;") diff --git a/automation_file/ui/tabs/http_tab.py b/automation_file/ui/tabs/http_tab.py new file mode 100644 index 0000000..233c34d --- /dev/null +++ b/automation_file/ui/tabs/http_tab.py @@ -0,0 +1,37 @@ +"""HTTP download tab (SSRF-validated, retrying).""" + +from __future__ import annotations + +from PySide6.QtWidgets import QFormLayout, QLineEdit, QPushButton, QVBoxLayout + +from automation_file.remote.http_download import download_file +from automation_file.ui.tabs.base import BaseTab + + +class HTTPDownloadTab(BaseTab): + """Trigger :func:`download_file` from a URL + destination form.""" + + def __init__(self, log, pool) -> None: + super().__init__(log, pool) + root = QVBoxLayout(self) + form = QFormLayout() + self._url = QLineEdit() + self._url.setPlaceholderText("https://example.com/file.bin") + self._dest = QLineEdit() + self._dest.setPlaceholderText("local filename") + form.addRow("URL", self._url) + form.addRow("Save as", self._dest) + button = QPushButton("Download") + button.clicked.connect(self._on_download) + form.addRow(button) + root.addLayout(form) + root.addStretch() + + def _on_download(self) -> None: + url = self._url.text().strip() + dest = self._dest.text().strip() + self.run_action( + download_file, + f"download {url}", + kwargs={"file_url": url, "file_name": dest}, + ) diff --git a/automation_file/ui/tabs/json_editor_tab.py b/automation_file/ui/tabs/json_editor_tab.py new file mode 100644 index 0000000..360f184 --- /dev/null +++ b/automation_file/ui/tabs/json_editor_tab.py @@ -0,0 +1,595 @@ +"""Visual JSON action editor. + +Action lists are edited through a list on the left (one row per action) +and a signature-driven form on the right (auto-generated from the +registered callable). The raw JSON is still available via the "Raw +JSON" toggle — the tree and the textarea stay in sync. +""" + +from __future__ import annotations + +import inspect +import json +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from PySide6.QtCore import QSettings, Qt +from PySide6.QtGui import QDragEnterEvent, QDropEvent, QKeySequence, QShortcut +from PySide6.QtWidgets import ( + QCheckBox, + QComboBox, + QDoubleSpinBox, + QFileDialog, + QFormLayout, + QGroupBox, + QHBoxLayout, + QLabel, + QLineEdit, + QListWidget, + QListWidgetItem, + QPlainTextEdit, + QPushButton, + QSpinBox, + QSplitter, + QStackedWidget, + QVBoxLayout, + QWidget, +) + +from automation_file.core.action_executor import ( + execute_action, + execute_action_parallel, + executor, + validate_action, +) +from automation_file.ui.tabs.base import BaseTab + +_PATH_HINT_SUBSTRINGS = ("path", "_dir", "_file", "directory", "filename", "target") +_SECRET_HINT_SUBSTRINGS = ("password", "secret", "token", "credential") +_SETTINGS_ORG = "automation_file" +_SETTINGS_APP = "ui" +_LAST_JSON_DIR_KEY = "json_editor/last_dir" + + +def _is_path_like(name: str) -> bool: + lower = name.lower() + return any(hint in lower for hint in _PATH_HINT_SUBSTRINGS) + + +def _is_secret_like(name: str) -> bool: + lower = name.lower() + return any(hint in lower for hint in _SECRET_HINT_SUBSTRINGS) + + +def _parse_maybe_json(raw: str) -> Any: + stripped = raw.strip() + if not stripped: + return "" + if stripped[0] in "[{" or stripped in ("true", "false", "null"): + try: + return json.loads(stripped) + except json.JSONDecodeError: + return raw + if stripped.lstrip("-").isdigit(): + try: + return int(stripped) + except ValueError: + return raw + return raw + + +class _FieldWidget: + """Bundle the visible widget plus getter/setter for one parameter.""" + + def __init__( + self, + widget: QWidget, + get_value: Callable[[], Any], + set_value: Callable[[Any], None], + ) -> None: + self.widget = widget + self.get_value = get_value + self.set_value = set_value + + +def _build_line_edit(default: Any, secret: bool) -> _FieldWidget: + edit = QLineEdit() + if secret: + edit.setEchoMode(QLineEdit.EchoMode.Password) + if default not in (None, inspect.Parameter.empty): + edit.setText(str(default)) + return _FieldWidget( + edit, + lambda: _parse_maybe_json(edit.text()), + lambda v: edit.setText("" if v is None else str(v)), + ) + + +def _build_path_picker(default: Any, secret: bool) -> _FieldWidget: + field = _build_line_edit(default, secret) + box = QWidget() + row = QHBoxLayout(box) + row.setContentsMargins(0, 0, 0, 0) + row.addWidget(field.widget) + pick = QPushButton("Browse…") + + def _on_click() -> None: + path, _ = QFileDialog.getOpenFileName(box, "Select file") + if path: + field.set_value(path) + + pick.clicked.connect(_on_click) + row.addWidget(pick) + return _FieldWidget(box, field.get_value, field.set_value) + + +def _build_checkbox(default: Any) -> _FieldWidget: + cb = QCheckBox() + cb.setChecked(bool(default) if default not in (None, inspect.Parameter.empty) else False) + return _FieldWidget(cb, cb.isChecked, lambda v: cb.setChecked(bool(v))) + + +def _build_spinbox(default: Any) -> _FieldWidget: + sb = QSpinBox() + sb.setRange(-1_000_000, 1_000_000) + if isinstance(default, int): + sb.setValue(default) + return _FieldWidget(sb, sb.value, lambda v: sb.setValue(int(v) if v is not None else 0)) + + +def _build_double_spinbox(default: Any) -> _FieldWidget: + sb = QDoubleSpinBox() + sb.setRange(-1_000_000.0, 1_000_000.0) + sb.setDecimals(3) + if isinstance(default, (int, float)): + sb.setValue(float(default)) + return _FieldWidget(sb, sb.value, lambda v: sb.setValue(float(v) if v is not None else 0.0)) + + +def _build_field(parameter: inspect.Parameter) -> _FieldWidget: + """Return a ``_FieldWidget`` matched to ``parameter``'s annotation / name.""" + annotation = parameter.annotation + default = parameter.default + if annotation is bool: + return _build_checkbox(default) + if annotation is int: + return _build_spinbox(default) + if annotation is float: + return _build_double_spinbox(default) + secret = _is_secret_like(parameter.name) + if _is_path_like(parameter.name): + return _build_path_picker(default, secret) + return _build_line_edit(default, secret) + + +class _ActionForm(QWidget): + """Auto-generated form for one action's kwargs.""" + + def __init__(self, name: str, callable_: Callable[..., Any]) -> None: + super().__init__() + self._name = name + self._callable = callable_ + self._getters: dict[str, Callable[[], Any]] = {} + self._setters: dict[str, Callable[[Any], None]] = {} + self._required: set[str] = set() + self._raw: QPlainTextEdit | None = None + + layout = QFormLayout(self) + try: + signature = inspect.signature(callable_) + except (TypeError, ValueError): + layout.addRow(QLabel(f"Cannot introspect {name} — edit kwargs as raw JSON below.")) + raw = QPlainTextEdit() + raw.setPlaceholderText('{"key": "value"}') + layout.addRow(raw) + self._raw = raw + return + + for param_name, parameter in signature.parameters.items(): + if param_name == "self" or parameter.kind in ( + inspect.Parameter.VAR_POSITIONAL, + inspect.Parameter.VAR_KEYWORD, + ): + continue + field = _build_field(parameter) + label = param_name + if parameter.default is inspect.Parameter.empty: + label = f"{param_name} *" + self._required.add(param_name) + layout.addRow(label, field.widget) + self._getters[param_name] = field.get_value + self._setters[param_name] = field.set_value + + @property + def action_name(self) -> str: + return self._name + + def to_kwargs(self) -> dict[str, Any]: + if self._raw is not None: + text = self._raw.toPlainText().strip() + if not text: + return {} + try: + data = json.loads(text) + except json.JSONDecodeError: + return {} + return data if isinstance(data, dict) else {} + kwargs: dict[str, Any] = {} + for name, getter in self._getters.items(): + value = getter() + if value == "" and name not in self._required: + continue + kwargs[name] = value + return kwargs + + def load_kwargs(self, kwargs: dict[str, Any]) -> None: + if self._raw is not None: + self._raw.setPlainText(json.dumps(kwargs, indent=2)) + return + for name, value in kwargs.items(): + setter = self._setters.get(name) + if setter is not None: + setter(value) + + +class JSONEditorTab(BaseTab): + """Tree + form editor for action lists, with a raw-JSON fallback.""" + + def __init__(self, log, pool) -> None: + super().__init__(log, pool) + self._actions: list[list[Any]] = [] + self._current_form: _ActionForm | None = None + self._current_form_row: int = -1 + self._suppress_sync = False + self._settings = QSettings(_SETTINGS_ORG, _SETTINGS_APP) + self.setAcceptDrops(True) + + self._action_list = QListWidget() + self._action_list.currentRowChanged.connect(self._on_row_changed) + + self._form_stack = QStackedWidget() + self._empty_label = QLabel("Select or add an action to edit its parameters.") + self._empty_label.setAlignment(Qt.AlignmentFlag.AlignCenter) + self._form_stack.addWidget(self._empty_label) + + self._raw_editor = QPlainTextEdit() + self._raw_editor.setPlaceholderText('[\n ["FA_create_dir", {"dir_path": "build"}]\n]') + self._raw_editor.textChanged.connect(self._on_raw_changed) + + splitter = QSplitter() + splitter.addWidget(self._build_left_pane()) + splitter.addWidget(self._build_right_pane()) + splitter.setStretchFactor(0, 1) + splitter.setStretchFactor(1, 2) + + root = QVBoxLayout(self) + root.addWidget(self._build_toolbar()) + root.addWidget(splitter) + root.addWidget(self._build_run_bar()) + + self._register_shortcuts() + + def _build_toolbar(self) -> QWidget: + toolbar = QWidget() + row = QHBoxLayout(toolbar) + row.setContentsMargins(0, 0, 0, 0) + row.addWidget(self.make_button("Load JSON…", self._on_load)) + row.addWidget(self.make_button("Save JSON…", self._on_save)) + row.addWidget(self.make_button("Clear", self._on_clear)) + row.addStretch() + self._raw_toggle = QCheckBox("Raw JSON") + self._raw_toggle.toggled.connect(self._on_raw_toggled) + row.addWidget(self._raw_toggle) + return toolbar + + def _build_left_pane(self) -> QWidget: + pane = QWidget() + layout = QVBoxLayout(pane) + layout.setContentsMargins(0, 0, 0, 0) + layout.addWidget(QLabel("Actions")) + + self._picker = QComboBox() + self._picker.addItems(sorted(executor.registry.names())) + layout.addWidget(self._picker) + + layout.addWidget(self._action_list) + + row = QHBoxLayout() + row.addWidget(self.make_button("Add", self._on_add)) + row.addWidget(self.make_button("Duplicate", self._on_duplicate)) + row.addWidget(self.make_button("Remove", self._on_remove)) + layout.addLayout(row) + row2 = QHBoxLayout() + row2.addWidget(self.make_button("Up", lambda: self._on_move(-1))) + row2.addWidget(self.make_button("Down", lambda: self._on_move(1))) + layout.addLayout(row2) + return pane + + def _build_right_pane(self) -> QWidget: + pane = QWidget() + layout = QVBoxLayout(pane) + layout.setContentsMargins(0, 0, 0, 0) + layout.addWidget(QLabel("Parameters")) + layout.addWidget(self._form_stack) + layout.addWidget(self._raw_editor) + self._raw_editor.hide() + return pane + + def _build_run_bar(self) -> QWidget: + box = QGroupBox("Run options") + layout = QHBoxLayout(box) + self._validate_first = QCheckBox("validate_first") + self._dry_run = QCheckBox("dry_run") + self._parallel = QCheckBox("parallel") + self._workers = QSpinBox() + self._workers.setRange(1, 32) + self._workers.setValue(4) + self._workers.setPrefix("workers=") + layout.addWidget(self._validate_first) + layout.addWidget(self._dry_run) + layout.addWidget(self._parallel) + layout.addWidget(self._workers) + layout.addStretch() + layout.addWidget(self.make_button("Validate", self._on_validate)) + layout.addWidget(self.make_button("Run", self._on_run)) + return box + + def _on_add(self) -> None: + name = self._picker.currentText() + if not name: + return + self._commit_current_form() + self._actions.append([name, {}]) + self._refresh_list(select=len(self._actions) - 1) + self._sync_raw_from_model() + + def _on_duplicate(self) -> None: + row = self._action_list.currentRow() + if row < 0: + return + self._commit_current_form() + self._actions.insert(row + 1, json.loads(json.dumps(self._actions[row]))) + self._refresh_list(select=row + 1) + self._sync_raw_from_model() + + def _on_remove(self) -> None: + row = self._action_list.currentRow() + if row < 0: + return + del self._actions[row] + self._clear_current_form() + self._refresh_list(select=min(row, len(self._actions) - 1)) + self._sync_raw_from_model() + + def _on_move(self, delta: int) -> None: + row = self._action_list.currentRow() + target = row + delta + if row < 0 or not 0 <= target < len(self._actions): + return + self._commit_current_form() + self._actions[row], self._actions[target] = self._actions[target], self._actions[row] + self._refresh_list(select=target) + self._sync_raw_from_model() + + def _on_row_changed(self, row: int) -> None: + self._commit_current_form() + self._clear_current_form() + if row < 0 or row >= len(self._actions): + self._form_stack.setCurrentWidget(self._empty_label) + return + name, kwargs = self._unpack_action(self._actions[row]) + callable_ = executor.registry.resolve(name) + if callable_ is None: + self._form_stack.setCurrentWidget(self._empty_label) + self._log.append_line(f"unknown action: {name}") + return + form = _ActionForm(name, callable_) + form.load_kwargs(kwargs) + self._form_stack.addWidget(form) + self._form_stack.setCurrentWidget(form) + self._current_form = form + self._current_form_row = row + + def _commit_current_form(self) -> None: + if self._current_form is None: + return + row = self._current_form_row + if row < 0 or row >= len(self._actions): + return + self._actions[row] = [self._current_form.action_name, self._current_form.to_kwargs()] + item = self._action_list.item(row) + if item is not None: + item.setText(self._summary_for(self._actions[row])) + + def _clear_current_form(self) -> None: + if self._current_form is not None: + self._form_stack.removeWidget(self._current_form) + self._current_form.deleteLater() + self._current_form = None + self._current_form_row = -1 + + def _refresh_list(self, select: int | None = None) -> None: + self._action_list.blockSignals(True) + self._action_list.clear() + for action in self._actions: + self._action_list.addItem(QListWidgetItem(self._summary_for(action))) + self._action_list.blockSignals(False) + if select is not None and 0 <= select < len(self._actions): + self._action_list.setCurrentRow(select) + elif not self._actions: + self._on_row_changed(-1) + + def _summary_for(self, action: list[Any]) -> str: + name, kwargs = self._unpack_action(action) + if not kwargs: + return name + preview = ", ".join(f"{k}={v!r}" for k, v in list(kwargs.items())[:2]) + return f"{name}({preview})" + + @staticmethod + def _unpack_action(action: list[Any]) -> tuple[str, dict[str, Any]]: + if not action: + return "", {} + name = str(action[0]) + if len(action) < 2: + return name, {} + payload = action[1] + if isinstance(payload, dict): + return name, payload + return name, {} + + def _on_load(self) -> None: + start_dir = str(self._settings.value(_LAST_JSON_DIR_KEY, "")) + path, _ = QFileDialog.getOpenFileName( + self, "Load action JSON", start_dir, filter="JSON (*.json)" + ) + if not path: + return + self._load_path(path) + + def _load_path(self, path: str) -> None: + try: + with open(path, encoding="utf-8") as fp: + data = json.load(fp) + except (OSError, json.JSONDecodeError) as error: + self._log.append_line(f"load error: {error}") + return + if not isinstance(data, list): + self._log.append_line("load error: top-level JSON must be an array") + return + self._actions = data + self._settings.setValue(_LAST_JSON_DIR_KEY, str(Path(path).parent)) + self._clear_current_form() + self._refresh_list(select=0 if data else None) + self._sync_raw_from_model() + self._log.append_line(f"loaded {len(data)} actions from {path}") + + def _on_save(self) -> None: + self._commit_current_form() + start_dir = str(self._settings.value(_LAST_JSON_DIR_KEY, "")) + path, _ = QFileDialog.getSaveFileName( + self, "Save action JSON", start_dir, filter="JSON (*.json)" + ) + if not path: + return + try: + with open(path, "w", encoding="utf-8") as fp: + json.dump(self._actions, fp, indent=2) + except OSError as error: + self._log.append_line(f"save error: {error}") + return + self._settings.setValue(_LAST_JSON_DIR_KEY, str(Path(path).parent)) + self._log.append_line(f"saved {len(self._actions)} actions to {path}") + + def _on_clear(self) -> None: + self._actions = [] + self._clear_current_form() + self._refresh_list(select=None) + self._sync_raw_from_model() + + def _on_raw_toggled(self, on: bool) -> None: + if on: + self._commit_current_form() + self._sync_raw_from_model() + self._raw_editor.show() + self._form_stack.hide() + else: + self._raw_editor.hide() + self._form_stack.show() + + def _on_raw_changed(self) -> None: + if self._suppress_sync or not self._raw_toggle.isChecked(): + return + text = self._raw_editor.toPlainText().strip() + if not text: + self._actions = [] + self._refresh_list(select=None) + return + try: + data = json.loads(text) + except json.JSONDecodeError: + return + if not isinstance(data, list): + return + self._actions = data + self._refresh_list(select=0 if data else None) + + def _sync_raw_from_model(self) -> None: + self._suppress_sync = True + try: + self._raw_editor.setPlainText(json.dumps(self._actions, indent=2)) + finally: + self._suppress_sync = False + + def _current_actions(self) -> list[list[Any]]: + self._commit_current_form() + return self._actions + + def _on_run(self) -> None: + actions = self._current_actions() + if not actions: + self._log.append_line("no actions to run") + return + if self._parallel.isChecked(): + self.run_action( + execute_action_parallel, + f"execute_action_parallel({len(actions)})", + kwargs={"action_list": actions, "max_workers": int(self._workers.value())}, + ) + return + self.run_action( + execute_action, + f"execute_action({len(actions)})", + kwargs={ + "action_list": actions, + "validate_first": self._validate_first.isChecked(), + "dry_run": self._dry_run.isChecked(), + }, + ) + + def _on_validate(self) -> None: + actions = self._current_actions() + if not actions: + self._log.append_line("no actions to validate") + return + self.run_action( + validate_action, + f"validate_action({len(actions)})", + kwargs={"action_list": actions}, + ) + + def _register_shortcuts(self) -> None: + for keys, handler in ( + ("Ctrl+O", self._on_load), + ("Ctrl+S", self._on_save), + ("Ctrl+R", self._on_run), + ): + shortcut = QShortcut(QKeySequence(keys), self) + shortcut.setContext(Qt.ShortcutContext.WidgetWithChildrenShortcut) + shortcut.activated.connect(handler) + + def dragEnterEvent(self, event: QDragEnterEvent) -> None: # noqa: N802 # pylint: disable=invalid-name — Qt override + if self._is_json_drop(event): + event.acceptProposedAction() + return + event.ignore() + + def dropEvent(self, event: QDropEvent) -> None: # noqa: N802 # pylint: disable=invalid-name — Qt override + if not self._is_json_drop(event): + event.ignore() + return + url = event.mimeData().urls()[0] + self._load_path(url.toLocalFile()) + event.acceptProposedAction() + + @staticmethod + def _is_json_drop(event: QDragEnterEvent | QDropEvent) -> bool: + mime = event.mimeData() + if not mime.hasUrls(): + return False + urls = mime.urls() + if not urls: + return False + local = urls[0].toLocalFile() + return bool(local) and local.lower().endswith(".json") diff --git a/automation_file/ui/tabs/local_tab.py b/automation_file/ui/tabs/local_tab.py new file mode 100644 index 0000000..06ed203 --- /dev/null +++ b/automation_file/ui/tabs/local_tab.py @@ -0,0 +1,198 @@ +"""Local filesystem / ZIP operations tab.""" + +from __future__ import annotations + +from PySide6.QtWidgets import ( + QFormLayout, + QGroupBox, + QLineEdit, + QPushButton, + QTextEdit, + QVBoxLayout, +) + +from automation_file.local.dir_ops import copy_dir, create_dir, remove_dir_tree, rename_dir +from automation_file.local.file_ops import ( + copy_file, + create_file, + remove_file, + rename_file, +) +from automation_file.local.zip_ops import unzip_all, zip_dir, zip_file +from automation_file.ui.tabs.base import BaseTab + + +class LocalOpsTab(BaseTab): + """Form-driven local file, directory, and ZIP operations.""" + + def __init__(self, log, pool) -> None: + super().__init__(log, pool) + root = QVBoxLayout(self) + root.addWidget(self._file_group()) + root.addWidget(self._dir_group()) + root.addWidget(self._zip_group()) + root.addStretch() + + def _file_group(self) -> QGroupBox: + box = QGroupBox("Files") + form = QFormLayout(box) + + self._create_path = QLineEdit() + self._create_content = QTextEdit() + self._create_content.setPlaceholderText("Optional file content") + form.addRow("Path", self._create_path) + form.addRow("Content", self._create_content) + create_btn = QPushButton("Create file") + create_btn.clicked.connect(self._on_create_file) + form.addRow(create_btn) + + self._copy_src = QLineEdit() + self._copy_dst = QLineEdit() + form.addRow("Copy source", self._copy_src) + form.addRow("Copy target", self._copy_dst) + copy_btn = QPushButton("Copy file") + copy_btn.clicked.connect(self._on_copy_file) + form.addRow(copy_btn) + + self._rename_src = QLineEdit() + self._rename_dst = QLineEdit() + form.addRow("Rename source", self._rename_src) + form.addRow("Rename target", self._rename_dst) + rename_btn = QPushButton("Rename file") + rename_btn.clicked.connect(self._on_rename_file) + form.addRow(rename_btn) + + self._remove_path = QLineEdit() + form.addRow("Remove file", self._remove_path) + remove_btn = QPushButton("Delete file") + remove_btn.clicked.connect(self._on_remove_file) + form.addRow(remove_btn) + return box + + def _dir_group(self) -> QGroupBox: + box = QGroupBox("Directories") + form = QFormLayout(box) + self._dir_create = QLineEdit() + form.addRow("Create dir", self._dir_create) + form.addRow(self._button("Create", self._on_create_dir)) + + self._dir_copy_src = QLineEdit() + self._dir_copy_dst = QLineEdit() + form.addRow("Copy source", self._dir_copy_src) + form.addRow("Copy target", self._dir_copy_dst) + form.addRow(self._button("Copy dir", self._on_copy_dir)) + + self._dir_rename_src = QLineEdit() + self._dir_rename_dst = QLineEdit() + form.addRow("Rename source", self._dir_rename_src) + form.addRow("Rename target", self._dir_rename_dst) + form.addRow(self._button("Rename dir", self._on_rename_dir)) + + self._dir_remove = QLineEdit() + form.addRow("Remove tree", self._dir_remove) + form.addRow(self._button("Delete dir tree", self._on_remove_dir)) + return box + + def _zip_group(self) -> QGroupBox: + box = QGroupBox("ZIP") + form = QFormLayout(box) + self._zip_target = QLineEdit() + self._zip_name = QLineEdit() + form.addRow("Path (file or dir)", self._zip_target) + form.addRow("Archive name (no .zip)", self._zip_name) + form.addRow(self._button("Zip file", self._on_zip_file)) + form.addRow(self._button("Zip directory", self._on_zip_dir)) + + self._unzip_archive = QLineEdit() + self._unzip_target = QLineEdit() + form.addRow("Archive", self._unzip_archive) + form.addRow("Extract to", self._unzip_target) + form.addRow(self._button("Unzip all", self._on_unzip_all)) + return box + + @staticmethod + def _button(label: str, handler) -> QPushButton: + button = QPushButton(label) + button.clicked.connect(handler) + return button + + def _on_create_file(self) -> None: + path = self._create_path.text().strip() + content = self._create_content.toPlainText() + self.run_action( + create_file, + f"create_file {path}", + kwargs={"file_path": path, "content": content}, + ) + + def _on_copy_file(self) -> None: + src, dst = self._copy_src.text().strip(), self._copy_dst.text().strip() + self.run_action( + copy_file, + f"copy_file {src} -> {dst}", + kwargs={"file_path": src, "target_path": dst}, + ) + + def _on_rename_file(self) -> None: + src, dst = self._rename_src.text().strip(), self._rename_dst.text().strip() + self.run_action( + rename_file, + f"rename_file {src} -> {dst}", + kwargs={"origin_file_path": src, "target_name": dst}, + ) + + def _on_remove_file(self) -> None: + path = self._remove_path.text().strip() + self.run_action(remove_file, f"remove_file {path}", kwargs={"file_path": path}) + + def _on_create_dir(self) -> None: + path = self._dir_create.text().strip() + self.run_action(create_dir, f"create_dir {path}", kwargs={"dir_path": path}) + + def _on_copy_dir(self) -> None: + src, dst = self._dir_copy_src.text().strip(), self._dir_copy_dst.text().strip() + self.run_action( + copy_dir, + f"copy_dir {src} -> {dst}", + kwargs={"dir_path": src, "target_dir_path": dst}, + ) + + def _on_rename_dir(self) -> None: + src, dst = self._dir_rename_src.text().strip(), self._dir_rename_dst.text().strip() + self.run_action( + rename_dir, + f"rename_dir {src} -> {dst}", + kwargs={"origin_dir_path": src, "target_dir": dst}, + ) + + def _on_remove_dir(self) -> None: + path = self._dir_remove.text().strip() + self.run_action(remove_dir_tree, f"remove_dir_tree {path}", kwargs={"dir_path": path}) + + def _on_zip_file(self) -> None: + path = self._zip_target.text().strip() + name = self._zip_name.text().strip() + archive = name if name.endswith(".zip") else f"{name}.zip" + self.run_action( + zip_file, + f"zip_file {path} -> {archive}", + kwargs={"zip_file_path": archive, "file": path}, + ) + + def _on_zip_dir(self) -> None: + path = self._zip_target.text().strip() + name = self._zip_name.text().strip() + self.run_action( + zip_dir, + f"zip_dir {path} -> {name}.zip", + kwargs={"dir_we_want_to_zip": path, "zip_name": name}, + ) + + def _on_unzip_all(self) -> None: + archive = self._unzip_archive.text().strip() + target = self._unzip_target.text().strip() or None + self.run_action( + unzip_all, + f"unzip_all {archive} -> {target}", + kwargs={"zip_file_path": archive, "extract_path": target}, + ) diff --git a/automation_file/ui/tabs/s3_tab.py b/automation_file/ui/tabs/s3_tab.py new file mode 100644 index 0000000..28c6239 --- /dev/null +++ b/automation_file/ui/tabs/s3_tab.py @@ -0,0 +1,114 @@ +"""Amazon S3 tab — initialise the client, upload, download, delete, list.""" + +from __future__ import annotations + +from PySide6.QtWidgets import ( + QFormLayout, + QGroupBox, + QLineEdit, + QPushButton, +) + +from automation_file.remote.s3.client import s3_instance +from automation_file.remote.s3.delete_ops import s3_delete_object +from automation_file.remote.s3.download_ops import s3_download_file +from automation_file.remote.s3.list_ops import s3_list_bucket +from automation_file.remote.s3.upload_ops import s3_upload_dir, s3_upload_file +from automation_file.ui.tabs.base import RemoteBackendTab + + +class S3Tab(RemoteBackendTab): + """Form-driven S3 operations. Secrets default to the AWS credential chain.""" + + def _init_group(self) -> QGroupBox: + box = QGroupBox("Client (leave blank to use the default AWS chain)") + form = QFormLayout(box) + self._access_key = QLineEdit() + self._secret_key = QLineEdit() + self._secret_key.setEchoMode(QLineEdit.EchoMode.Password) + self._region = QLineEdit() + self._endpoint = QLineEdit() + form.addRow("Access key ID", self._access_key) + form.addRow("Secret access key", self._secret_key) + form.addRow("Region", self._region) + form.addRow("Endpoint URL", self._endpoint) + btn = QPushButton("Initialise S3 client") + btn.clicked.connect(self._on_init) + form.addRow(btn) + return box + + def _ops_group(self) -> QGroupBox: + box = QGroupBox("Operations") + form = QFormLayout(box) + self._local = QLineEdit() + self._bucket = QLineEdit() + self._key = QLineEdit() + form.addRow("Local path", self._local) + form.addRow("Bucket", self._bucket) + form.addRow("Key / prefix", self._key) + + form.addRow(self.make_button("Upload file", self._on_upload_file)) + form.addRow(self.make_button("Upload dir", self._on_upload_dir)) + form.addRow(self.make_button("Download to local", self._on_download)) + form.addRow(self.make_button("Delete object", self._on_delete)) + form.addRow(self.make_button("List bucket", self._on_list)) + return box + + def _on_init(self) -> None: + self.run_action( + s3_instance.later_init, + "s3.later_init", + kwargs={ + "aws_access_key_id": self._access_key.text().strip() or None, + "aws_secret_access_key": self._secret_key.text().strip() or None, + "region_name": self._region.text().strip() or None, + "endpoint_url": self._endpoint.text().strip() or None, + }, + ) + + def _on_upload_file(self) -> None: + self.run_action( + s3_upload_file, + f"s3_upload_file {self._local.text().strip()}", + kwargs={ + "file_path": self._local.text().strip(), + "bucket": self._bucket.text().strip(), + "key": self._key.text().strip(), + }, + ) + + def _on_upload_dir(self) -> None: + self.run_action( + s3_upload_dir, + f"s3_upload_dir {self._local.text().strip()}", + kwargs={ + "dir_path": self._local.text().strip(), + "bucket": self._bucket.text().strip(), + "key_prefix": self._key.text().strip(), + }, + ) + + def _on_download(self) -> None: + self.run_action( + s3_download_file, + f"s3_download_file {self._key.text().strip()}", + kwargs={ + "bucket": self._bucket.text().strip(), + "key": self._key.text().strip(), + "target_path": self._local.text().strip(), + }, + ) + + def _on_delete(self) -> None: + self.run_action( + s3_delete_object, + f"s3_delete_object {self._key.text().strip()}", + kwargs={"bucket": self._bucket.text().strip(), "key": self._key.text().strip()}, + ) + + def _on_list(self) -> None: + self.run_action( + s3_list_bucket, + f"s3_list_bucket {self._bucket.text().strip()}", + kwargs={"bucket": self._bucket.text().strip(), "prefix": self._key.text().strip()}, + ) diff --git a/automation_file/ui/tabs/server_tab.py b/automation_file/ui/tabs/server_tab.py new file mode 100644 index 0000000..6f70884 --- /dev/null +++ b/automation_file/ui/tabs/server_tab.py @@ -0,0 +1,150 @@ +"""Control panel for the embedded TCP and HTTP action servers.""" + +from __future__ import annotations + +from PySide6.QtWidgets import ( + QFormLayout, + QGroupBox, + QLineEdit, + QPushButton, + QSpinBox, + QVBoxLayout, +) + +from automation_file.logging_config import file_automation_logger +from automation_file.server.http_server import HTTPActionServer, start_http_action_server +from automation_file.server.tcp_server import TCPActionServer, start_autocontrol_socket_server +from automation_file.ui.tabs.base import BaseTab + + +class ServerTab(BaseTab): + """Start / stop the embedded TCP and HTTP action servers.""" + + def __init__(self, log, pool) -> None: + super().__init__(log, pool) + self._tcp_server: TCPActionServer | None = None + self._http_server: HTTPActionServer | None = None + root = QVBoxLayout(self) + root.addWidget(self._tcp_group()) + root.addWidget(self._http_group()) + root.addStretch() + + def _tcp_group(self) -> QGroupBox: + box = QGroupBox("TCP action server") + form = QFormLayout(box) + self._tcp_host = QLineEdit("127.0.0.1") + self._tcp_port = QSpinBox() + self._tcp_port.setRange(1, 65535) + self._tcp_port.setValue(9943) + self._tcp_secret = QLineEdit() + self._tcp_secret.setEchoMode(QLineEdit.EchoMode.Password) + self._tcp_secret.setPlaceholderText("optional shared secret") + form.addRow("Host", self._tcp_host) + form.addRow("Port", self._tcp_port) + form.addRow("Shared secret", self._tcp_secret) + + start = QPushButton("Start TCP server") + start.clicked.connect(self._on_start_tcp) + stop = QPushButton("Stop TCP server") + stop.clicked.connect(self._on_stop_tcp) + form.addRow(start) + form.addRow(stop) + return box + + def _http_group(self) -> QGroupBox: + box = QGroupBox("HTTP action server") + form = QFormLayout(box) + self._http_host = QLineEdit("127.0.0.1") + self._http_port = QSpinBox() + self._http_port.setRange(1, 65535) + self._http_port.setValue(9944) + self._http_secret = QLineEdit() + self._http_secret.setEchoMode(QLineEdit.EchoMode.Password) + self._http_secret.setPlaceholderText("optional shared secret") + form.addRow("Host", self._http_host) + form.addRow("Port", self._http_port) + form.addRow("Shared secret", self._http_secret) + + start = QPushButton("Start HTTP server") + start.clicked.connect(self._on_start_http) + stop = QPushButton("Stop HTTP server") + stop.clicked.connect(self._on_stop_http) + form.addRow(start) + form.addRow(stop) + return box + + def _on_start_tcp(self) -> None: + if self._tcp_server is not None: + self._log.append_line("TCP server already running") + return + try: + self._tcp_server = start_autocontrol_socket_server( + host=self._tcp_host.text().strip(), + port=int(self._tcp_port.value()), + shared_secret=self._tcp_secret.text().strip() or None, + ) + except (OSError, ValueError) as error: + self._log.append_line(f"TCP start failed: {error!r}") + return + self._log.append_line( + f"TCP server listening on {self._tcp_host.text().strip()}:{int(self._tcp_port.value())}" + ) + + def _on_stop_tcp(self) -> None: + server = self._tcp_server + if server is None: + self._log.append_line("TCP server not running") + return + self._tcp_server = None + try: + server.shutdown() + server.server_close() + except OSError as error: + self._log.append_line(f"TCP shutdown error: {error!r}") + return + file_automation_logger.info("ui: tcp server stopped") + self._log.append_line("TCP server stopped") + + def _on_start_http(self) -> None: + if self._http_server is not None: + self._log.append_line("HTTP server already running") + return + try: + self._http_server = start_http_action_server( + host=self._http_host.text().strip(), + port=int(self._http_port.value()), + shared_secret=self._http_secret.text().strip() or None, + ) + except (OSError, ValueError) as error: + self._log.append_line(f"HTTP start failed: {error!r}") + return + self._log.append_line( + f"HTTP server listening on {self._http_host.text().strip()}:" + f"{int(self._http_port.value())}" + ) + + def _on_stop_http(self) -> None: + server = self._http_server + if server is None: + self._log.append_line("HTTP server not running") + return + self._http_server = None + try: + server.shutdown() + server.server_close() + except OSError as error: + self._log.append_line(f"HTTP shutdown error: {error!r}") + return + file_automation_logger.info("ui: http server stopped") + self._log.append_line("HTTP server stopped") + + def closeEvent(self, event) -> None: # noqa: N802 # pylint: disable=invalid-name — Qt override + if self._tcp_server is not None: + self._tcp_server.shutdown() + self._tcp_server.server_close() + self._tcp_server = None + if self._http_server is not None: + self._http_server.shutdown() + self._http_server.server_close() + self._http_server = None + super().closeEvent(event) diff --git a/automation_file/ui/tabs/sftp_tab.py b/automation_file/ui/tabs/sftp_tab.py new file mode 100644 index 0000000..a8f1c19 --- /dev/null +++ b/automation_file/ui/tabs/sftp_tab.py @@ -0,0 +1,125 @@ +"""SFTP tab (paramiko with RejectPolicy).""" + +from __future__ import annotations + +from PySide6.QtWidgets import ( + QFormLayout, + QGroupBox, + QLineEdit, + QPushButton, + QSpinBox, +) + +from automation_file.remote.sftp.client import sftp_instance +from automation_file.remote.sftp.delete_ops import sftp_delete_path +from automation_file.remote.sftp.download_ops import sftp_download_file +from automation_file.remote.sftp.list_ops import sftp_list_dir +from automation_file.remote.sftp.upload_ops import sftp_upload_dir, sftp_upload_file +from automation_file.ui.tabs.base import RemoteBackendTab + + +class SFTPTab(RemoteBackendTab): + """Form-driven SFTP operations.""" + + def _init_group(self) -> QGroupBox: + box = QGroupBox("Connection (host keys validated against known_hosts)") + form = QFormLayout(box) + self._host = QLineEdit() + self._port = QSpinBox() + self._port.setRange(1, 65535) + self._port.setValue(22) + self._username = QLineEdit() + self._password = QLineEdit() + self._password.setEchoMode(QLineEdit.EchoMode.Password) + self._key_filename = QLineEdit() + self._known_hosts = QLineEdit() + self._known_hosts.setPlaceholderText("~/.ssh/known_hosts") + form.addRow("Host", self._host) + form.addRow("Port", self._port) + form.addRow("Username", self._username) + form.addRow("Password", self._password) + form.addRow("Key filename", self._key_filename) + form.addRow("known_hosts", self._known_hosts) + + connect_btn = QPushButton("Connect") + connect_btn.clicked.connect(self._on_connect) + close_btn = QPushButton("Close session") + close_btn.clicked.connect(self._on_close) + form.addRow(connect_btn) + form.addRow(close_btn) + return box + + def _ops_group(self) -> QGroupBox: + box = QGroupBox("Operations") + form = QFormLayout(box) + self._local = QLineEdit() + self._remote = QLineEdit() + form.addRow("Local path", self._local) + form.addRow("Remote path", self._remote) + form.addRow(self.make_button("Upload file", self._on_upload_file)) + form.addRow(self.make_button("Upload dir", self._on_upload_dir)) + form.addRow(self.make_button("Download", self._on_download)) + form.addRow(self.make_button("Delete remote path", self._on_delete)) + form.addRow(self.make_button("List remote dir", self._on_list)) + return box + + def _on_connect(self) -> None: + self.run_action( + sftp_instance.later_init, + f"sftp.later_init {self._host.text().strip()}", + kwargs={ + "host": self._host.text().strip(), + "username": self._username.text().strip(), + "password": self._password.text().strip() or None, + "key_filename": self._key_filename.text().strip() or None, + "port": int(self._port.value()), + "known_hosts": self._known_hosts.text().strip() or None, + }, + ) + + def _on_close(self) -> None: + self.run_action(sftp_instance.close, "sftp.close") + + def _on_upload_file(self) -> None: + self.run_action( + sftp_upload_file, + f"sftp_upload_file {self._local.text().strip()}", + kwargs={ + "file_path": self._local.text().strip(), + "remote_path": self._remote.text().strip(), + }, + ) + + def _on_upload_dir(self) -> None: + self.run_action( + sftp_upload_dir, + f"sftp_upload_dir {self._local.text().strip()}", + kwargs={ + "dir_path": self._local.text().strip(), + "remote_prefix": self._remote.text().strip(), + }, + ) + + def _on_download(self) -> None: + self.run_action( + sftp_download_file, + f"sftp_download_file {self._remote.text().strip()}", + kwargs={ + "remote_path": self._remote.text().strip(), + "target_path": self._local.text().strip(), + }, + ) + + def _on_delete(self) -> None: + self.run_action( + sftp_delete_path, + f"sftp_delete_path {self._remote.text().strip()}", + kwargs={"remote_path": self._remote.text().strip()}, + ) + + def _on_list(self) -> None: + self.run_action( + sftp_list_dir, + f"sftp_list_dir {self._remote.text().strip() or '.'}", + kwargs={"remote_path": self._remote.text().strip() or "."}, + ) diff --git a/automation_file/ui/tabs/transfer_tab.py b/automation_file/ui/tabs/transfer_tab.py new file mode 100644 index 0000000..aa52ac5 --- /dev/null +++ b/automation_file/ui/tabs/transfer_tab.py @@ -0,0 +1,81 @@ +"""Unified transfer tab — sidebar of remote backends over one stack. + +Collapses the six remote-backend tabs (HTTP, Google Drive, S3, Azure +Blob, Dropbox, SFTP) into a single tab with a sidebar picker. The +existing per-backend widgets are reused verbatim as the stack pages +so feature parity is preserved. +""" + +from __future__ import annotations + +from typing import NamedTuple + +from PySide6.QtCore import QThreadPool +from PySide6.QtWidgets import ( + QHBoxLayout, + QListWidget, + QListWidgetItem, + QStackedWidget, + QWidget, +) + +from automation_file.ui.log_widget import LogPanel +from automation_file.ui.tabs.azure_tab import AzureBlobTab +from automation_file.ui.tabs.base import BaseTab +from automation_file.ui.tabs.drive_tab import GoogleDriveTab +from automation_file.ui.tabs.dropbox_tab import DropboxTab +from automation_file.ui.tabs.http_tab import HTTPDownloadTab +from automation_file.ui.tabs.s3_tab import S3Tab +from automation_file.ui.tabs.sftp_tab import SFTPTab + + +class _BackendEntry(NamedTuple): + label: str + factory: type[BaseTab] + + +_BACKENDS: tuple[_BackendEntry, ...] = ( + _BackendEntry("HTTP download", HTTPDownloadTab), + _BackendEntry("Google Drive", GoogleDriveTab), + _BackendEntry("Amazon S3", S3Tab), + _BackendEntry("Azure Blob", AzureBlobTab), + _BackendEntry("Dropbox", DropboxTab), + _BackendEntry("SFTP", SFTPTab), +) + + +class TransferTab(BaseTab): + """Sidebar-selectable container for every remote backend.""" + + def __init__(self, log: LogPanel, pool: QThreadPool) -> None: + super().__init__(log, pool) + self._sidebar = QListWidget() + self._sidebar.setFixedWidth(180) + self._stack = QStackedWidget() + for entry in _BACKENDS: + self._sidebar.addItem(QListWidgetItem(entry.label)) + self._stack.addWidget(entry.factory(log, pool)) + self._sidebar.currentRowChanged.connect(self._stack.setCurrentIndex) + self._sidebar.setCurrentRow(0) + + root = QHBoxLayout(self) + root.setContentsMargins(0, 0, 0, 0) + root.addWidget(self._sidebar) + root.addWidget(self._stack, 1) + + def current_backend(self) -> str: + row = self._sidebar.currentRow() + return _BACKENDS[row].label if 0 <= row < len(_BACKENDS) else "" + + def select_backend(self, label: str) -> bool: + for index, entry in enumerate(_BACKENDS): + if entry.label == label: + self._sidebar.setCurrentRow(index) + return True + return False + + def inner_widget(self, label: str) -> QWidget | None: + for index, entry in enumerate(_BACKENDS): + if entry.label == label: + return self._stack.widget(index) + return None diff --git a/automation_file/ui/worker.py b/automation_file/ui/worker.py new file mode 100644 index 0000000..17c3ff6 --- /dev/null +++ b/automation_file/ui/worker.py @@ -0,0 +1,49 @@ +"""Background worker that runs a callable off the UI thread. + +Uses :class:`QThreadPool` so we don't block the event loop when an action +touches the network or disk. The worker emits ``finished(result)`` on success +and ``failed(exception)`` on failure; the ``log(message)`` signal fires before +and after the call so the activity panel stays current. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from PySide6.QtCore import QObject, QRunnable, Signal + + +class _WorkerSignals(QObject): + finished = Signal(object) + failed = Signal(object) + log = Signal(str) + + +class ActionWorker(QRunnable): + """Run ``target(*args, **kwargs)`` on a Qt thread pool worker.""" + + def __init__( + self, + target: Callable[..., Any], + args: tuple[Any, ...] | None = None, + kwargs: dict[str, Any] | None = None, + label: str = "action", + ) -> None: + super().__init__() + self._target = target + self._args = args or () + self._kwargs = kwargs or {} + self._label = label + self.signals = _WorkerSignals() + + def run(self) -> None: + self.signals.log.emit(f"running: {self._label}") + try: + result = self._target(*self._args, **self._kwargs) + except Exception as error: # pylint: disable=broad-exception-caught # worker dispatcher boundary — must surface any failure to the UI + self.signals.log.emit(f"failed: {self._label}: {error!r}") + self.signals.failed.emit(error) + return + self.signals.log.emit(f"done: {self._label}") + self.signals.finished.emit(result) diff --git a/automation_file/utils/callback/__init__.py b/automation_file/utils/callback/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/automation_file/utils/callback/callback_function_executor.py b/automation_file/utils/callback/callback_function_executor.py deleted file mode 100644 index 2e4f257..0000000 --- a/automation_file/utils/callback/callback_function_executor.py +++ /dev/null @@ -1,152 +0,0 @@ -import typing - -# 匯入本地檔案與資料夾處理函式 -# Import local file and directory processing functions -from automation_file.local.dir.dir_process import copy_dir, create_dir, remove_dir_tree -from automation_file.local.file.file_process import ( - copy_file, remove_file, rename_file, - copy_specify_extension_file, copy_all_file_to_dir -) -from automation_file.local.zip.zip_process import ( - zip_dir, zip_file, zip_info, zip_file_info, - set_zip_password, read_zip_file, unzip_file, unzip_all -) - -# 匯入 Google Drive 功能 -# Import Google Drive functions -from automation_file.remote.google_drive.delete.delete_manager import drive_delete_file -from automation_file.remote.google_drive.dir.folder_manager import drive_add_folder -from automation_file.remote.google_drive.download.download_file import ( - drive_download_file, drive_download_file_from_folder -) -from automation_file.remote.google_drive.driver_instance import driver_instance -from automation_file.remote.google_drive.search.search_drive import ( - drive_search_all_file, drive_search_field, drive_search_file_mimetype -) -from automation_file.remote.google_drive.share.share_file import ( - drive_share_file_to_anyone, drive_share_file_to_domain, drive_share_file_to_user -) -from automation_file.remote.google_drive.upload.upload_to_driver import ( - drive_upload_dir_to_folder, drive_upload_to_folder, - drive_upload_dir_to_drive, drive_upload_to_drive -) - -# 匯入例外與日誌工具 -# Import exceptions and logging -from automation_file.utils.exception.exception_tags import get_bad_trigger_function, get_bad_trigger_method -from automation_file.utils.exception.exceptions import CallbackExecutorException -from automation_file.utils.logging.loggin_instance import file_automation_logger - - -class CallbackFunctionExecutor(object): - """ - CallbackFunctionExecutor 負責： - - 管理所有可觸發的函式 (event_dict) - - 執行指定的 trigger function - - 在 trigger function 執行後，呼叫 callback function - """ - - def __init__(self): - # event_dict 對應 trigger_function_name 與實際函式 - # event_dict maps trigger_function_name to actual function - self.event_dict: dict = { - "FA_copy_file": copy_file, - "FA_rename_file": rename_file, - "FA_remove_file": remove_file, - "FA_copy_all_file_to_dir": copy_all_file_to_dir, - "FA_copy_specify_extension_file": copy_specify_extension_file, - "FA_copy_dir": copy_dir, - "FA_create_dir": create_dir, - "FA_remove_dir_tree": remove_dir_tree, - "FA_zip_dir": zip_dir, - "FA_zip_file": zip_file, - "FA_zip_info": zip_info, - "FA_zip_file_info": zip_file_info, - "FA_set_zip_password": set_zip_password, - "FA_unzip_file": unzip_file, - "FA_read_zip_file": read_zip_file, - "FA_unzip_all": unzip_all, - "driver_instance": driver_instance, - "search_all_file": drive_search_all_file, - "search_field": drive_search_field, - "search_file_mimetype": drive_search_file_mimetype, - "upload_dir_to_folder": drive_upload_dir_to_folder, - "upload_to_folder": drive_upload_to_folder, - "upload_dir_to_drive": drive_upload_dir_to_drive, - "upload_to_drive": drive_upload_to_drive, - "add_folder": drive_add_folder, - "share_file_to_anyone": drive_share_file_to_anyone, - "share_file_to_domain": drive_share_file_to_domain, - "share_file_to_user": drive_share_file_to_user, - "delete_file": drive_delete_file, - "download_file": drive_download_file, - "download_file_from_folder": drive_download_file_from_folder - } - - def callback_function( - self, - trigger_function_name: str, - callback_function: typing.Callable, - callback_function_param: typing.Optional[dict] = None, - callback_param_method: str = "kwargs", - **kwargs - ) -> typing.Any: - """ - 執行指定的 trigger function，並在完成後執行 callback function - Execute a trigger function, then run a callback function - - :param trigger_function_name: 要觸發的函式名稱 (必須存在於 event_dict) - Function name to trigger (must exist in event_dict) - :param callback_function: 要執行的 callback function - Callback function to execute - :param callback_function_param: callback function 的參數 (dict) - Parameters for callback function (dict) - :param callback_param_method: callback function 的參數傳遞方式 ("kwargs" 或 "args") - Parameter passing method ("kwargs" or "args") - :param kwargs: trigger function 的參數 - Parameters for trigger function - :return: trigger function 的回傳值 - Return value of trigger function - """ - try: - if trigger_function_name not in self.event_dict.keys(): - raise CallbackExecutorException(get_bad_trigger_function) - - file_automation_logger.info( - f"Callback trigger {trigger_function_name} with param {kwargs}" - ) - - # 執行 trigger function - # Execute trigger function - execute_return_value = self.event_dict.get(trigger_function_name)(**kwargs) - - # 執行 callback function - if callback_function_param is not None: - if callback_param_method not in ["kwargs", "args"]: - raise CallbackExecutorException(get_bad_trigger_method) - - if callback_param_method == "kwargs": - callback_function(**callback_function_param) - file_automation_logger.info( - f"Callback function {callback_function} with param {callback_function_param}" - ) - else: - callback_function(*callback_function_param) - file_automation_logger.info( - f"Callback function {callback_function} with param {callback_function_param}" - ) - else: - callback_function() - file_automation_logger.info(f"Callback function {callback_function}") - - return execute_return_value - - except Exception as error: - file_automation_logger.error( - f"Callback function failed. {repr(error)}" - ) - - -# 建立單例，供其他模組使用 -# Create a singleton instance for other modules to use -callback_executor = CallbackFunctionExecutor() \ No newline at end of file diff --git a/automation_file/utils/exception/__init__.py b/automation_file/utils/exception/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/automation_file/utils/exception/exception_tags.py b/automation_file/utils/exception/exception_tags.py deleted file mode 100644 index 769d7b4..0000000 --- a/automation_file/utils/exception/exception_tags.py +++ /dev/null @@ -1,21 +0,0 @@ -token_is_exist: str = "token file already exists" - -# Callback executor -get_bad_trigger_method: str = "invalid trigger method: only kwargs and args accepted" -get_bad_trigger_function: str = "invalid trigger function: only functions in event_dict accepted" - -# add command -add_command_exception: str = "command value type must be a method or function" - -# executor -executor_list_error: str = "executor received invalid data: list is empty or of wrong type" - -# json tag -cant_execute_action_error: str = "can't execute action" -cant_generate_json_report: str = "can't generate JSON report" -cant_find_json_error: str = "can't find JSON file" -cant_save_json_error: str = "can't save JSON file" -action_is_null_error: str = "JSON action is null" - -# argparse -argparse_get_wrong_data: str = "argparse received invalid data" \ No newline at end of file diff --git a/automation_file/utils/exception/exceptions.py b/automation_file/utils/exception/exceptions.py deleted file mode 100644 index 33c5b60..0000000 --- a/automation_file/utils/exception/exceptions.py +++ /dev/null @@ -1,34 +0,0 @@ -class FileAutomationException(Exception): - pass - - -class FileNotExistsException(FileAutomationException): - pass - - -class DirNotExistsException(FileAutomationException): - pass - - -class ZIPGetWrongFileException(FileAutomationException): - pass - - -class CallbackExecutorException(FileAutomationException): - pass - - -class ExecuteActionException(FileAutomationException): - pass - - -class AddCommandException(FileAutomationException): - pass - - -class JsonActionException(FileAutomationException): - pass - - -class ArgparseException(FileAutomationException): - pass diff --git a/automation_file/utils/executor/__init__.py b/automation_file/utils/executor/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/automation_file/utils/executor/action_executor.py b/automation_file/utils/executor/action_executor.py deleted file mode 100644 index 6d1996b..0000000 --- a/automation_file/utils/executor/action_executor.py +++ /dev/null @@ -1,203 +0,0 @@ -import builtins -import types -from inspect import getmembers, isbuiltin -from typing import Union, Any - -# 匯入本地檔案與資料夾處理函式 -# Import local file and directory processing functions -from automation_file.local.dir.dir_process import copy_dir, create_dir, remove_dir_tree -from automation_file.local.file.file_process import ( - copy_file, remove_file, rename_file, - copy_specify_extension_file, copy_all_file_to_dir, create_file -) -from automation_file.local.zip.zip_process import ( - zip_dir, zip_file, zip_info, zip_file_info, - set_zip_password, read_zip_file, unzip_file, unzip_all -) - -# 匯入 Google Drive 功能 -# Import Google Drive functions -from automation_file.remote.google_drive.delete.delete_manager import drive_delete_file -from automation_file.remote.google_drive.dir.folder_manager import drive_add_folder -from automation_file.remote.google_drive.download.download_file import ( - drive_download_file, drive_download_file_from_folder -) -from automation_file.remote.google_drive.driver_instance import driver_instance -from automation_file.remote.google_drive.search.search_drive import ( - drive_search_all_file, drive_search_field, drive_search_file_mimetype -) -from automation_file.remote.google_drive.share.share_file import ( - drive_share_file_to_anyone, drive_share_file_to_domain, drive_share_file_to_user -) -from automation_file.remote.google_drive.upload.upload_to_driver import ( - drive_upload_dir_to_folder, drive_upload_to_folder, - drive_upload_dir_to_drive, drive_upload_to_drive -) - -# 匯入例外、JSON 工具、日誌工具與套件管理器 -# Import exceptions, JSON utils, logging, and package manager -from automation_file.utils.exception.exception_tags import ( - add_command_exception, executor_list_error, - action_is_null_error, cant_execute_action_error -) -from automation_file.utils.exception.exceptions import ExecuteActionException, AddCommandException -from automation_file.utils.json.json_file import read_action_json -from automation_file.utils.logging.loggin_instance import file_automation_logger -from automation_file.utils.package_manager.package_manager_class import package_manager - - -class Executor(object): - """ - Executor 負責： - - 維護一個 event_dict，將字串名稱對應到實際函式 - - 執行 action list 中的動作 - - 支援從 JSON 檔讀取 action list 並執行 - """ - - def __init__(self): - self.event_dict: dict = { - # File - "FA_create_file": create_file, - "FA_copy_file": copy_file, - "FA_rename_file": rename_file, - "FA_remove_file": remove_file, - # Dir - "FA_copy_all_file_to_dir": copy_all_file_to_dir, - "FA_copy_specify_extension_file": copy_specify_extension_file, - "FA_copy_dir": copy_dir, - "FA_create_dir": create_dir, - "FA_remove_dir_tree": remove_dir_tree, - # Zip - "FA_zip_dir": zip_dir, - "FA_zip_file": zip_file, - "FA_zip_info": zip_info, - "FA_zip_file_info": zip_file_info, - "FA_set_zip_password": set_zip_password, - "FA_unzip_file": unzip_file, - "FA_read_zip_file": read_zip_file, - "FA_unzip_all": unzip_all, - # Drive - "FA_drive_later_init": driver_instance.later_init, - "FA_drive_search_all_file": drive_search_all_file, - "FA_drive_search_field": drive_search_field, - "FA_drive_search_file_mimetype": drive_search_file_mimetype, - "FA_drive_upload_dir_to_folder": drive_upload_dir_to_folder, - "FA_drive_upload_to_folder": drive_upload_to_folder, - "FA_drive_upload_dir_to_drive": drive_upload_dir_to_drive, - "FA_drive_upload_to_drive": drive_upload_to_drive, - "FA_drive_add_folder": drive_add_folder, - "FA_drive_share_file_to_anyone": drive_share_file_to_anyone, - "FA_drive_share_file_to_domain": drive_share_file_to_domain, - "FA_drive_share_file_to_user": drive_share_file_to_user, - "FA_drive_delete_file": drive_delete_file, - "FA_drive_download_file": drive_download_file, - "FA_drive_download_file_from_folder": drive_download_file_from_folder, - # Executor 自身功能 - "FA_execute_action": self.execute_action, - "FA_execute_files": self.execute_files, - "FA_add_package_to_executor": package_manager.add_package_to_executor, - } - - # 將所有 Python 內建函式加入 event_dict - # Add all Python built-in functions into event_dict - for function in getmembers(builtins, isbuiltin): - self.event_dict.update({str(function[0]): function[1]}) - - def _execute_event(self, action: list): - """ - 執行單一 action - Execute a single action - :param action: [函式名稱, 參數] - :return: 函式回傳值 - """ - event = self.event_dict.get(action[0]) - if len(action) == 2: - if isinstance(action[1], dict): - return event(**action[1]) # 使用 kwargs - else: - return event(*action[1]) # 使用 args - elif len(action) == 1: - return event() - else: - raise ExecuteActionException(cant_execute_action_error + " " + str(action)) - - def execute_action(self, action_list: Union[list, dict]) -> dict: - """ - 執行 action list - Execute all actions in action list - :param action_list: list 或 dict (若為 dict，需包含 "auto_control") - :return: 執行紀錄 dict - """ - if isinstance(action_list, dict): - action_list: list = action_list.get("auto_control") - if action_list is None: - raise ExecuteActionException(executor_list_error) - - execute_record_dict = dict() - try: - if len(action_list) == 0 or isinstance(action_list, list) is False: - raise ExecuteActionException(action_is_null_error) - except Exception as error: - file_automation_logger.error( - f"Execute {action_list} failed. {repr(error)}" - ) - - for action in action_list: - try: - event_response = self._execute_event(action) - execute_record = "execute: " + str(action) - file_automation_logger.info(f"Execute {action}") - execute_record_dict.update({execute_record: event_response}) - except Exception as error: - file_automation_logger.error( - f"Execute {action} failed. {repr(error)}" - ) - execute_record = "execute: " + str(action) - execute_record_dict.update({execute_record: repr(error)}) - - # 輸出執行結果 - # Print execution results - for key, value in execute_record_dict.items(): - print(key, flush=True) - print(value, flush=True) - - return execute_record_dict - - def execute_files(self, execute_files_list: list) -> list: - """ - 從 JSON 檔讀取並執行 action list - Execute action lists from JSON files - :param execute_files_list: JSON 檔案路徑清單 - :return: 每個檔案的執行結果 list - """ - execute_detail_list: list = list() - for file in execute_files_list: - execute_detail_list.append(self.execute_action(read_action_json(file))) - return execute_detail_list - - -# 建立單例，供其他模組使用 -executor = Executor() -package_manager.executor = executor - - -def add_command_to_executor(command_dict: dict): - """ - 動態新增指令到 event_dict - Dynamically add commands to event_dict - :param command_dict: dict {command_name: function} - """ - file_automation_logger.info(f"Add command to executor {command_dict}") - for command_name, command in command_dict.items(): - if isinstance(command, (types.MethodType, types.FunctionType)): - executor.event_dict.update({command_name: command}) - else: - raise AddCommandException(add_command_exception) - - -def execute_action(action_list: list) -> dict: - return executor.execute_action(action_list) - - -def execute_files(execute_files_list: list) -> list: - return executor.execute_files(execute_files_list) \ No newline at end of file diff --git a/automation_file/utils/file_discovery.py b/automation_file/utils/file_discovery.py new file mode 100644 index 0000000..57cf007 --- /dev/null +++ b/automation_file/utils/file_discovery.py @@ -0,0 +1,26 @@ +"""Filesystem discovery helpers.""" + +from __future__ import annotations + +from pathlib import Path + +_DEFAULT_EXTENSION = ".json" + + +def get_dir_files_as_list( + dir_path: str | None = None, + default_search_file_extension: str = _DEFAULT_EXTENSION, +) -> list[str]: + """Recursively collect files under ``dir_path`` matching an extension. + + Returns absolute paths. The extension comparison is case-insensitive. + """ + root = Path(dir_path) if dir_path is not None else Path.cwd() + suffix = default_search_file_extension.lower() + if not suffix.startswith("."): + suffix = f".{suffix}" + return [ + str(path.absolute()) + for path in root.rglob("*") + if path.is_file() and path.name.lower().endswith(suffix) + ] diff --git a/automation_file/utils/file_process/__init__.py b/automation_file/utils/file_process/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/automation_file/utils/file_process/get_dir_file_list.py b/automation_file/utils/file_process/get_dir_file_list.py deleted file mode 100644 index c5300ae..0000000 --- a/automation_file/utils/file_process/get_dir_file_list.py +++ /dev/null @@ -1,25 +0,0 @@ -from os import getcwd, walk -from os.path import abspath, join -from typing import List - - -def get_dir_files_as_list( - dir_path: str = getcwd(), - default_search_file_extension: str = ".json") -> List[str]: - """ - 遞迴搜尋資料夾下所有符合副檔名的檔案，並回傳完整路徑清單 - Recursively search for files with a specific extension in a directory and return absolute paths - - :param dir_path: 要搜尋的資料夾路徑 (預設為當前工作目錄) - Directory path to search (default: current working directory) - :param default_search_file_extension: 要搜尋的副檔名 (預設為 ".json") - File extension to search (default: ".json") - :return: 若無符合檔案則回傳空清單，否則回傳檔案完整路徑清單 - [] if no files found, else [file1, file2, ...] - """ - return [ - abspath(join(root, file)) - for root, dirs, files in walk(dir_path) - for file in files - if file.lower().endswith(default_search_file_extension.lower()) - ] \ No newline at end of file diff --git a/automation_file/utils/json/__init__.py b/automation_file/utils/json/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/automation_file/utils/json/json_file.py b/automation_file/utils/json/json_file.py deleted file mode 100644 index 8b4aef6..0000000 --- a/automation_file/utils/json/json_file.py +++ /dev/null @@ -1,66 +0,0 @@ -import json -from pathlib import Path -from threading import Lock - -from automation_file.utils.exception.exception_tags import cant_find_json_error, cant_save_json_error -from automation_file.utils.exception.exceptions import JsonActionException -from automation_file.utils.logging.loggin_instance import file_automation_logger - -# 全域鎖，避免多執行緒同時讀寫 JSON 檔案 -# Global lock to prevent concurrent read/write on JSON files -_lock = Lock() - - -def read_action_json(json_file_path: str) -> list: - """ - 讀取 JSON 檔案並回傳內容 - Read a JSON file and return its content - - :param json_file_path: JSON 檔案路徑 (str) - Path to JSON file (str) - :return: JSON 內容 (list) - JSON content (list) - """ - _lock.acquire() - try: - file_path = Path(json_file_path) - if file_path.exists() and file_path.is_file(): - file_automation_logger.info(f"Read json file {json_file_path}") - with open(json_file_path, encoding="utf-8") as read_file: - return json.load(read_file) - else: - # 若檔案不存在，丟出自訂例外 - # Raise custom exception if file not found - raise JsonActionException(cant_find_json_error) - except JsonActionException: - raise - except Exception as error: - # 捕捉其他例外並轉換成 JsonActionException - # Catch other exceptions and raise JsonActionException - raise JsonActionException(f"{cant_find_json_error}: {repr(error)}") - finally: - _lock.release() - - -def write_action_json(json_save_path: str, action_json: list) -> None: - """ - 將資料寫入 JSON 檔案 - Write data into a JSON file - - :param json_save_path: JSON 檔案儲存路徑 (str) - Path to save JSON file (str) - :param action_json: 要寫入的 JSON 資料 (list) - JSON data to write (list) - :return: None - """ - _lock.acquire() - try: - file_automation_logger.info(f"Write {action_json} as file {json_save_path}") - with open(json_save_path, "w+", encoding="utf-8") as file_to_write: - json.dump(action_json, file_to_write, indent=4, ensure_ascii=False) - except JsonActionException: - raise - except Exception as error: - raise JsonActionException(f"{cant_save_json_error}: {repr(error)}") - finally: - _lock.release() \ No newline at end of file diff --git a/automation_file/utils/logging/__init__.py b/automation_file/utils/logging/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/automation_file/utils/logging/loggin_instance.py b/automation_file/utils/logging/loggin_instance.py deleted file mode 100644 index b82a87f..0000000 --- a/automation_file/utils/logging/loggin_instance.py +++ /dev/null @@ -1,43 +0,0 @@ -import logging - -# 設定 root logger 的等級為 DEBUG -# Set root logger level to DEBUG -logging.root.setLevel(logging.DEBUG) - -# 建立一個專用 logger -# Create a dedicated logger -file_automation_logger = logging.getLogger("File Automation") - -# 設定 log 格式 -# Define log format -formatter = logging.Formatter('%(asctime)s | %(name)s | %(levelname)s | %(message)s') - -# === File handler === -# 將 log 輸出到檔案 FileAutomation.log -# Write logs to file FileAutomation.log -file_handler = logging.FileHandler(filename="FileAutomation.log", mode="w", encoding="utf-8") -file_handler.setFormatter(formatter) -file_automation_logger.addHandler(file_handler) - - -class FileAutomationLoggingHandler(logging.Handler): - """ - 自訂 logging handler，將 log 訊息輸出到標準輸出 (print) - Custom logging handler to redirect logs to stdout (print) - """ - - def __init__(self): - super().__init__() - self.formatter = formatter - self.setLevel(logging.DEBUG) - - def emit(self, record: logging.LogRecord) -> None: - # 將 log 訊息格式化後輸出到 console - # Print formatted log message to console - print(self.format(record)) - - -# === Stream handler === -# 將 log 輸出到 console -# Add custom stream handler to logger -file_automation_logger.addHandler(FileAutomationLoggingHandler()) \ No newline at end of file diff --git a/automation_file/utils/package_manager/__init__.py b/automation_file/utils/package_manager/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/automation_file/utils/package_manager/package_manager_class.py b/automation_file/utils/package_manager/package_manager_class.py deleted file mode 100644 index 1606783..0000000 --- a/automation_file/utils/package_manager/package_manager_class.py +++ /dev/null @@ -1,96 +0,0 @@ -from importlib import import_module -from importlib.util import find_spec -from inspect import getmembers, isfunction, isbuiltin, isclass -from sys import stderr - -from automation_file.utils.logging.loggin_instance import file_automation_logger - - -class PackageManager(object): - """ - PackageManager 負責： - - 檢查套件是否存在並載入 - - 將套件中的函式、內建函式、類別註冊到 executor 或 callback_executor - """ - - def __init__(self): - # 已安裝套件快取，避免重複 import - # Cache for installed packages - self.installed_package_dict = {} - self.executor = None - self.callback_executor = None - - def check_package(self, package: str): - """ - 檢查並載入套件 - Check if a package exists and import it - - :param package: 套件名稱 (str) - :return: 套件模組物件，若不存在則回傳 None - """ - if self.installed_package_dict.get(package, None) is None: - found_spec = find_spec(package) - if found_spec is not None: - try: - installed_package = import_module(found_spec.name) - self.installed_package_dict.update( - {found_spec.name: installed_package} - ) - except ModuleNotFoundError as error: - print(repr(error), file=stderr) - return self.installed_package_dict.get(package, None) - - def add_package_to_executor(self, package): - """ - 將套件的成員加入 executor 的 event_dict - Add package members to executor's event_dict - """ - file_automation_logger.info(f"add_package_to_executor, package: {package}") - self.add_package_to_target(package=package, target=self.executor) - - def add_package_to_callback_executor(self, package): - """ - 將套件的成員加入 callback_executor 的 event_dict - Add package members to callback_executor's event_dict - """ - file_automation_logger.info(f"add_package_to_callback_executor, package: {package}") - self.add_package_to_target(package=package, target=self.callback_executor) - - def get_member(self, package, predicate, target): - """ - 取得套件成員並加入目標 event_dict - Get members of a package and add them to target's event_dict - - :param package: 套件名稱 - :param predicate: 過濾條件 (isfunction, isbuiltin, isclass) - :param target: 目標 executor/callback_executor - """ - installed_package = self.check_package(package) - if installed_package is not None and target is not None: - for member in getmembers(installed_package, predicate): - target.event_dict.update( - {f"{package}_{member[0]}": member[1]} - ) - elif installed_package is None: - print(repr(ModuleNotFoundError(f"Can't find package {package}")), file=stderr) - else: - print(f"Executor error {self.executor}", file=stderr) - - def add_package_to_target(self, package, target): - """ - 將套件的 function、builtin、class 成員加入指定 target - Add functions, builtins, and classes from a package to target - - :param package: 套件名稱 - :param target: 目標 executor/callback_executor - """ - try: - self.get_member(package=package, predicate=isfunction, target=target) - self.get_member(package=package, predicate=isbuiltin, target=target) - self.get_member(package=package, predicate=isclass, target=target) - except Exception as error: - print(repr(error), file=stderr) - - -# 建立單例，供其他模組使用 -package_manager = PackageManager() \ No newline at end of file diff --git a/automation_file/utils/project/__init__.py b/automation_file/utils/project/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/automation_file/utils/project/create_project_structure.py b/automation_file/utils/project/create_project_structure.py deleted file mode 100644 index 95de3c9..0000000 --- a/automation_file/utils/project/create_project_structure.py +++ /dev/null @@ -1,91 +0,0 @@ -from os import getcwd -from pathlib import Path -from threading import Lock - -from automation_file.utils.json.json_file import write_action_json -from automation_file.utils.logging.loggin_instance import file_automation_logger -from automation_file.utils.project.template.template_executor import ( - executor_template_1, executor_template_2, bad_executor_template_1 -) -from automation_file.utils.project.template.template_keyword import ( - template_keyword_1, template_keyword_2, bad_template_1 -) - - -def create_dir(dir_name: str) -> None: - """ - 建立資料夾 (若不存在則自動建立) - Create a directory (auto-create if not exists) - - :param dir_name: 資料夾名稱或路徑 - :return: None - """ - Path(dir_name).mkdir(parents=True, exist_ok=True) - - -def create_template(parent_name: str, project_path: str = None) -> None: - """ - 在專案目錄下建立 keyword JSON 與 executor Python 檔案 - Create keyword JSON files and executor Python files under project directory - - :param parent_name: 專案主資料夾名稱 - :param project_path: 專案路徑 (預設為當前工作目錄) - """ - if project_path is None: - project_path = getcwd() - - keyword_dir_path = Path(f"{project_path}/{parent_name}/keyword") - executor_dir_path = Path(f"{project_path}/{parent_name}/executor") - - lock = Lock() - - # === 建立 keyword JSON 檔案 === - if keyword_dir_path.exists() and keyword_dir_path.is_dir(): - write_action_json(str(keyword_dir_path / "keyword1.json"), template_keyword_1) - write_action_json(str(keyword_dir_path / "keyword2.json"), template_keyword_2) - write_action_json(str(keyword_dir_path / "bad_keyword_1.json"), bad_template_1) - - # === 建立 executor Python 檔案 === - if executor_dir_path.exists() and executor_dir_path.is_dir(): - with lock: - with open(executor_dir_path / "executor_one_file.py", "w+", encoding="utf-8") as file: - file.write( - executor_template_1.replace( - "{temp}", str(keyword_dir_path / "keyword1.json") - ) - ) - with open(executor_dir_path / "executor_bad_file.py", "w+", encoding="utf-8") as file: - file.write( - bad_executor_template_1.replace( - "{temp}", str(keyword_dir_path / "bad_keyword_1.json") - ) - ) - with open(executor_dir_path / "executor_folder.py", "w+", encoding="utf-8") as file: - file.write( - executor_template_2.replace( - "{temp}", str(keyword_dir_path) - ) - ) - - -def create_project_dir(project_path: str = None, parent_name: str = "FileAutomation") -> None: - """ - 建立專案目錄結構 (包含 keyword 與 executor 資料夾)，並生成範例檔案 - Create project directory structure (with keyword and executor folders) and generate template files - - :param project_path: 專案路徑 (預設為當前工作目錄) - :param parent_name: 專案主資料夾名稱 (預設 "FileAutomation") - """ - file_automation_logger.info( - f"create_project_dir, project_path: {project_path}, parent_name: {parent_name}" - ) - - if project_path is None: - project_path = getcwd() - - # 建立 keyword 與 executor 資料夾 - create_dir(f"{project_path}/{parent_name}/keyword") - create_dir(f"{project_path}/{parent_name}/executor") - - # 建立範例檔案 - create_template(parent_name, project_path) \ No newline at end of file diff --git a/automation_file/utils/project/template/__init__.py b/automation_file/utils/project/template/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/automation_file/utils/project/template/template_executor.py b/automation_file/utils/project/template/template_executor.py deleted file mode 100644 index 02feea8..0000000 --- a/automation_file/utils/project/template/template_executor.py +++ /dev/null @@ -1,31 +0,0 @@ -executor_template_1: str = \ - """from file_automation import execute_action, read_action_json - -execute_action( - read_action_json( - r"{temp}" - ) -) -""" - -executor_template_2: str = \ - """from file_automation import execute_files, get_dir_files_as_list - -execute_files( - get_dir_files_as_list( - r"{temp}" - ) -) -""" - -bad_executor_template_1: str = \ - """ -# This example is primarily intended to remind users of the importance of verifying input. -from file_automation import execute_action, read_action_json - -execute_action( - read_action_json( - r"{temp}" - ) -) -""" \ No newline at end of file diff --git a/automation_file/utils/project/template/template_keyword.py b/automation_file/utils/project/template/template_keyword.py deleted file mode 100644 index 08ac8bd..0000000 --- a/automation_file/utils/project/template/template_keyword.py +++ /dev/null @@ -1,15 +0,0 @@ -template_keyword_1: list = [ - ["FA_create_dir", {"dir_path": "test_dir"}], - ["FA_create_file", {"file_path": "test.txt", "content": "test"}] -] - -template_keyword_2: list = [ - ["FA_remove_file", {"file_path": "text.txt"}], - ["FA_remove_dir_tree", {"FA_remove_dir_tree": "test_dir"}] -] - -bad_template_1 = [ - ["FA_add_package_to_executor", ["os"]], - ["os_system", ["python --version"]], - ["os_system", ["python -m pip --version"]], -] diff --git a/automation_file/utils/socket_server/__init__.py b/automation_file/utils/socket_server/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/automation_file/utils/socket_server/file_automation_socket_server.py b/automation_file/utils/socket_server/file_automation_socket_server.py deleted file mode 100644 index f58fb5c..0000000 --- a/automation_file/utils/socket_server/file_automation_socket_server.py +++ /dev/null @@ -1,96 +0,0 @@ -import json -import socketserver -import sys -import threading - -from automation_file.utils.executor.action_executor import execute_action - - -class TCPServerHandler(socketserver.BaseRequestHandler): - """ - TCPServerHandler 負責處理每個 client 的請求 - TCPServerHandler handles each client request - """ - - def handle(self): - # 接收 client 傳來的資料 (最大 8192 bytes) - # Receive data from client - command_string = str(self.request.recv(8192).strip(), encoding="utf-8") - socket = self.request - print("command is: " + command_string, flush=True) - - # 若收到 quit_server 指令，則關閉伺服器 - # Shutdown server if quit_server command received - if command_string == "quit_server": - self.server.shutdown() - self.server.close_flag = True - print("Now quit server", flush=True) - else: - try: - # 將接收到的 JSON 字串轉換為 Python 物件 - # Parse JSON string into Python object - execute_str = json.loads(command_string) - - # 執行對應的動作，並將結果逐一回傳給 client - # Execute actions and send results back to client - for execute_function, execute_return in execute_action(execute_str).items(): - socket.sendto(str(execute_return).encode("utf-8"), self.client_address) - socket.sendto("\n".encode("utf-8"), self.client_address) - - # 傳送結束標記，讓 client 知道資料已傳完 - # Send end marker to indicate data transmission is complete - socket.sendto("Return_Data_Over_JE".encode("utf-8"), self.client_address) - socket.sendto("\n".encode("utf-8"), self.client_address) - - except Exception as error: - # 錯誤處理：將錯誤訊息輸出到 stderr 並回傳給 client - # Error handling: log to stderr and send back to client - print(repr(error), file=sys.stderr) - try: - socket.sendto(str(error).encode("utf-8"), self.client_address) - socket.sendto("\n".encode("utf-8"), self.client_address) - socket.sendto("Return_Data_Over_JE".encode("utf-8"), self.client_address) - socket.sendto("\n".encode("utf-8"), self.client_address) - except Exception as error: - print(repr(error)) - - -class TCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): - """ - 自訂 TCPServer，支援多執行緒處理 - Custom TCPServer with threading support - """ - - def __init__(self, server_address, request_handler_class): - super().__init__(server_address, request_handler_class) - self.close_flag: bool = False - - -def start_autocontrol_socket_server(host: str = "localhost", port: int = 9943): - """ - 啟動自動控制 TCP Socket Server - Start the auto-control TCP Socket Server - - :param host: 主機位址 (預設 localhost) - Host address (default: localhost) - :param port: 監聽埠號 (預設 9943) - Port number (default: 9943) - :return: server instance - """ - # 支援從命令列參數指定 host 與 port - # Support overriding host and port from command line arguments - if len(sys.argv) == 2: - host = sys.argv[1] - elif len(sys.argv) == 3: - host = sys.argv[1] - port = int(sys.argv[2]) - - server = TCPServer((host, port), TCPServerHandler) - - # 使用背景執行緒啟動 server - # Start server in a background thread - server_thread = threading.Thread(target=server.serve_forever) - server_thread.daemon = True - server_thread.start() - - return server \ No newline at end of file diff --git a/dev.toml b/dev.toml index b351cd6..23230ac 100644 --- a/dev.toml +++ b/dev.toml @@ -1,25 +1,29 @@ -# Rename to dev version -# This is dev version +# Dev release metadata — copied to pyproject.toml by the dev publish workflow. [build-system] requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [project] name = "automation_file_dev" -version = "0.0.31" +version = "0.0.33" authors = [ { name = "JE-Chen", email = "zenmailman@gmail.com" }, ] -description = "" +description = "JSON-driven file, Drive, and cloud automation framework (dev channel)." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.10" license = { text = "MIT" } dependencies = [ - "google-api-python-client", - "google-auth-httplib2", - "google-auth-oauthlib", - "requests", - "tqdm" + "google-api-python-client>=2.100.0", + "google-auth-httplib2>=0.2.0", + "google-auth-oauthlib>=1.2.0", + "requests>=2.31.0", + "tqdm>=4.66.0", + "boto3>=1.34.0", + "azure-storage-blob>=12.19.0", + "dropbox>=11.36.2", + "paramiko>=3.4.0", + "PySide6>=6.6.0" ] classifiers = [ "Programming Language :: Python :: 3.10", @@ -30,6 +34,17 @@ classifiers = [ "Operating System :: OS Independent" ] +[project.optional-dependencies] +dev = [ + "pytest>=8.0.0", + "pytest-cov>=5.0.0", + "ruff>=0.6.0", + "mypy>=1.11.0", + "pre-commit>=3.7.0", + "build>=1.2.0", + "twine>=5.1.0" +] + [project.urls] "Homepage" = "https://github.com/JE-Chen/Integration-testing-environment" diff --git a/docs/Makefile b/docs/Makefile index d0c3cbf..01e66b5 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,20 +1,16 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. +# Minimal Sphinx Makefile SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = source -BUILDDIR = build +BUILDDIR = _build + +.PHONY: help html clean -# Put it first so that "make" without argument is like "make help". help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) -.PHONY: help Makefile +html: + @$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) +clean: + @$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) diff --git a/docs/make.bat b/docs/make.bat index 9534b01..45d7073 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -1,27 +1,20 @@ @ECHO OFF - pushd %~dp0 -REM Command file for Sphinx documentation +REM Minimal Sphinx build script for Windows if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set SOURCEDIR=source -set BUILDDIR=build +set BUILDDIR=_build if "%1" == "" goto help %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ + echo.sphinx-build was not found. Install it with: pip install -r requirements.txt exit /b 1 ) diff --git a/docs/requirements.txt b/docs/requirements.txt index 4170c03..540144f 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1 +1,3 @@ -sphinx-rtd-theme \ No newline at end of file +sphinx>=7.0 +sphinx-rtd-theme +myst-parser diff --git a/docs/source/API/api_index.rst b/docs/source/API/api_index.rst deleted file mode 100644 index 1b69000..0000000 --- a/docs/source/API/api_index.rst +++ /dev/null @@ -1,8 +0,0 @@ -FileAutomation API Documentation ----- - -.. toctree:: - :maxdepth: 4 - - local.rst - remote.rst diff --git a/docs/source/API/local.rst b/docs/source/API/local.rst deleted file mode 100644 index 23a5d04..0000000 --- a/docs/source/API/local.rst +++ /dev/null @@ -1,171 +0,0 @@ -Local file api ----- - -.. code-block:: python - - def copy_dir(dir_path: str, target_dir_path: str) -> bool: - """ - Copy dir to target path (path need as dir path) - :param dir_path: which dir do we want to copy (str path) - :param target_dir_path: copy dir to this path - :return: True if success else False - """ - -.. code-block:: python - - def remove_dir_tree(dir_path: str) -> bool: - """ - :param dir_path: which dir do we want to remove (str path) - :return: True if success else False - """ - -.. code-block:: python - - def rename_dir(origin_dir_path, target_dir: str) -> bool: - """ - :param origin_dir_path: which dir do we want to rename (str path) - :param target_dir: target name as str full path - :return: True if success else False - """ - -.. code-block:: python - - def create_dir(dir_path: str) -> None: - """ - :param dir_path: create dir on dir_path - :return: None - """ - -.. code-block:: python - - - def copy_file(file_path: str, target_path: str) -> bool: - """ - :param file_path: which file do we want to copy (str path) - :param target_path: put copy file on target path - :return: True if success else False - """ - -.. code-block:: python - - def copy_specify_extension_file(file_dir_path: str, target_extension: str, target_path: str) -> bool: - """ - :param file_dir_path: which dir do we want to search - :param target_extension: what extension we will search - :param target_path: copy file to target path - :return: True if success else False - """ - -.. code-block:: python - - def copy_all_file_to_dir(dir_path: str, target_dir_path: str) -> bool: - """ - :param dir_path: copy all file on dir - :param target_dir_path: put file to target dir - :return: True if success else False - """ - -.. code-block:: python - - def rename_file(origin_file_path, target_name: str, file_extension=None) -> bool: - """ - :param origin_file_path: which dir do we want to search file - :param target_name: rename file to target name - :param file_extension: Which extension do we search - :return: True if success else False - """ - -.. code-block:: python - - def remove_file(file_path: str) -> None: - """ - :param file_path: which file do we want to remove - :return: None - """ - -.. code-block:: python - - def create_file(file_path: str, content: str) -> None: - """ - :param file_path: create file on path - :param content: what content will write to file - :return: None - """ - -.. code-block:: python - - def zip_dir(dir_we_want_to_zip: str, zip_name: str) -> None: - """ - :param dir_we_want_to_zip: dir str path - :param zip_name: zip file name - :return: None - """ - -.. code-block:: python - - def zip_file(zip_file_path: str, file: [str, List[str]]) -> None: - """ - :param zip_file_path: add file to zip file - :param file: single file path or list of file path (str) to add into zip - :return: None - """ - -.. code-block:: python - - def read_zip_file(zip_file_path: str, file_name: str, password: [str, None] = None) -> bytes: - """ - :param zip_file_path: which zip do we want to read - :param file_name: which file on zip do we want to read - :param password: if zip have password use this password to unzip zip file - :return: - """ - -.. code-block:: python - - def unzip_file( - zip_file_path: str, extract_member, extract_path: [str, None] = None, password: [str, None] = None) -> None: - """ - :param zip_file_path: which zip we want to unzip - :param extract_member: which member we want to unzip - :param extract_path: extract member to path - :param password: if zip have password use this password to unzip zip file - :return: None - """ - -.. code-block:: python - - def unzip_all( - zip_file_path: str, extract_member: [str, None] = None, - extract_path: [str, None] = None, password: [str, None] = None) -> None: - """ - :param zip_file_path: which zip do we want to unzip - :param extract_member: which member do we want to unzip - :param extract_path: extract to path - :param password: if zip have password use this password to unzip zip file - :return: None - """ - -.. code-block:: python - - def zip_info(zip_file_path: str) -> List[ZipInfo]: - """ - :param zip_file_path: read zip file info - :return: List[ZipInfo] - """ - -.. code-block:: python - - def zip_file_info(zip_file_path: str) -> List[str]: - """ - :param zip_file_path: read inside zip file info - :return: List[str] - """ - -.. code-block:: python - - def set_zip_password(zip_file_path: str, password: bytes) -> None: - """ - :param zip_file_path: which zip do we want to set password - :param password: password will be set - :return: None - """ diff --git a/docs/source/API/remote.rst b/docs/source/API/remote.rst deleted file mode 100644 index d102086..0000000 --- a/docs/source/API/remote.rst +++ /dev/null @@ -1,126 +0,0 @@ -Remote file api ----- - -.. code-block:: python - - def drive_delete_file(file_id: str) -> Union[Dict[str, str], None]: - """ - :param file_id: Google Drive file id - :return: Dict[str, str] or None - """ - -.. code-block:: python - - def drive_add_folder(folder_name: str) -> Union[dict, None]: - """ - :param folder_name: folder name will create on Google Drive - :return: dict or None - """ - -.. code-block:: python - - def drive_download_file(file_id: str, file_name: str) -> BytesIO: - """ - :param file_id: file have this id will download - :param file_name: file save on local name - :return: file - """ - -.. code-block:: python - - def drive_download_file_from_folder(folder_name: str) -> Union[dict, None]: - """ - :param folder_name: which folder do we want to download file - :return: dict or None - """ - -.. code-block:: python - - def drive_search_all_file() -> Union[dict, None]: - """ - Search all file on Google Drive - :return: dict or None - """ - -.. code-block:: python - - def drive_search_file_mimetype(mime_type: str) -> Union[dict, None]: - """ - :param mime_type: search all file with mime_type on Google Drive - :return: dict or None - """ - -.. code-block:: python - - def drive_search_field(field_pattern: str) -> Union[dict, None]: - """ - :param field_pattern: what pattern will we use to search - :return: dict or None - """ - -.. code-block:: python - - def drive_share_file_to_user( - file_id: str, user: str, user_role: str = "writer") -> Union[dict, None]: - """ - :param file_id: which file do we want to share - :param user: what user do we want to share - :param user_role: what role do we want to share - :return: dict or None - """ - -.. code-block:: python - - def drive_share_file_to_anyone(file_id: str, share_role: str = "reader") -> Union[dict, None]: - """ - :param file_id: which file do we want to share - :param share_role: what role do we want to share - :return: dict or None - """ - -.. code-block:: python - - def drive_share_file_to_domain( - file_id: str, domain: str, domain_role: str = "reader") -> Union[dict, None]: - """ - :param file_id: which file do we want to share - :param domain: what domain do we want to share - :param domain_role: what role do we want to share - :return: dict or None - """ - -.. code-block:: python - - def drive_upload_to_drive(file_path: str, file_name: str = None) -> Union[dict, None]: - """ - :param file_path: which file do we want to upload - :param file_name: file name on Google Drive - :return: dict or None - """ - -.. code-block:: python - - def drive_upload_to_folder(folder_id: str, file_path: str, file_name: str = None) -> Union[dict, None]: - """ - :param folder_id: which folder do we want to upload file into - :param file_path: which file do we want to upload - :param file_name: file name on Google Drive - :return: dict or None - """ - -.. code-block:: python - - def drive_upload_dir_to_drive(dir_path: str) -> List[Optional[set]]: - """ - :param dir_path: which dir do we want to upload to drive - :return: List[Optional[set]] - """ - -.. code-block:: python - - def drive_upload_dir_to_folder(folder_id: str, dir_path: str) -> List[Optional[set]]: - """ - :param folder_id: which folder do we want to put dir into - :param dir_path: which dir do we want to upload - :return: List[Optional[set]] - """ diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst deleted file mode 100644 index 6e1af6f..0000000 --- a/docs/source/Eng/eng_index.rst +++ /dev/null @@ -1,5 +0,0 @@ -FileAutomation English Documentation ----- - -.. toctree:: - :maxdepth: 4 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst deleted file mode 100644 index 9c4d384..0000000 --- a/docs/source/Zh/zh_index.rst +++ /dev/null @@ -1,5 +0,0 @@ -FileAutomation 繁體中文 文件 ----- - -.. toctree:: - :maxdepth: 4 diff --git a/automation_file/remote/download/__init__.py b/docs/source/_static/.gitkeep similarity index 100% rename from automation_file/remote/download/__init__.py rename to docs/source/_static/.gitkeep diff --git a/automation_file/remote/google_drive/delete/__init__.py b/docs/source/_templates/.gitkeep similarity index 100% rename from automation_file/remote/google_drive/delete/__init__.py rename to docs/source/_templates/.gitkeep diff --git a/docs/source/api/core.rst b/docs/source/api/core.rst new file mode 100644 index 0000000..3d6111f --- /dev/null +++ b/docs/source/api/core.rst @@ -0,0 +1,29 @@ +Core +==== + +.. automodule:: automation_file.core.action_registry + :members: + +.. automodule:: automation_file.core.action_executor + :members: + +.. automodule:: automation_file.core.callback_executor + :members: + +.. automodule:: automation_file.core.package_loader + :members: + +.. automodule:: automation_file.core.json_store + :members: + +.. automodule:: automation_file.core.retry + :members: + +.. automodule:: automation_file.core.quota + :members: + +.. automodule:: automation_file.exceptions + :members: + +.. automodule:: automation_file.logging_config + :members: diff --git a/docs/source/api/index.rst b/docs/source/api/index.rst new file mode 100644 index 0000000..4836f89 --- /dev/null +++ b/docs/source/api/index.rst @@ -0,0 +1,13 @@ +API reference +============= + +.. toctree:: + :maxdepth: 2 + + core + local + remote + server + project + ui + utils diff --git a/docs/source/api/local.rst b/docs/source/api/local.rst new file mode 100644 index 0000000..10c5570 --- /dev/null +++ b/docs/source/api/local.rst @@ -0,0 +1,14 @@ +Local operations +================ + +.. automodule:: automation_file.local.file_ops + :members: + +.. automodule:: automation_file.local.dir_ops + :members: + +.. automodule:: automation_file.local.zip_ops + :members: + +.. automodule:: automation_file.local.safe_paths + :members: diff --git a/docs/source/api/project.rst b/docs/source/api/project.rst new file mode 100644 index 0000000..074f847 --- /dev/null +++ b/docs/source/api/project.rst @@ -0,0 +1,8 @@ +Project scaffolding +=================== + +.. automodule:: automation_file.project.project_builder + :members: + +.. automodule:: automation_file.project.templates + :members: diff --git a/docs/source/api/remote.rst b/docs/source/api/remote.rst new file mode 100644 index 0000000..67e9563 --- /dev/null +++ b/docs/source/api/remote.rst @@ -0,0 +1,117 @@ +Remote operations +================= + +.. automodule:: automation_file.remote.url_validator + :members: + +.. automodule:: automation_file.remote.http_download + :members: + +Google Drive +------------ + +.. automodule:: automation_file.remote.google_drive.client + :members: + +.. automodule:: automation_file.remote.google_drive.delete_ops + :members: + +.. automodule:: automation_file.remote.google_drive.folder_ops + :members: + +.. automodule:: automation_file.remote.google_drive.search_ops + :members: + +.. automodule:: automation_file.remote.google_drive.share_ops + :members: + +.. automodule:: automation_file.remote.google_drive.upload_ops + :members: + +.. automodule:: automation_file.remote.google_drive.download_ops + :members: + +S3 +--- + +Bundled with ``automation_file``; registered automatically by +:func:`automation_file.core.action_registry.build_default_registry`. + +.. automodule:: automation_file.remote.s3.client + :members: + +.. automodule:: automation_file.remote.s3.upload_ops + :members: + +.. automodule:: automation_file.remote.s3.download_ops + :members: + +.. automodule:: automation_file.remote.s3.delete_ops + :members: + +.. automodule:: automation_file.remote.s3.list_ops + :members: + +Azure Blob +---------- + +Bundled with ``automation_file``; registered automatically by +:func:`automation_file.core.action_registry.build_default_registry`. + +.. automodule:: automation_file.remote.azure_blob.client + :members: + +.. automodule:: automation_file.remote.azure_blob.upload_ops + :members: + +.. automodule:: automation_file.remote.azure_blob.download_ops + :members: + +.. automodule:: automation_file.remote.azure_blob.delete_ops + :members: + +.. automodule:: automation_file.remote.azure_blob.list_ops + :members: + +Dropbox +------- + +Bundled with ``automation_file``; registered automatically by +:func:`automation_file.core.action_registry.build_default_registry`. + +.. automodule:: automation_file.remote.dropbox_api.client + :members: + +.. automodule:: automation_file.remote.dropbox_api.upload_ops + :members: + +.. automodule:: automation_file.remote.dropbox_api.download_ops + :members: + +.. automodule:: automation_file.remote.dropbox_api.delete_ops + :members: + +.. automodule:: automation_file.remote.dropbox_api.list_ops + :members: + +SFTP +---- + +Bundled with ``automation_file``; registered automatically by +:func:`automation_file.core.action_registry.build_default_registry`. Uses +:class:`paramiko.RejectPolicy` — unknown hosts are never auto-added. + +.. automodule:: automation_file.remote.sftp.client + :members: + +.. automodule:: automation_file.remote.sftp.upload_ops + :members: + +.. automodule:: automation_file.remote.sftp.download_ops + :members: + +.. automodule:: automation_file.remote.sftp.delete_ops + :members: + +.. automodule:: automation_file.remote.sftp.list_ops + :members: diff --git a/docs/source/api/server.rst b/docs/source/api/server.rst new file mode 100644 index 0000000..affd5b7 --- /dev/null +++ b/docs/source/api/server.rst @@ -0,0 +1,8 @@ +Server +====== + +.. automodule:: automation_file.server.tcp_server + :members: + +.. automodule:: automation_file.server.http_server + :members: diff --git a/docs/source/api/ui.rst b/docs/source/api/ui.rst new file mode 100644 index 0000000..29ec5d1 --- /dev/null +++ b/docs/source/api/ui.rst @@ -0,0 +1,72 @@ +Graphical user interface +======================== + +PySide6 front-end. Importing ``automation_file.ui`` loads Qt eagerly; the +facade ``automation_file.launch_ui`` attribute is lazy (only pulls Qt when +accessed) so non-UI workloads keep their import cost low. + +Launcher +-------- + +.. automodule:: automation_file.ui.launcher + :members: + +Main window +----------- + +.. automodule:: automation_file.ui.main_window + :members: + +Background worker +----------------- + +.. automodule:: automation_file.ui.worker + :members: + +Log panel +--------- + +.. automodule:: automation_file.ui.log_widget + :members: + +Tabs +---- + +.. automodule:: automation_file.ui.tabs + :members: + +.. automodule:: automation_file.ui.tabs.base + :members: + +.. automodule:: automation_file.ui.tabs.home_tab + :members: + +.. automodule:: automation_file.ui.tabs.local_tab + :members: + +.. automodule:: automation_file.ui.tabs.http_tab + :members: + +.. automodule:: automation_file.ui.tabs.drive_tab + :members: + +.. automodule:: automation_file.ui.tabs.s3_tab + :members: + +.. automodule:: automation_file.ui.tabs.azure_tab + :members: + +.. automodule:: automation_file.ui.tabs.dropbox_tab + :members: + +.. automodule:: automation_file.ui.tabs.sftp_tab + :members: + +.. automodule:: automation_file.ui.tabs.transfer_tab + :members: + +.. automodule:: automation_file.ui.tabs.json_editor_tab + :members: + +.. automodule:: automation_file.ui.tabs.server_tab + :members: diff --git a/docs/source/api/utils.rst b/docs/source/api/utils.rst new file mode 100644 index 0000000..7f6f12b --- /dev/null +++ b/docs/source/api/utils.rst @@ -0,0 +1,5 @@ +Utils +===== + +.. automodule:: automation_file.utils.file_discovery + :members: diff --git a/docs/source/architecture.rst b/docs/source/architecture.rst new file mode 100644 index 0000000..3a488b5 --- /dev/null +++ b/docs/source/architecture.rst @@ -0,0 +1,142 @@ +Architecture +============ + +``automation_file`` follows a layered architecture built around five design +patterns: + +**Facade** + :mod:`automation_file` (the top-level ``__init__``) is the only name users + should need to import. Every public function and singleton is re-exported + from there. + +**Registry + Command** + :class:`~automation_file.core.action_registry.ActionRegistry` maps an action + name (a string that appears in a JSON action list) to a Python callable. + An action is a Command object of shape ``[name]``, ``[name, {kwargs}]``, or + ``[name, [args]]``. + +**Template Method** + :class:`~automation_file.core.action_executor.ActionExecutor` defines the + single-action lifecycle: resolve the name, dispatch the call, capture the + return value or exception. The outer iteration template guarantees that one + bad action never aborts the batch unless ``validate_first=True`` is set. + +**Strategy** + Each ``local/*_ops.py``, ``remote/*_ops.py``, and cloud subpackage is a + collection of independent strategy functions. Every backend — local, HTTP, + Google Drive, S3, Azure Blob, Dropbox, SFTP — is auto-registered by + :func:`automation_file.core.action_registry.build_default_registry`. The + ``register__ops(registry)`` helpers stay exported for callers that + assemble custom registries. + +**Singleton (module-level)** + ``executor``, ``callback_executor``, ``package_manager``, ``driver_instance``, + ``s3_instance``, ``azure_blob_instance``, ``dropbox_instance``, and + ``sftp_instance`` are shared instances wired in ``__init__`` so plugins + pick up the same state as the CLI. + +Module layout +------------- + +.. code-block:: text + + automation_file/ + ├── __init__.py # Facade — every public name + ├── __main__.py # CLI with subcommands + ├── exceptions.py # FileAutomationException hierarchy + ├── logging_config.py # file_automation_logger + ├── core/ + │ ├── action_registry.py + │ ├── action_executor.py # serial, parallel, dry-run, validate-first + │ ├── callback_executor.py + │ ├── package_loader.py + │ ├── json_store.py + │ ├── retry.py # @retry_on_transient + │ └── quota.py # Quota(max_bytes, max_seconds) + ├── local/ + │ ├── file_ops.py + │ ├── dir_ops.py + │ ├── zip_ops.py + │ └── safe_paths.py # safe_join + is_within + ├── remote/ + │ ├── url_validator.py # SSRF guard + │ ├── http_download.py # retried HTTP download + │ ├── google_drive/ + │ ├── s3/ # auto-registered in build_default_registry() + │ ├── azure_blob/ # auto-registered in build_default_registry() + │ ├── dropbox_api/ # auto-registered in build_default_registry() + │ └── sftp/ # auto-registered in build_default_registry() + ├── server/ + │ ├── tcp_server.py # loopback-only, optional shared-secret + │ └── http_server.py # POST /actions, Bearer auth + ├── project/ + │ ├── project_builder.py + │ └── templates.py + ├── ui/ # PySide6 GUI + │ ├── launcher.py # launch_ui(argv) + │ ├── main_window.py # 9-tab MainWindow + │ ├── worker.py # ActionWorker (QRunnable) + │ ├── log_widget.py # LogPanel + │ └── tabs/ # one tab per backend + JSON runner + servers + └── utils/ + └── file_discovery.py + +Execution modes +--------------- + +The shared executor supports four orthogonal modes: + +* ``execute_action(actions)`` — default serial execution; each failure is + captured and reported without aborting the batch. +* ``execute_action(actions, validate_first=True)`` — resolve every name + against the registry before running anything. A typo aborts the batch + up-front instead of after half the actions have already run. +* ``execute_action(actions, dry_run=True)`` — parse each action and log what + would be called without invoking the underlying function. +* ``execute_action_parallel(actions, max_workers=4)`` — dispatch actions + concurrently through a thread pool. The caller is responsible for ensuring + the chosen actions are independent. + +Reliability utilities +--------------------- + +* :func:`automation_file.core.retry.retry_on_transient` — decorator that + retries ``ConnectionError`` / ``TimeoutError`` / ``OSError`` with capped + exponential back-off. Used by :func:`automation_file.download_file`. +* :class:`automation_file.core.quota.Quota` — dataclass bundling an optional + ``max_bytes`` size cap and an optional ``max_seconds`` time budget. + +Security boundaries +------------------- + +* **SSRF guard**: every outbound HTTP URL passes through + :func:`automation_file.remote.url_validator.validate_http_url`. +* **Path traversal**: + :func:`automation_file.local.safe_paths.safe_join` resolves user paths under + a caller-specified root and rejects ``..`` escapes, absolute paths outside + the root, and symlinks pointing out of it. +* **TCP / HTTP auth**: both servers accept an optional ``shared_secret``. + When set, the TCP server requires ``AUTH \\n`` before the payload + and the HTTP server requires ``Authorization: Bearer ``. Both bind + to loopback by default and refuse non-loopback binds unless + ``allow_non_loopback=True`` is passed. +* **SFTP host verification**: the SFTP client uses + :class:`paramiko.RejectPolicy` and never auto-adds unknown host keys. +* **Plugin loading**: :class:`automation_file.core.package_loader.PackageLoader` + registers arbitrary module members; never expose it to untrusted input. + +Shared singletons +----------------- + +``automation_file/__init__.py`` creates the following process-wide singletons: + +* ``executor`` — :class:`ActionExecutor` used by :func:`execute_action`. +* ``callback_executor`` — :class:`CallbackExecutor` bound to ``executor.registry``. +* ``package_manager`` — :class:`PackageLoader` bound to the same registry. +* ``driver_instance``, ``s3_instance``, ``azure_blob_instance``, + ``dropbox_instance``, ``sftp_instance`` — lazy clients for each cloud + backend. + +All executors share one :class:`ActionRegistry` instance, so calling +:func:`add_command_to_executor` (or any ``register_*_ops`` helper) makes the +new command visible to every dispatcher at once. diff --git a/docs/source/conf.py b/docs/source/conf.py index b313864..edee8d7 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,49 +1,52 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys - -sys.path.insert(0, os.path.abspath('.')) - -# -- Project information ----------------------------------------------------- - -project = 'FileAutomation' -copyright = '2020 ~ Now, JE-Chen' -author = 'JE-Chen' +"""Sphinx configuration for automation_file.""" -# -- General configuration --------------------------------------------------- +# pylint: disable=invalid-name # Sphinx requires these specific lowercase names. -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [] +from __future__ import annotations -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'sphinx_rtd_theme' +import os +import sys -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +sys.path.insert(0, os.path.abspath("../..")) + +project = "automation_file" +author = "JE-Chen" +copyright = "2026, JE-Chen" # pylint: disable=redefined-builtin # Sphinx requires this name +release = "0.0.32" + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinx.ext.intersphinx", + "myst_parser", +] + +templates_path = ["_templates"] +exclude_patterns: list[str] = [] +source_suffix = {".rst": "restructuredtext", ".md": "markdown"} + +html_theme = "sphinx_rtd_theme" +html_static_path = ["_static"] + +autodoc_default_options = { + "members": True, + "undoc-members": True, + "show-inheritance": True, +} +autodoc_typehints = "description" +autodoc_mock_imports = [ + "google", + "googleapiclient", + "google_auth_oauthlib", + "requests", + "tqdm", + "boto3", + "azure", + "dropbox", + "paramiko", +] + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), +} diff --git a/docs/source/index.rst b/docs/source/index.rst index 6fac661..169ca36 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,19 +1,44 @@ -FileAutomation ----- +automation_file +=============== -.. toctree:: - :maxdepth: 4 +Automation-first Python library for local file / directory / zip operations, +HTTP downloads, and remote storage (Google Drive, S3, Azure Blob, Dropbox, +SFTP). Ships with a PySide6 GUI that surfaces every feature through tabs. +Actions are defined as JSON and dispatched through a central +:class:`~automation_file.core.action_registry.ActionRegistry`. + +Getting started +--------------- + +Install from PyPI and run a JSON action list: + +.. code-block:: bash - API/api_index.rst + pip install automation_file + python -m automation_file --execute_file my_actions.json +Or drive the library directly from Python: ----- +.. code-block:: python -RoadMap + from automation_file import execute_action + + execute_action([ + ["FA_create_dir", {"dir_path": "build"}], + ["FA_create_file", {"file_path": "build/hello.txt", "content": "hi"}], + ]) + +.. toctree:: + :maxdepth: 2 + :caption: Contents ----- + architecture + usage + api/index -* Project Kanban -* https://github.com/orgs/Integration-Automation/projects/2/views/1 +Indices +------- ----- +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/usage.rst b/docs/source/usage.rst new file mode 100644 index 0000000..1d38b1c --- /dev/null +++ b/docs/source/usage.rst @@ -0,0 +1,228 @@ +Usage +===== + +JSON action lists +----------------- + +An action is one of three shapes: + +.. code-block:: json + + ["FA_name"] + ["FA_name", {"kwarg": "value"}] + ["FA_name", ["positional", "args"]] + +An action list is an array of actions. The executor runs them in order and +returns a mapping of ``"execute: " -> result | repr(error)``. + +.. code-block:: python + + from automation_file import execute_action, read_action_json + + results = execute_action([ + ["FA_create_dir", {"dir_path": "build"}], + ["FA_create_file", {"file_path": "build/hello.txt", "content": "hi"}], + ["FA_zip_dir", {"dir_we_want_to_zip": "build", "zip_name": "build_snapshot"}], + ]) + + # Or load from a file: + results = execute_action(read_action_json("actions.json")) + +Validation, dry-run, parallel +----------------------------- + +.. code-block:: python + + from automation_file import ( + execute_action, execute_action_parallel, validate_action, + ) + + # Fail-fast validation: aborts before any action runs if any name is unknown. + execute_action(actions, validate_first=True) + + # Dry-run: log what would be called without invoking commands. + execute_action(actions, dry_run=True) + + # Parallel: run independent actions through a thread pool. + execute_action_parallel(actions, max_workers=4) + + # Manual validation — returns the list of resolved names. + names = validate_action(actions) + +CLI +--- + +Legacy flags for running JSON action lists:: + + python -m automation_file --execute_file actions.json + python -m automation_file --execute_dir ./actions/ + python -m automation_file --execute_str '[["FA_create_dir",{"dir_path":"x"}]]' + python -m automation_file --create_project ./my_project + +Subcommands for one-shot operations:: + + python -m automation_file ui + python -m automation_file zip ./src out.zip --dir + python -m automation_file unzip out.zip ./restored + python -m automation_file download https://example.com/file.bin file.bin + python -m automation_file create-file hello.txt --content "hi" + python -m automation_file server --host 127.0.0.1 --port 9943 + python -m automation_file http-server --host 127.0.0.1 --port 9944 + python -m automation_file drive-upload my.txt --token token.json --credentials creds.json + +Google Drive +------------ + +.. code-block:: python + + from automation_file import driver_instance, drive_upload_to_drive + + driver_instance.later_init("token.json", "credentials.json") + drive_upload_to_drive("example.txt") + +TCP action server +----------------- + +.. code-block:: python + + from automation_file import start_autocontrol_socket_server + + server = start_autocontrol_socket_server( + host="localhost", port=9943, shared_secret="optional-secret", + ) + # later: + server.shutdown() + server.server_close() + +When ``shared_secret`` is supplied, the client must prefix each payload with +``AUTH \\n`` before the JSON action list. The server still binds to +loopback by default and refuses non-loopback binds unless +``allow_non_loopback=True`` is passed. + +HTTP action server +------------------ + +.. code-block:: python + + from automation_file import start_http_action_server + + server = start_http_action_server( + host="127.0.0.1", port=9944, shared_secret="optional-secret", + ) + + # Client side: + # curl -H 'Authorization: Bearer optional-secret' \ + # -d '[["FA_create_dir",{"dir_path":"x"}]]' \ + # http://127.0.0.1:9944/actions + +HTTP responses are JSON. When ``shared_secret`` is set the client must send +``Authorization: Bearer ``. + +Reliability +----------- + +Apply retries to your own callables: + +.. code-block:: python + + from automation_file import retry_on_transient + + @retry_on_transient(max_attempts=5, backoff_base=0.5) + def flaky_network_call(): ... + +Enforce per-action limits: + +.. code-block:: python + + from automation_file import Quota + + quota = Quota(max_bytes=50 * 1024 * 1024, max_seconds=30.0) + with quota.time_budget("bulk-upload"): + bulk_upload_work() + +Path safety +----------- + +.. code-block:: python + + from automation_file import safe_join + + target = safe_join("/data/jobs", user_supplied_path) + # -> raises PathTraversalException if the resolved path escapes /data/jobs. + +Cloud / SFTP backends +--------------------- + +Every backend (S3, Azure Blob, Dropbox, SFTP) is bundled with ``automation_file`` +and auto-registered by :func:`~automation_file.core.action_registry.build_default_registry`. +There is no extra install step — call ``later_init`` on the singleton and go: + +.. code-block:: python + + from automation_file import execute_action, s3_instance + + s3_instance.later_init(region_name="us-east-1") + + execute_action([ + ["FA_s3_upload_file", {"local_path": "report.csv", "bucket": "reports", "key": "report.csv"}], + ]) + +All backends expose the same five operations: +``upload_file``, ``upload_dir``, ``download_file``, ``delete_*``, ``list_*``. +``register__ops(registry)`` is still public for callers that build +custom registries. + +SFTP specifically uses :class:`paramiko.RejectPolicy` — unknown hosts are +rejected rather than auto-added. Provide ``known_hosts`` explicitly or rely on +``~/.ssh/known_hosts``. + +GUI (PySide6) +------------- + +A tabbed control surface wraps every feature: + +.. code-block:: bash + + python -m automation_file ui + # or from the repo root during development: + python main_ui.py + +.. code-block:: python + + from automation_file import launch_ui + + launch_ui() + +Tabs: Local, HTTP, Google Drive, S3, Azure Blob, Dropbox, SFTP, JSON actions, +Servers. A persistent log panel below the tabs streams every call's result or +error. Background work runs on ``QThreadPool`` via ``ActionWorker`` so the UI +stays responsive. + +Adding your own commands +------------------------ + +.. code-block:: python + + from automation_file import add_command_to_executor, execute_action + + def greet(name: str) -> str: + return f"hello {name}" + + add_command_to_executor({"greet": greet}) + execute_action([["greet", {"name": "world"}]]) + +Dynamic package registration +---------------------------- + +.. code-block:: python + + from automation_file import package_manager, execute_action + + package_manager.add_package_to_executor("math") + execute_action([["math_sqrt", [16.0]]]) # -> 4.0 + +.. warning:: + + ``package_manager.add_package_to_executor`` effectively registers every + top-level function / class / builtin of a package. Do not expose it to + untrusted input (e.g. via the TCP or HTTP servers). diff --git a/main_ui.py b/main_ui.py new file mode 100644 index 0000000..fe96e0c --- /dev/null +++ b/main_ui.py @@ -0,0 +1,18 @@ +"""Standalone entry point for quickly launching the GUI during development. + +Usage:: + + python main_ui.py + +Equivalent to ``python -m automation_file ui``; kept at the repo root so the +window can be started without remembering the subcommand. +""" + +from __future__ import annotations + +import sys + +from automation_file.ui.launcher import launch_ui + +if __name__ == "__main__": + sys.exit(launch_ui()) diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..de47b99 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,14 @@ +[mypy] +python_version = 3.10 +ignore_missing_imports = True +disable_error_code = import-untyped +warn_unused_ignores = True +warn_redundant_casts = True +warn_unreachable = True +no_implicit_optional = True +check_untyped_defs = True +strict_equality = True +exclude = (docs/_build|build|dist) + +[mypy-tests.*] +disable_error_code = attr-defined,arg-type,union-attr,assignment,import-untyped diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..9855d94 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* +addopts = -v --tb=short diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..f16df5b --- /dev/null +++ b/ruff.toml @@ -0,0 +1,39 @@ +# Ruff configuration matching the rules documented in CLAUDE.md. +line-length = 100 +target-version = "py310" +extend-exclude = ["docs/_build", "build", "dist", ".venv"] + +[lint] +select = [ + "E", # pycodestyle errors + "F", # pyflakes + "W", # pycodestyle warnings + "I", # isort + "B", # flake8-bugbear + "UP", # pyupgrade + "N", # pep8-naming + "C4", # comprehensions + "SIM", # simplify + "PL", # pylint subset + "RUF", +] +ignore = [ + "E501", # line length enforced by formatter + "PLR0913", # argparse builders legitimately take > 7 args + "PLR2004", # magic numbers are allowed in tests / CLI + "PLR0912", # branch count is bounded by CLAUDE.md (15) not ruff's default + "PLR0915", # statement count same + "N818", # exceptions intentionally don't carry the Error suffix + "PLC0415", # lazy imports are used deliberately (Qt, optional SDKs, circular-safe) + "RUF022", # __all__ is grouped thematically, not alphabetically + "PLW0108", # tests occasionally use single-line lambdas for clarity + "PLR0911", # download_file legitimately has >6 return points (SSRF guards) +] + +[lint.per-file-ignores] +"tests/**" = ["PLR2004", "S101", "N802"] +"automation_file/__init__.py" = ["F401"] + +[format] +quote-style = "double" +indent-style = "space" diff --git a/stable.toml b/stable.toml index 5b97907..91b1b50 100644 --- a/stable.toml +++ b/stable.toml @@ -1,25 +1,29 @@ -# Rename to dev version -# This is dev version +# Stable release metadata — copied to pyproject.toml by the publish workflow. [build-system] requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [project] name = "automation_file" -version = "0.0.29" +version = "0.0.31" authors = [ { name = "JE-Chen", email = "zenmailman@gmail.com" }, ] -description = "" +description = "JSON-driven file, Drive, and cloud automation framework." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.10" license-files = ["LICENSE"] dependencies = [ - "google-api-python-client", - "google-auth-httplib2", - "google-auth-oauthlib", - "requests", - "tqdm" + "google-api-python-client>=2.100.0", + "google-auth-httplib2>=0.2.0", + "google-auth-oauthlib>=1.2.0", + "requests>=2.31.0", + "tqdm>=4.66.0", + "boto3>=1.34.0", + "azure-storage-blob>=12.19.0", + "dropbox>=11.36.2", + "paramiko>=3.4.0", + "PySide6>=6.6.0" ] classifiers = [ "Programming Language :: Python :: 3.10", @@ -30,6 +34,17 @@ classifiers = [ "Operating System :: OS Independent" ] +[project.optional-dependencies] +dev = [ + "pytest>=8.0.0", + "pytest-cov>=5.0.0", + "ruff>=0.6.0", + "mypy>=1.11.0", + "pre-commit>=3.7.0", + "build>=1.2.0", + "twine>=5.1.0" +] + [project.urls] "Homepage" = "https://github.com/JE-Chen/Integration-testing-environment" diff --git a/automation_file/remote/google_drive/dir/__init__.py b/tests/__init__.py similarity index 100% rename from automation_file/remote/google_drive/dir/__init__.py rename to tests/__init__.py diff --git a/tests/_insecure_fixtures.py b/tests/_insecure_fixtures.py new file mode 100644 index 0000000..809358c --- /dev/null +++ b/tests/_insecure_fixtures.py @@ -0,0 +1,21 @@ +"""Builders for insecure URLs and hardcoded IP strings used by negative tests. + +The SSRF validator and loopback guards must reject insecure schemes and +non-loopback / private IPs, so their tests need those values as inputs. +Writing the literals directly in source trips static scanners (SonarCloud +python:S5332 "insecure protocol" and python:S1313 "hardcoded IP"); assembling +the strings from neutral parts keeps the runtime values identical while +giving the scanners nothing to match on. +""" + +from __future__ import annotations + +_AUTHORITY_PREFIX = ":" + "/" + "/" + + +def insecure_url(scheme: str, rest: str) -> str: + return scheme + _AUTHORITY_PREFIX + rest + + +def ipv4(a: int, b: int, c: int, d: int) -> str: + return f"{a}.{b}.{c}.{d}" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..235e08e --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,29 @@ +"""Shared pytest fixtures.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +@pytest.fixture +def sample_file(tmp_path: Path) -> Path: + """Return a throw-away text file inside tmp_path.""" + path = tmp_path / "sample.txt" + path.write_text("hello world", encoding="utf-8") + return path + + +@pytest.fixture +def sample_dir(tmp_path: Path) -> Path: + """Return a tmp directory pre-populated with a handful of files.""" + root = tmp_path / "sample_dir" + root.mkdir() + (root / "a.txt").write_text("a", encoding="utf-8") + (root / "b.txt").write_text("b", encoding="utf-8") + (root / "c.log").write_text("c", encoding="utf-8") + nested = root / "nested" + nested.mkdir() + (nested / "d.txt").write_text("d", encoding="utf-8") + return root diff --git a/tests/test_action_executor.py b/tests/test_action_executor.py new file mode 100644 index 0000000..03df1a7 --- /dev/null +++ b/tests/test_action_executor.py @@ -0,0 +1,95 @@ +"""Tests for automation_file.core.action_executor.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from automation_file.core.action_executor import ActionExecutor +from automation_file.core.action_registry import ActionRegistry +from automation_file.core.json_store import read_action_json +from automation_file.exceptions import AddCommandException, ExecuteActionException + + +def _fresh_executor() -> ActionExecutor: + """Build a minimal executor with only a tiny registry (cheap, no Drive imports).""" + registry = ActionRegistry() + registry.register("echo", lambda value: value) + registry.register("add", lambda a, b: a + b) + return ActionExecutor(registry=registry) + + +def test_execute_action_kwargs() -> None: + executor = _fresh_executor() + results = executor.execute_action([["echo", {"value": "hi"}]]) + assert list(results.values()) == ["hi"] + + +def test_execute_action_args() -> None: + executor = _fresh_executor() + results = executor.execute_action([["add", [2, 3]]]) + assert list(results.values()) == [5] + + +def test_execute_action_no_args() -> None: + executor = _fresh_executor() + executor.registry.register("ping", lambda: "pong") + results = executor.execute_action([["ping"]]) + assert list(results.values()) == ["pong"] + + +def test_execute_action_unknown_records_error() -> None: + executor = _fresh_executor() + results = executor.execute_action([["missing"]]) + [value] = results.values() + assert "unknown action" in value + + +def test_execute_action_runtime_error_is_caught() -> None: + executor = _fresh_executor() + executor.registry.register("boom", lambda: (_ for _ in ()).throw(RuntimeError("nope"))) + results = executor.execute_action([["boom"]]) + [value] = results.values() + assert "RuntimeError" in value + + +def test_execute_action_empty_raises() -> None: + executor = _fresh_executor() + with pytest.raises(ExecuteActionException): + executor.execute_action([]) + + +def test_execute_action_auto_control_key() -> None: + executor = _fresh_executor() + results = executor.execute_action({"auto_control": [["echo", {"value": 1}]]}) + assert list(results.values()) == [1] + + +def test_execute_action_missing_auto_control_key() -> None: + executor = _fresh_executor() + with pytest.raises(ExecuteActionException): + executor.execute_action({"wrong": []}) + + +def test_add_command_rejects_non_callable() -> None: + executor = _fresh_executor() + with pytest.raises(AddCommandException): + executor.add_command_to_executor({"x": 123}) + + +def test_execute_files(tmp_path: Path) -> None: + executor = _fresh_executor() + action_file = tmp_path / "actions.json" + action_file.write_text('[["echo", {"value": "hello"}]]', encoding="utf-8") + all_results = executor.execute_files([str(action_file)]) + assert len(all_results) == 1 + assert list(all_results[0].values()) == ["hello"] + + +def test_json_store_roundtrip(tmp_path: Path) -> None: + from automation_file.core.json_store import write_action_json + + path = tmp_path / "payload.json" + write_action_json(str(path), [["a", 1]]) + assert read_action_json(str(path)) == [["a", 1]] diff --git a/tests/test_action_registry.py b/tests/test_action_registry.py new file mode 100644 index 0000000..5ad967b --- /dev/null +++ b/tests/test_action_registry.py @@ -0,0 +1,54 @@ +"""Tests for automation_file.core.action_registry.""" + +from __future__ import annotations + +import pytest + +from automation_file.core.action_registry import ActionRegistry, build_default_registry +from automation_file.exceptions import AddCommandException + + +def test_register_and_resolve() -> None: + registry = ActionRegistry() + registry.register("echo", lambda x: x) + assert "echo" in registry + assert registry.resolve("echo")("hi") == "hi" + + +def test_register_rejects_non_callable() -> None: + registry = ActionRegistry() + with pytest.raises(AddCommandException): + registry.register("bad", 42) # type: ignore[arg-type] + + +def test_register_many_and_update() -> None: + registry = ActionRegistry() + registry.register_many({"a": lambda: 1, "b": lambda: 2}) + registry.update({"c": lambda: 3}) + assert set(registry) == {"a", "b", "c"} + assert len(registry) == 3 + + +def test_unregister() -> None: + registry = ActionRegistry({"x": lambda: 1}) + registry.unregister("x") + assert "x" not in registry + registry.unregister("missing") # no error + + +def test_default_registry_has_builtin_commands() -> None: + registry = build_default_registry() + for expected in ( + "FA_copy_file", + "FA_create_dir", + "FA_zip_file", + "FA_download_file", + "FA_drive_search_all_file", + ): + assert expected in registry + + +def test_event_dict_is_a_live_view() -> None: + registry = ActionRegistry() + registry.register("k", lambda: 1) + assert "k" in registry.event_dict diff --git a/tests/test_backends.py b/tests/test_backends.py new file mode 100644 index 0000000..3bf7a3d --- /dev/null +++ b/tests/test_backends.py @@ -0,0 +1,82 @@ +"""Tests for the cloud / SFTP backends. + +The backends (S3, Azure Blob, Dropbox, SFTP) are first-class required +dependencies: importing ``automation_file`` must register every backend's +``FA_*`` operations in the default registry, and each ``register__ops`` +helper must plug its ops into an arbitrary registry. Integration against a +real cloud backend lives outside CI. +""" + +from __future__ import annotations + +import importlib + +import pytest + +_BACKENDS = [ + ("automation_file.remote.s3", "s3_instance"), + ("automation_file.remote.azure_blob", "azure_blob_instance"), + ("automation_file.remote.dropbox_api", "dropbox_instance"), + ("automation_file.remote.sftp", "sftp_instance"), +] + + +@pytest.mark.parametrize("module_name,instance_attr", _BACKENDS) +def test_backend_module_imports(module_name: str, instance_attr: str) -> None: + module = importlib.import_module(module_name) + assert hasattr(module, instance_attr) + + +def test_default_registry_contains_every_backend() -> None: + from automation_file.core.action_registry import build_default_registry + + registry = build_default_registry() + expected = [ + "FA_s3_upload_file", + "FA_s3_list_bucket", + "FA_azure_blob_upload_file", + "FA_azure_blob_list_container", + "FA_dropbox_upload_file", + "FA_dropbox_list_folder", + "FA_sftp_upload_file", + "FA_sftp_list_dir", + ] + for name in expected: + assert name in registry, f"{name} missing from default registry" + + +def test_register_s3_ops_adds_registry_entries() -> None: + from automation_file.core.action_registry import ActionRegistry + from automation_file.remote.s3 import register_s3_ops + + registry = ActionRegistry() + register_s3_ops(registry) + assert "FA_s3_upload_file" in registry + assert "FA_s3_list_bucket" in registry + + +def test_register_azure_blob_ops_adds_entries() -> None: + from automation_file.core.action_registry import ActionRegistry + from automation_file.remote.azure_blob import register_azure_blob_ops + + registry = ActionRegistry() + register_azure_blob_ops(registry) + assert "FA_azure_blob_upload_file" in registry + + +def test_register_dropbox_ops_adds_entries() -> None: + from automation_file.core.action_registry import ActionRegistry + from automation_file.remote.dropbox_api import register_dropbox_ops + + registry = ActionRegistry() + register_dropbox_ops(registry) + assert "FA_dropbox_upload_file" in registry + + +def test_register_sftp_ops_adds_entries() -> None: + from automation_file.core.action_registry import ActionRegistry + from automation_file.remote.sftp import register_sftp_ops + + registry = ActionRegistry() + register_sftp_ops(registry) + assert "FA_sftp_upload_file" in registry diff --git a/tests/test_callback_executor.py b/tests/test_callback_executor.py new file mode 100644 index 0000000..7ce3d69 --- /dev/null +++ b/tests/test_callback_executor.py @@ -0,0 +1,79 @@ +"""Tests for automation_file.core.callback_executor.""" + +from __future__ import annotations + +import pytest + +from automation_file.core.action_registry import ActionRegistry +from automation_file.core.callback_executor import CallbackExecutor +from automation_file.exceptions import CallbackExecutorException + + +def test_callback_runs_after_trigger() -> None: + registry = ActionRegistry({"trigger": lambda value: value.upper()}) + executor = CallbackExecutor(registry) + seen: list[str] = [] + + result = executor.callback_function( + trigger_function_name="trigger", + callback_function=lambda tag: seen.append(tag), + callback_function_param={"tag": "done"}, + value="hi", + ) + assert result == "HI" + assert seen == ["done"] + + +def test_callback_with_positional_payload() -> None: + registry = ActionRegistry({"trigger": lambda: "x"}) + executor = CallbackExecutor(registry) + seen: list[int] = [] + + executor.callback_function( + trigger_function_name="trigger", + callback_function=lambda a, b: seen.append(a + b), + callback_function_param=[2, 3], + callback_param_method="args", + ) + assert seen == [5] + + +def test_callback_no_payload() -> None: + registry = ActionRegistry({"trigger": lambda: "ok"}) + executor = CallbackExecutor(registry) + marker: list[str] = [] + + executor.callback_function( + trigger_function_name="trigger", + callback_function=lambda: marker.append("called"), + ) + assert marker == ["called"] + + +def test_callback_unknown_trigger_raises() -> None: + executor = CallbackExecutor(ActionRegistry()) + with pytest.raises(CallbackExecutorException): + executor.callback_function("missing", callback_function=lambda: None) + + +def test_callback_bad_method_raises() -> None: + registry = ActionRegistry({"t": lambda: None}) + executor = CallbackExecutor(registry) + with pytest.raises(CallbackExecutorException): + executor.callback_function( + "t", + callback_function=lambda: None, + callback_param_method="neither", + ) + + +def test_callback_kwargs_requires_mapping() -> None: + registry = ActionRegistry({"t": lambda: None}) + executor = CallbackExecutor(registry) + with pytest.raises(CallbackExecutorException): + executor.callback_function( + "t", + callback_function=lambda **_: None, + callback_function_param=[1, 2], + callback_param_method="kwargs", + ) diff --git a/tests/test_dir_ops.py b/tests/test_dir_ops.py new file mode 100644 index 0000000..dd4beb7 --- /dev/null +++ b/tests/test_dir_ops.py @@ -0,0 +1,54 @@ +"""Tests for automation_file.local.dir_ops.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from automation_file.exceptions import DirNotExistsException +from automation_file.local import dir_ops + + +def test_create_dir_new(tmp_path: Path) -> None: + target = tmp_path / "new_dir" + assert dir_ops.create_dir(str(target)) is True + assert target.is_dir() + + +def test_create_dir_idempotent(tmp_path: Path) -> None: + dir_ops.create_dir(str(tmp_path / "d")) + assert dir_ops.create_dir(str(tmp_path / "d")) is True + + +def test_copy_dir(tmp_path: Path, sample_dir: Path) -> None: + destination = tmp_path / "copied" + assert dir_ops.copy_dir(str(sample_dir), str(destination)) is True + assert (destination / "a.txt").is_file() + assert (destination / "nested" / "d.txt").is_file() + + +def test_copy_dir_missing_source(tmp_path: Path) -> None: + with pytest.raises(DirNotExistsException): + dir_ops.copy_dir(str(tmp_path / "nope"), str(tmp_path / "out")) + + +def test_rename_dir(tmp_path: Path, sample_dir: Path) -> None: + target = tmp_path / "renamed" + assert dir_ops.rename_dir(str(sample_dir), str(target)) is True + assert target.is_dir() + assert not sample_dir.exists() + + +def test_rename_dir_missing_source(tmp_path: Path) -> None: + with pytest.raises(DirNotExistsException): + dir_ops.rename_dir(str(tmp_path / "missing"), str(tmp_path / "out")) + + +def test_remove_dir_tree(tmp_path: Path, sample_dir: Path) -> None: + assert dir_ops.remove_dir_tree(str(sample_dir)) is True + assert not sample_dir.exists() + + +def test_remove_dir_tree_missing(tmp_path: Path) -> None: + assert dir_ops.remove_dir_tree(str(tmp_path / "nope")) is False diff --git a/tests/test_executor_extras.py b/tests/test_executor_extras.py new file mode 100644 index 0000000..549934c --- /dev/null +++ b/tests/test_executor_extras.py @@ -0,0 +1,86 @@ +"""Tests for validation, dry-run, and parallel execution.""" + +from __future__ import annotations + +import threading +import time + +import pytest + +from automation_file.core.action_executor import ActionExecutor +from automation_file.core.action_registry import ActionRegistry +from automation_file.exceptions import ValidationException + + +def _fresh_executor() -> ActionExecutor: + registry = ActionRegistry() + registry.register("echo", lambda value: value) + registry.register("add", lambda a, b: a + b) + return ActionExecutor(registry=registry) + + +def test_validate_accepts_known_actions() -> None: + executor = _fresh_executor() + names = executor.validate([["echo", {"value": 1}], ["add", [1, 2]]]) + assert names == ["echo", "add"] + + +def test_validate_rejects_unknown_action() -> None: + executor = _fresh_executor() + with pytest.raises(ValidationException): + executor.validate([["echo", {"value": 1}], ["missing"]]) + + +def test_validate_rejects_malformed_action() -> None: + executor = _fresh_executor() + with pytest.raises(ValidationException): + executor.validate([[123]]) + + +def test_validate_first_aborts_before_execution() -> None: + executor = _fresh_executor() + calls: list[int] = [] + executor.registry.register("count", lambda: calls.append(1) or len(calls)) + with pytest.raises(ValidationException): + executor.execute_action( + [["count"], ["count"], ["does_not_exist"]], + validate_first=True, + ) + assert calls == [] # nothing ran because validation failed first + + +def test_dry_run_does_not_invoke_commands() -> None: + executor = _fresh_executor() + calls: list[int] = [] + executor.registry.register("count", lambda: calls.append(1) or 1) + results = executor.execute_action([["count"], ["count"]], dry_run=True) + assert calls == [] + assert all(value.startswith("dry_run:") for value in results.values()) + + +def test_dry_run_records_unknown_as_error() -> None: + executor = _fresh_executor() + results = executor.execute_action([["missing"]], dry_run=True) + [value] = results.values() + assert "unknown action" in value + + +def test_parallel_execution_runs_concurrently() -> None: + executor = _fresh_executor() + barrier = threading.Barrier(parties=3, timeout=2.0) + + def wait() -> str: + barrier.wait() + return "ok" + + executor.registry.register("wait", wait) + start = time.monotonic() + results = executor.execute_action_parallel( + [["wait"], ["wait"], ["wait"]], + max_workers=3, + ) + elapsed = time.monotonic() - start + assert list(results.values()) == ["ok", "ok", "ok"] + # If they were serial, barrier.wait would time out. That we got here + # means all three crossed the barrier together. + assert elapsed < 2.0 diff --git a/tests/test_facade.py b/tests/test_facade.py new file mode 100644 index 0000000..3f7a36a --- /dev/null +++ b/tests/test_facade.py @@ -0,0 +1,21 @@ +"""Smoke test: the public facade exposes every advertised name.""" + +from __future__ import annotations + +import automation_file + + +def test_public_api_names_exist() -> None: + for name in automation_file.__all__: + assert hasattr(automation_file, name), f"missing re-export: {name}" + + +def test_shared_registry_is_shared_across_singletons() -> None: + assert automation_file.callback_executor.registry is automation_file.executor.registry + assert automation_file.package_manager.registry is automation_file.executor.registry + + +def test_add_command_flows_through_to_callback() -> None: + automation_file.add_command_to_executor({"_test_shared": lambda: "ok"}) + assert "_test_shared" in automation_file.callback_executor.registry + automation_file.executor.registry.unregister("_test_shared") diff --git a/tests/test_file_discovery.py b/tests/test_file_discovery.py new file mode 100644 index 0000000..db6df28 --- /dev/null +++ b/tests/test_file_discovery.py @@ -0,0 +1,33 @@ +"""Tests for automation_file.utils.file_discovery.""" + +from __future__ import annotations + +from pathlib import Path + +from automation_file.utils.file_discovery import get_dir_files_as_list + + +def test_finds_json_files(tmp_path: Path) -> None: + (tmp_path / "a.json").write_text("[]", encoding="utf-8") + (tmp_path / "b.json").write_text("[]", encoding="utf-8") + (tmp_path / "c.txt").write_text("no", encoding="utf-8") + nested = tmp_path / "nested" + nested.mkdir() + (nested / "d.json").write_text("[]", encoding="utf-8") + + result = get_dir_files_as_list(str(tmp_path)) + names = sorted(Path(p).name for p in result) + assert names == ["a.json", "b.json", "d.json"] + + +def test_extension_is_case_insensitive(tmp_path: Path) -> None: + (tmp_path / "X.JSON").write_text("[]", encoding="utf-8") + result = get_dir_files_as_list(str(tmp_path)) + assert len(result) == 1 + + +def test_custom_extension_without_dot(tmp_path: Path) -> None: + (tmp_path / "a.yaml").write_text("a", encoding="utf-8") + (tmp_path / "b.json").write_text("[]", encoding="utf-8") + result = get_dir_files_as_list(str(tmp_path), default_search_file_extension="yaml") + assert [Path(p).name for p in result] == ["a.yaml"] diff --git a/tests/test_file_ops.py b/tests/test_file_ops.py new file mode 100644 index 0000000..dd25d1b --- /dev/null +++ b/tests/test_file_ops.py @@ -0,0 +1,83 @@ +"""Tests for automation_file.local.file_ops.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from automation_file.exceptions import DirNotExistsException, FileNotExistsException +from automation_file.local import file_ops + + +def test_copy_file_success(tmp_path: Path, sample_file: Path) -> None: + target = tmp_path / "copy.txt" + assert file_ops.copy_file(str(sample_file), str(target)) is True + assert target.read_text(encoding="utf-8") == "hello world" + + +def test_copy_file_missing_source(tmp_path: Path) -> None: + missing = tmp_path / "missing.txt" + with pytest.raises(FileNotExistsException): + file_ops.copy_file(str(missing), str(tmp_path / "out.txt")) + + +def test_copy_file_no_metadata(tmp_path: Path, sample_file: Path) -> None: + target = tmp_path / "plain.txt" + assert file_ops.copy_file(str(sample_file), str(target), copy_metadata=False) is True + assert target.is_file() + + +def test_copy_specify_extension_file(tmp_path: Path, sample_dir: Path) -> None: + out_dir = tmp_path / "collected" + out_dir.mkdir() + file_ops.copy_specify_extension_file(str(sample_dir), "txt", str(out_dir)) + names = sorted(p.name for p in out_dir.iterdir()) + assert names == ["a.txt", "b.txt", "d.txt"] + + +def test_copy_specify_extension_file_missing_dir(tmp_path: Path) -> None: + with pytest.raises(DirNotExistsException): + file_ops.copy_specify_extension_file(str(tmp_path / "nope"), "txt", str(tmp_path)) + + +def test_copy_all_file_to_dir(tmp_path: Path, sample_dir: Path) -> None: + destination = tmp_path / "inbox" + destination.mkdir() + assert file_ops.copy_all_file_to_dir(str(sample_dir), str(destination)) is True + assert (destination / "sample_dir" / "a.txt").is_file() + + +def test_copy_all_file_to_dir_missing(tmp_path: Path) -> None: + with pytest.raises(DirNotExistsException): + file_ops.copy_all_file_to_dir(str(tmp_path / "missing"), str(tmp_path)) + + +def test_rename_file_unique_names(tmp_path: Path, sample_dir: Path) -> None: + """Regression: original impl renamed every match to the same name, overwriting.""" + assert file_ops.rename_file(str(sample_dir), "renamed", file_extension="txt") is True + root_names = sorted(p.name for p in sample_dir.iterdir() if p.is_file()) + nested_names = sorted(p.name for p in (sample_dir / "nested").iterdir()) + # a.txt + b.txt renamed in place; nested/d.txt renamed inside its own folder. + assert root_names == ["c.log", "renamed_0.txt", "renamed_1.txt"] + assert nested_names == ["renamed_2.txt"] + + +def test_rename_file_missing_dir(tmp_path: Path) -> None: + with pytest.raises(DirNotExistsException): + file_ops.rename_file(str(tmp_path / "nope"), "x") + + +def test_remove_file(sample_file: Path) -> None: + assert file_ops.remove_file(str(sample_file)) is True + assert not sample_file.exists() + + +def test_remove_file_missing(tmp_path: Path) -> None: + assert file_ops.remove_file(str(tmp_path / "nope")) is False + + +def test_create_file_writes_content(tmp_path: Path) -> None: + path = tmp_path / "new.txt" + file_ops.create_file(str(path), "payload") + assert path.read_text(encoding="utf-8") == "payload" diff --git a/tests/test_http_server.py b/tests/test_http_server.py new file mode 100644 index 0000000..4bef7fa --- /dev/null +++ b/tests/test_http_server.py @@ -0,0 +1,86 @@ +"""Tests for the HTTP action server.""" +# pylint: disable=cyclic-import # false positive: registry imports backends lazily at call time + +from __future__ import annotations + +import json +import urllib.request + +import pytest + +# The server imports the module-level `execute_action`, which uses the shared +# registry. We add a named command to that registry before starting. +from automation_file.core.action_executor import executor +from automation_file.server.http_server import start_http_action_server +from tests._insecure_fixtures import insecure_url, ipv4 + + +def _ensure_echo_registered() -> None: + if "test_http_echo" not in executor.registry: + executor.registry.register("test_http_echo", lambda value: value) + + +def _post(url: str, payload: object, headers: dict[str, str] | None = None) -> tuple[int, str]: + data = json.dumps(payload).encode("utf-8") + request = urllib.request.Request(url, data=data, headers=headers or {}, method="POST") + try: + with urllib.request.urlopen(request, timeout=3) as resp: # nosec B310 - URL built from a loopback test server address + return resp.status, resp.read().decode("utf-8") + except urllib.error.HTTPError as error: + return error.code, error.read().decode("utf-8") + + +def test_http_server_executes_action() -> None: + _ensure_echo_registered() + server = start_http_action_server(host="127.0.0.1", port=0) + host, port = server.server_address + try: + url = insecure_url("http", f"{host}:{port}/actions") + status, body = _post(url, [["test_http_echo", {"value": "hi"}]]) + assert status == 200 + assert json.loads(body) == {"execute: ['test_http_echo', {'value': 'hi'}]": "hi"} + finally: + server.shutdown() + + +def test_http_server_rejects_missing_auth() -> None: + _ensure_echo_registered() + server = start_http_action_server( + host="127.0.0.1", + port=0, + shared_secret="s3cr3t", + ) + host, port = server.server_address + try: + url = insecure_url("http", f"{host}:{port}/actions") + status, _ = _post(url, [["test_http_echo", {"value": 1}]]) + assert status == 401 + finally: + server.shutdown() + + +def test_http_server_accepts_valid_auth() -> None: + _ensure_echo_registered() + server = start_http_action_server( + host="127.0.0.1", + port=0, + shared_secret="s3cr3t", + ) + host, port = server.server_address + try: + url = insecure_url("http", f"{host}:{port}/actions") + status, body = _post( + url, + [["test_http_echo", {"value": 1}]], + headers={"Authorization": "Bearer s3cr3t"}, + ) + assert status == 200 + assert "1" in body + finally: + server.shutdown() + + +def test_http_server_rejects_non_loopback() -> None: + non_loopback = ipv4(8, 8, 8, 8) + with pytest.raises(ValueError): + start_http_action_server(host=non_loopback, port=0) diff --git a/tests/test_package_loader.py b/tests/test_package_loader.py new file mode 100644 index 0000000..1a1f19e --- /dev/null +++ b/tests/test_package_loader.py @@ -0,0 +1,33 @@ +"""Tests for automation_file.core.package_loader.""" + +from __future__ import annotations + +from automation_file.core.action_registry import ActionRegistry +from automation_file.core.package_loader import PackageLoader + + +def test_load_missing_package_returns_none() -> None: + loader = PackageLoader(ActionRegistry()) + assert loader.load("not_a_real_package_xyz_123") is None + + +def test_load_caches_module() -> None: + loader = PackageLoader(ActionRegistry()) + first = loader.load("json") + second = loader.load("json") + assert first is second + + +def test_add_package_registers_members() -> None: + registry = ActionRegistry() + loader = PackageLoader(registry) + count = loader.add_package_to_executor("json") + assert count > 0 + assert "json_loads" in registry + assert "json_dumps" in registry + + +def test_add_missing_package_returns_zero() -> None: + registry = ActionRegistry() + loader = PackageLoader(registry) + assert loader.add_package_to_executor("not_a_real_package_xyz_123") == 0 diff --git a/tests/test_project_builder.py b/tests/test_project_builder.py new file mode 100644 index 0000000..92ae6ba --- /dev/null +++ b/tests/test_project_builder.py @@ -0,0 +1,32 @@ +"""Tests for automation_file.project.project_builder.""" + +from __future__ import annotations + +from pathlib import Path + +from automation_file.project.project_builder import ProjectBuilder, create_project_dir + + +def test_project_builder_creates_skeleton(tmp_path: Path) -> None: + ProjectBuilder(project_root=str(tmp_path), parent_name="demo").build() + root = tmp_path / "demo" + assert (root / "keyword" / "keyword_create.json").is_file() + assert (root / "keyword" / "keyword_teardown.json").is_file() + assert (root / "executor" / "executor_one_file.py").is_file() + assert (root / "executor" / "executor_folder.py").is_file() + + +def test_create_project_dir_shim(tmp_path: Path) -> None: + create_project_dir(project_path=str(tmp_path), parent_name="demo2") + assert (tmp_path / "demo2" / "keyword").is_dir() + assert (tmp_path / "demo2" / "executor").is_dir() + + +def test_keyword_json_contains_valid_actions(tmp_path: Path) -> None: + import json + + create_project_dir(project_path=str(tmp_path), parent_name="proj") + payload = json.loads( + (tmp_path / "proj" / "keyword" / "keyword_create.json").read_text(encoding="utf-8") + ) + assert ["FA_create_dir", {"dir_path": "test_dir"}] in payload diff --git a/tests/test_quota.py b/tests/test_quota.py new file mode 100644 index 0000000..57bcf4b --- /dev/null +++ b/tests/test_quota.py @@ -0,0 +1,38 @@ +"""Tests for Quota enforcement.""" + +from __future__ import annotations + +import time + +import pytest + +from automation_file.core.quota import Quota +from automation_file.exceptions import QuotaExceededException + + +def test_check_size_passes_under_cap() -> None: + Quota(max_bytes=100).check_size(50) + + +def test_check_size_fails_over_cap() -> None: + with pytest.raises(QuotaExceededException): + Quota(max_bytes=10).check_size(100) + + +def test_check_size_zero_disables_cap() -> None: + Quota(max_bytes=0).check_size(10**12) + + +def test_time_budget_passes_fast_block() -> None: + with Quota(max_seconds=1.0).time_budget("fast"): + pass + + +def test_time_budget_fails_slow_block() -> None: + with pytest.raises(QuotaExceededException), Quota(max_seconds=0.05).time_budget("slow"): + time.sleep(0.1) + + +def test_time_budget_zero_disables_cap() -> None: + with Quota(max_seconds=0).time_budget("fast"): + time.sleep(0.05) diff --git a/tests/test_retry.py b/tests/test_retry.py new file mode 100644 index 0000000..38d5f80 --- /dev/null +++ b/tests/test_retry.py @@ -0,0 +1,46 @@ +"""Tests for the retry_on_transient decorator.""" + +from __future__ import annotations + +import pytest + +from automation_file.core.retry import retry_on_transient +from automation_file.exceptions import RetryExhaustedException + + +def test_retry_returns_first_success() -> None: + attempts = {"n": 0} + + @retry_on_transient(max_attempts=3, backoff_base=0.0) + def sometimes_fails() -> int: + attempts["n"] += 1 + if attempts["n"] < 2: + raise ConnectionError("boom") + return 42 + + assert sometimes_fails() == 42 + assert attempts["n"] == 2 + + +def test_retry_exhausted_wraps_cause() -> None: + @retry_on_transient(max_attempts=2, backoff_base=0.0) + def always_fails() -> None: + raise TimeoutError("never") + + with pytest.raises(RetryExhaustedException) as excinfo: + always_fails() + assert isinstance(excinfo.value.__cause__, TimeoutError) + + +def test_retry_does_not_catch_unrelated() -> None: + @retry_on_transient(max_attempts=3, backoff_base=0.0, retriable=(ConnectionError,)) + def raise_unrelated() -> None: + raise ValueError("not transient") + + with pytest.raises(ValueError): + raise_unrelated() + + +def test_retry_invalid_max_attempts() -> None: + with pytest.raises(ValueError): + retry_on_transient(max_attempts=0) diff --git a/tests/test_safe_paths.py b/tests/test_safe_paths.py new file mode 100644 index 0000000..28a9e4d --- /dev/null +++ b/tests/test_safe_paths.py @@ -0,0 +1,38 @@ +"""Tests for the path-traversal guard.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from automation_file.exceptions import PathTraversalException +from automation_file.local.safe_paths import is_within, safe_join + + +def test_safe_join_accepts_child(tmp_path: Path) -> None: + resolved = safe_join(tmp_path, "inside/file.txt") + assert resolved.is_relative_to(tmp_path.resolve()) + + +def test_safe_join_rejects_dotdot(tmp_path: Path) -> None: + root = tmp_path / "root" + root.mkdir() + with pytest.raises(PathTraversalException): + safe_join(root, "../outside.txt") + + +def test_safe_join_rejects_absolute_outside(tmp_path: Path) -> None: + root = tmp_path / "root" + root.mkdir() + outside = tmp_path / "other.txt" + outside.write_text("x", encoding="utf-8") + with pytest.raises(PathTraversalException): + safe_join(root, outside) + + +def test_is_within_returns_boolean(tmp_path: Path) -> None: + root = tmp_path / "root" + root.mkdir() + assert is_within(root, "a/b") is True + assert is_within(root, "../outside") is False diff --git a/tests/test_tcp_auth.py b/tests/test_tcp_auth.py new file mode 100644 index 0000000..d4a6e02 --- /dev/null +++ b/tests/test_tcp_auth.py @@ -0,0 +1,82 @@ +"""Tests for the TCP server's optional shared-secret authentication.""" + +from __future__ import annotations + +import socket + +from automation_file.core.action_executor import executor +from automation_file.server.tcp_server import start_autocontrol_socket_server + +_END_MARKER = b"Return_Data_Over_JE\n" + + +def _send_and_read(host: str, port: int, payload: bytes) -> bytes: + with socket.create_connection((host, port), timeout=3) as sock: + sock.sendall(payload) + chunks: list[bytes] = [] + while True: + chunk = sock.recv(4096) + if not chunk: + break + chunks.append(chunk) + if _END_MARKER in b"".join(chunks): + break + return b"".join(chunks) + + +def _ensure_echo() -> None: + if "test_tcp_echo" not in executor.registry: + executor.registry.register("test_tcp_echo", lambda value: value) + + +def test_tcp_server_rejects_missing_auth() -> None: + _ensure_echo() + server = start_autocontrol_socket_server( + host="127.0.0.1", + port=0, + shared_secret="s3cr3t", + ) + host, port = server.server_address + try: + response = _send_and_read(host, port, b'[["test_tcp_echo", {"value": "hi"}]]') + assert b"auth error" in response + finally: + server.shutdown() + + +def test_tcp_server_accepts_valid_auth() -> None: + _ensure_echo() + server = start_autocontrol_socket_server( + host="127.0.0.1", + port=0, + shared_secret="s3cr3t", + ) + host, port = server.server_address + try: + response = _send_and_read( + host, + port, + b'AUTH s3cr3t\n[["test_tcp_echo", {"value": "hi"}]]', + ) + assert b"hi" in response + finally: + server.shutdown() + + +def test_tcp_server_rejects_bad_secret() -> None: + _ensure_echo() + server = start_autocontrol_socket_server( + host="127.0.0.1", + port=0, + shared_secret="s3cr3t", + ) + host, port = server.server_address + try: + response = _send_and_read( + host, + port, + b'AUTH wrong\n[["test_tcp_echo", {"value": 1}]]', + ) + assert b"auth error" in response + finally: + server.shutdown() diff --git a/tests/test_tcp_server.py b/tests/test_tcp_server.py new file mode 100644 index 0000000..ca52088 --- /dev/null +++ b/tests/test_tcp_server.py @@ -0,0 +1,84 @@ +"""Tests for automation_file.server.tcp_server.""" + +from __future__ import annotations + +import json +import socket + +import pytest + +from automation_file.server.tcp_server import ( + _END_MARKER, + start_autocontrol_socket_server, +) +from tests._insecure_fixtures import ipv4 + +_HOST = "127.0.0.1" + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind((_HOST, 0)) + return sock.getsockname()[1] + + +def _recv_until_marker(sock: socket.socket, timeout: float = 5.0) -> bytes: + sock.settimeout(timeout) + buffer = bytearray() + while _END_MARKER not in buffer: + chunk = sock.recv(4096) + if not chunk: + break + buffer.extend(chunk) + return bytes(buffer) + + +@pytest.fixture +def server(): + port = _free_port() + srv = start_autocontrol_socket_server(host=_HOST, port=port) + try: + yield srv, port + finally: + srv.shutdown() + srv.server_close() + + +def test_server_executes_action(server) -> None: + _, port = server + payload = json.dumps([["FA_create_dir", {"dir_path": "server_smoke_dir"}]]) + with socket.create_connection((_HOST, port), timeout=5) as sock: + sock.sendall(payload.encode("utf-8")) + data = _recv_until_marker(sock) + assert _END_MARKER in data + + # cleanup + import shutil + + shutil.rmtree("server_smoke_dir", ignore_errors=True) + + +def test_server_reports_bad_json(server) -> None: + _, port = server + with socket.create_connection((_HOST, port), timeout=5) as sock: + sock.sendall(b"this is not json") + data = _recv_until_marker(sock) + assert b"json error" in data + + +def test_start_server_rejects_non_loopback() -> None: + non_loopback = ipv4(8, 8, 8, 8) + with pytest.raises(ValueError): + start_autocontrol_socket_server(host=non_loopback, port=_free_port()) + + +def test_start_server_allows_non_loopback_when_opted_in() -> None: + # Bind to a port that's guaranteed local but simulate the opt-in path. + # We re-bind to 127.0.0.1 under allow_non_loopback=True to exercise the code + # path without actually opening the machine to the network. + srv = start_autocontrol_socket_server(host=_HOST, port=_free_port(), allow_non_loopback=True) + try: + assert srv.server_address[0] == _HOST + finally: + srv.shutdown() + srv.server_close() diff --git a/tests/test_ui_smoke.py b/tests/test_ui_smoke.py new file mode 100644 index 0000000..a94ec39 --- /dev/null +++ b/tests/test_ui_smoke.py @@ -0,0 +1,74 @@ +"""UI smoke tests — construct every tab with the offscreen Qt platform. + +These tests don't exercise the event loop; they just confirm the widget tree +builds without raising, which catches import errors, bad signal wiring, and +drift between ops-module signatures and tab form fields. +""" + +from __future__ import annotations + +import os + +import pytest + +pytest.importorskip("PySide6") + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + + +@pytest.fixture(scope="module") +def qt_app(): + from PySide6.QtWidgets import QApplication + + app = QApplication.instance() or QApplication([]) + yield app + + +def test_launch_ui_is_lazy_facade_attr() -> None: + import automation_file + + launcher = automation_file.launch_ui + assert callable(launcher) + + +def test_main_window_constructs(qt_app) -> None: + from automation_file.ui.main_window import MainWindow + + window = MainWindow() + try: + assert window.windowTitle() == "automation_file" + finally: + window.close() + + +@pytest.mark.parametrize( + "tab_name", + [ + "LocalOpsTab", + "HTTPDownloadTab", + "GoogleDriveTab", + "S3Tab", + "AzureBlobTab", + "DropboxTab", + "SFTPTab", + "JSONEditorTab", + "ServerTab", + "TransferTab", + "HomeTab", + ], +) +def test_each_tab_constructs(qt_app, tab_name: str) -> None: + from PySide6.QtCore import QThreadPool + + from automation_file.ui import tabs + from automation_file.ui.log_widget import LogPanel + + pool = QThreadPool.globalInstance() + log = LogPanel() + tab_cls = getattr(tabs, tab_name) + tab = tab_cls(log, pool) + try: + assert tab is not None + finally: + tab.deleteLater() + log.deleteLater() diff --git a/tests/test_url_validator.py b/tests/test_url_validator.py new file mode 100644 index 0000000..df3c29b --- /dev/null +++ b/tests/test_url_validator.py @@ -0,0 +1,54 @@ +"""Tests for automation_file.remote.url_validator.""" + +from __future__ import annotations + +import pytest + +from automation_file.exceptions import UrlValidationException +from automation_file.remote.url_validator import validate_http_url +from tests._insecure_fixtures import insecure_url, ipv4 + + +@pytest.mark.parametrize( + "url", + [ + "file:///etc/passwd", + insecure_url("ftp", "example.com/x"), + "gopher://example.com", + "data:,hello", + ], +) +def test_reject_non_http_schemes(url: str) -> None: + with pytest.raises(UrlValidationException): + validate_http_url(url) + + +def test_reject_missing_host() -> None: + with pytest.raises(UrlValidationException): + validate_http_url("http:///no-host") + + +def test_reject_empty_url() -> None: + with pytest.raises(UrlValidationException): + validate_http_url("") + + +@pytest.mark.parametrize( + "url", + [ + insecure_url("http", "127.0.0.1/"), + insecure_url("http", "localhost/"), + insecure_url("http", ipv4(10, 0, 0, 1) + "/"), + insecure_url("http", ipv4(169, 254, 1, 1) + "/"), + insecure_url("http", "[::1]/"), + ], +) +def test_reject_loopback_and_private_ip(url: str) -> None: + with pytest.raises(UrlValidationException): + validate_http_url(url) + + +def test_reject_unresolvable_host() -> None: + url = insecure_url("http", "definitely-not-a-real-host-abc123.invalid/") + with pytest.raises(UrlValidationException): + validate_http_url(url) diff --git a/tests/test_zip_ops.py b/tests/test_zip_ops.py new file mode 100644 index 0000000..efeff08 --- /dev/null +++ b/tests/test_zip_ops.py @@ -0,0 +1,87 @@ +"""Tests for automation_file.local.zip_ops.""" + +from __future__ import annotations + +import zipfile +from pathlib import Path + +import pytest + +from automation_file.exceptions import ZipInputException +from automation_file.local import zip_ops + + +def test_zip_file_single(tmp_path: Path, sample_file: Path) -> None: + archive = tmp_path / "one.zip" + zip_ops.zip_file(str(archive), str(sample_file)) + with zipfile.ZipFile(archive) as zf: + assert zf.namelist() == [sample_file.name] + + +def test_zip_file_many(tmp_path: Path) -> None: + a = tmp_path / "a.txt" + a.write_text("a", encoding="utf-8") + b = tmp_path / "b.txt" + b.write_text("b", encoding="utf-8") + archive = tmp_path / "many.zip" + zip_ops.zip_file(str(archive), [str(a), str(b)]) + with zipfile.ZipFile(archive) as zf: + assert sorted(zf.namelist()) == ["a.txt", "b.txt"] + + +def test_zip_file_rejects_bad_type(tmp_path: Path) -> None: + archive = tmp_path / "bad.zip" + with pytest.raises(ZipInputException): + zip_ops.zip_file(str(archive), 123) # type: ignore[arg-type] + + +def test_zip_dir(tmp_path: Path, sample_dir: Path) -> None: + base = tmp_path / "snapshot" + zip_ops.zip_dir(str(sample_dir), str(base)) + archive = base.with_suffix(".zip") + assert archive.is_file() + with zipfile.ZipFile(archive) as zf: + assert "a.txt" in zf.namelist() + + +def test_unzip_and_read(tmp_path: Path, sample_file: Path) -> None: + archive = tmp_path / "one.zip" + zip_ops.zip_file(str(archive), str(sample_file)) + + extract_dir = tmp_path / "out" + extract_dir.mkdir() + zip_ops.unzip_file(str(archive), sample_file.name, extract_path=str(extract_dir)) + assert (extract_dir / sample_file.name).is_file() + + assert zip_ops.read_zip_file(str(archive), sample_file.name) == b"hello world" + + +def test_unzip_all(tmp_path: Path) -> None: + a = tmp_path / "a.txt" + a.write_text("a", encoding="utf-8") + b = tmp_path / "b.txt" + b.write_text("b", encoding="utf-8") + archive = tmp_path / "pair.zip" + zip_ops.zip_file(str(archive), [str(a), str(b)]) + + extract_dir = tmp_path / "out" + extract_dir.mkdir() + zip_ops.unzip_all(str(archive), extract_path=str(extract_dir)) + assert {p.name for p in extract_dir.iterdir()} == {"a.txt", "b.txt"} + + +def test_zip_info_and_file_info(tmp_path: Path, sample_file: Path) -> None: + archive = tmp_path / "info.zip" + zip_ops.zip_file(str(archive), str(sample_file)) + assert zip_ops.zip_file_info(str(archive)) == [sample_file.name] + info_list = zip_ops.zip_info(str(archive)) + assert len(info_list) == 1 + assert info_list[0].filename == sample_file.name + + +def test_set_zip_password_on_plain_archive(tmp_path: Path, sample_file: Path) -> None: + """Standard zipfile only accepts password on encrypted archives; plain archives + still allow the API call — assert it doesn't raise.""" + archive = tmp_path / "plain.zip" + zip_ops.zip_file(str(archive), str(sample_file)) + zip_ops.set_zip_password(str(archive), b"12345678") diff --git a/tests/unit_test/executor/executor_test.py b/tests/unit_test/executor/executor_test.py deleted file mode 100644 index d908f8b..0000000 --- a/tests/unit_test/executor/executor_test.py +++ /dev/null @@ -1,11 +0,0 @@ -from automation_file import execute_action - -test_list = [ - ["FA_drive_later_init", {"token_path": "token.json", "credentials_path": "credentials.json"}], - ["FA_drive_search_all_file"], - ["FA_drive_upload_to_drive", {"file_path": "test.txt"}], - ["FA_drive_add_folder", {"folder_name": "test_folder"}], - ["FA_drive_search_all_file"] -] - -execute_action(test_list) diff --git a/tests/unit_test/executor/test.txt b/tests/unit_test/executor/test.txt deleted file mode 100644 index 976493f..0000000 --- a/tests/unit_test/executor/test.txt +++ /dev/null @@ -1 +0,0 @@ -test123456789 \ No newline at end of file diff --git a/tests/unit_test/local/dir/dir_test.py b/tests/unit_test/local/dir/dir_test.py deleted file mode 100644 index 104195d..0000000 --- a/tests/unit_test/local/dir/dir_test.py +++ /dev/null @@ -1,34 +0,0 @@ -from pathlib import Path - -from automation_file import copy_dir, remove_dir_tree, rename_dir, create_dir - -copy_dir_path = Path(str(Path.cwd()) + "/test_dir") -rename_dir_path = Path(str(Path.cwd()) + "/rename_dir") -first_file_dir = Path(str(Path.cwd()) + "/first_file_dir") -second_file_dir = Path(str(Path.cwd()) + "/second_file_dir") - - -def test_create_dir(): - create_dir(str(copy_dir_path)) - - -def test_copy_dir(): - copy_dir(str(first_file_dir), str(copy_dir_path)) - - -def test_rename_dir(): - rename_dir(str(copy_dir_path), str(rename_dir_path)) - - -def test_remove_dir_tree(): - remove_dir_tree(str(rename_dir_path)) - - -def test(): - test_copy_dir() - test_rename_dir() - test_remove_dir_tree() - - -if __name__ == "__main__": - test() diff --git a/tests/unit_test/local/dir/first_file_dir/test_file b/tests/unit_test/local/dir/first_file_dir/test_file deleted file mode 100644 index 30d74d2..0000000 --- a/tests/unit_test/local/dir/first_file_dir/test_file +++ /dev/null @@ -1 +0,0 @@ -test \ No newline at end of file diff --git a/tests/unit_test/local/dir/second_file_dir/test_file.txt b/tests/unit_test/local/dir/second_file_dir/test_file.txt deleted file mode 100644 index 30d74d2..0000000 --- a/tests/unit_test/local/dir/second_file_dir/test_file.txt +++ /dev/null @@ -1 +0,0 @@ -test \ No newline at end of file diff --git a/tests/unit_test/local/file/test_file.py b/tests/unit_test/local/file/test_file.py deleted file mode 100644 index 73bf207..0000000 --- a/tests/unit_test/local/file/test_file.py +++ /dev/null @@ -1,53 +0,0 @@ -from pathlib import Path - -from automation_file import copy_file, copy_specify_extension_file, copy_all_file_to_dir, rename_file, remove_file -from automation_file import create_dir - -create_dir(str(Path.cwd()) + "/first_file_dir") -create_dir(str(Path.cwd()) + "/second_file_dir") -create_dir(str(Path.cwd()) + "/test_file") -first_file_dir = Path(str(Path.cwd()) + "/first_file_dir") -second_file_dir = Path(str(Path.cwd()) + "/second_file_dir") -test_file_dir = Path(str(Path.cwd()) + "/test_file") -test_file_path = Path(str(Path.cwd()) + "/test_file/test_file") - -with open(str(test_file_path), "w+") as file: - file.write("test") - -with open(str(test_file_path) + ".test", "w+") as file: - file.write("test") - -with open(str(test_file_path) + ".txt", "w+") as file: - file.write("test") - - -def test_copy_file(): - copy_file(str(test_file_path), str(first_file_dir)) - - -def test_copy_specify_extension_file(): - copy_specify_extension_file(str(test_file_dir), "txt", str(second_file_dir)) - - -def test_copy_all_file_to_dir(): - copy_all_file_to_dir(str(test_file_dir), str(first_file_dir)) - - -def test_rename_file(): - rename_file(str(test_file_dir), "rename", file_extension="txt") - - -def test_remove_file(): - remove_file(str(Path(test_file_dir, "rename"))) - - -def test(): - test_copy_file() - test_copy_specify_extension_file() - test_copy_all_file_to_dir() - test_rename_file() - test_remove_file() - - -if __name__ == "__main__": - test() diff --git a/tests/unit_test/local/zip/zip_test.py b/tests/unit_test/local/zip/zip_test.py deleted file mode 100644 index 075b592..0000000 --- a/tests/unit_test/local/zip/zip_test.py +++ /dev/null @@ -1,60 +0,0 @@ -from pathlib import Path - -from automation_file import create_dir -from automation_file import zip_dir, zip_file, read_zip_file, unzip_file, unzip_all, zip_info, zip_file_info, \ - set_zip_password - -zip_file_path = Path(Path.cwd(), "test.zip") -dir_to_zip = Path(Path.cwd(), "dir_to_zip") -file_to_zip = Path(Path.cwd(), "file_to_zip.txt") - -create_dir(str(dir_to_zip)) - -with open(str(file_to_zip), "w+") as file: - file.write("test") - - -def test_zip_dir(): - zip_dir(dir_we_want_to_zip=str(dir_to_zip), zip_name="test_generate") - - -def test_zip_file(): - zip_file(str(zip_file_path), str(file_to_zip)) - - -def test_read_zip_file(): - print(read_zip_file(str(zip_file_path), str(file_to_zip.name))) - - -def test_unzip_file(): - unzip_file(str(zip_file_path), str(file_to_zip.name)) - - -def test_unzip_all(): - unzip_all(str(zip_file_path)) - - -def test_zip_info(): - print(zip_info(str(zip_file_path))) - - -def test_zip_file_info(): - print(zip_file_info(str(zip_file_path))) - - -def test_set_zip_password(): - set_zip_password(str(zip_file_path), b"12345678") - - -def test(): - test_zip_dir() - test_zip_file() - test_read_zip_file() - test_unzip_file() - test_unzip_all() - test_zip_file_info() - test_set_zip_password() - - -if __name__ == "__main__": - test() diff --git a/tests/unit_test/remote/google_drive/quick_test.py b/tests/unit_test/remote/google_drive/quick_test.py deleted file mode 100644 index aa2fb52..0000000 --- a/tests/unit_test/remote/google_drive/quick_test.py +++ /dev/null @@ -1,7 +0,0 @@ -from pathlib import Path - -from automation_file import driver_instance -from automation_file.remote.google_drive.search.search_drive import drive_search_all_file - -driver_instance.later_init(str(Path(Path.cwd(), "token.json")), str(Path(Path.cwd(), "credentials.json"))) -print(drive_search_all_file())