Improvements

.dockerignore

@@ -0,0 +1,22 @@
+# The NiFi image only copies pyproject.toml and the nifi source tree.
+# Keep the build context small and prevent unrelated repository content,
+# credentials, and local development state from entering image layers.
+**
+!pyproject.toml
+!uv.lock
+!nifi/
+!nifi/**
+
+# Local NiFi runtime state. These files may contain environment-specific
+# endpoints, identifiers, or other data and are not reproducible inputs.
+nifi/conf/flow.json.gz
+nifi/conf/archive/
+
+# Development and operating-system artefacts within the included source tree.
+nifi/**/__pycache__/
+nifi/**/*.py[cod]
+nifi/**/.DS_Store
+nifi/**/.pytest_cache/
+nifi/**/.ruff_cache/
+nifi/**/.mypy_cache/
+nifi/**/*.egg-info/

.github/dependabot.yml

@@ -6,17 +6,29 @@ updates:
     schedule:
       interval: "weekly"
 
-  # Keep Dockerfile base images (in repo root) up to date
+  # Keep the owned NiFi Dockerfile base image up to date.
   - package-ecosystem: "docker"
-    directory: "/"
+    directory: "/nifi"
     schedule:
       interval: "weekly"
     registries:
       - dockerhub
 
+  # Keep the root Python project and lock file up to date.
+  - package-ecosystem: "uv"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  # Keep the documentation project and lock file up to date.
+  - package-ecosystem: "uv"
+    directory: "/docs"
+    schedule:
+      interval: "weekly"
+
 registries:
   dockerhub:
     type: "docker-registry"
     url: "https://index.docker.io"
     username: "${{secrets.DOCKER_HUB_USERNAME}}"
-    password: "${{secrets.DOCKER_HUB_ACCESS_TOKEN}}"
+    password: "${{secrets.DOCKER_HUB_ACCESS_TOKEN}}"

.github/workflows/_docker_template.yml

@@ -125,20 +125,40 @@ jobs:
             type=ref,event=branch,enable=${{ github.event_name != 'pull_request' }}
             type=sha,format=short
 
-      - name: Wait for Docker Hub to register images
-        run: sleep 60
-
       - name: Create multi-arch manifest
         shell: bash
         run: |
           set -euo pipefail
+
+          wait_for_image() {
+            local image_ref="$1"
+            local attempt
+
+            for attempt in {1..12}; do
+              if docker buildx imagetools inspect "$image_ref" >/dev/null 2>&1; then
+                echo "Available: ${image_ref}"
+                return 0
+              fi
+
+              if [[ "$attempt" -lt 12 ]]; then
+                echo "Waiting for ${image_ref} (${attempt}/12)..."
+                sleep 5
+              fi
+            done
+
+            echo "Timed out waiting for ${image_ref}" >&2
+            return 1
+          }
+
           while IFS= read -r ref; do
             [[ -z "$ref" ]] && continue
             img="${ref%%:*}"
             tag="${ref#*:}"
+            wait_for_image "${img}:${tag}-amd64"
+            wait_for_image "${img}:${tag}-arm64"
             echo "🌀 Creating manifest for ${img}:${tag}"
             docker buildx imagetools create \
               --tag "${img}:${tag}" \
               "${img}:${tag}-amd64" \
               "${img}:${tag}-arm64"
-          done < <(printf "%s" "${{ steps.meta_manifest.outputs.tags }}")
+          done < <(printf "%s\n" "${{ steps.meta_manifest.outputs.tags }}")

.github/workflows/codeql.yml

@@ -19,6 +19,10 @@ on:
   schedule:
     - cron: '35 5 * * 4'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   analyze:
     name: Analyze (${{ matrix.language }})

.github/workflows/compose_config.yml

@@ -0,0 +1,41 @@
+name: compose-config
+
+on:
+  push:
+    branches: ['**']
+    paths:
+      - "deploy/*.env"
+      - "deploy/export_env_vars.sh"
+      - "deploy/services*.yml"
+      - "security/env/*.env"
+      - ".github/workflows/compose_config.yml"
+  pull_request:
+    paths:
+      - "deploy/*.env"
+      - "deploy/export_env_vars.sh"
+      - "deploy/services*.yml"
+      - "security/env/*.env"
+      - ".github/workflows/compose_config.yml"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    name: Validate Docker Compose configuration
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Validate Compose files
+        shell: bash
+        run: |
+          set -euo pipefail
+          source deploy/export_env_vars.sh
+          docker compose -f deploy/services.yml config --quiet
+          docker compose -f deploy/services.dev.yml config --quiet

.github/workflows/dependency_hygiene.yml

@@ -4,11 +4,27 @@ on:
   workflow_dispatch:
   schedule:
     - cron: "0 6 * * 1" # weekly Monday UTC
+  push:
+    branches: ['**']
+    paths:
+      - "docs/pyproject.toml"
+      - "docs/uv.lock"
+      - "pyproject.toml"
+      - "uv.lock"
+      - "scripts/maintenance/audit_pip_dependency.sh"
+      - ".github/workflows/dependency_hygiene.yml"
   pull_request:
     paths:
-      - "docs/requirements.txt"
-      - "nifi/requirements.txt"
+      - "docs/pyproject.toml"
+      - "docs/uv.lock"
+      - "pyproject.toml"
+      - "uv.lock"
       - "scripts/maintenance/audit_pip_dependency.sh"
+      - ".github/workflows/dependency_hygiene.yml"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 permissions:
   contents: read
@@ -23,7 +39,7 @@ jobs:
         uses: actions/cache@v6
         with:
           path: ~/.cache/pip
-          key: pip-${{ runner.os }}-${{ hashFiles('docs/requirements.txt', 'nifi/requirements.txt', 'scripts/maintenance/audit_pip_dependency.sh') }}
+          key: pip-${{ runner.os }}-${{ hashFiles('docs/pyproject.toml', 'docs/uv.lock', 'pyproject.toml', 'uv.lock', 'scripts/maintenance/audit_pip_dependency.sh') }}
           restore-keys: |
             pip-${{ runner.os }}-
 
@@ -32,7 +48,12 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Install pip-audit
+      - name: Install uv
+        uses: astral-sh/setup-uv@v10.0.1
+        with:
+          version-file: "pyproject.toml"
+
+      - name: Install audit tools
         run: python -m pip install --upgrade pip pip-audit
 
       - name: Ensure script is executable

.github/workflows/doc_build.yml

@@ -2,15 +2,18 @@ name: doc-build
 
 on:
   push:
-    branches: [ main ]
+    branches: ['**']
   pull_request:
-    branches: [ main ]
 
   release:
     types: [published]
 
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
 
@@ -22,8 +25,9 @@ jobs:
       - uses: actions/checkout@v7
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v10.0.1
         with:
+          version-file: "pyproject.toml"
           python-version: "3.12"
           enable-cache: true
           cache-dependency-glob: "docs/uv.lock"
@@ -39,4 +43,4 @@ jobs:
 
       - name: Build docs
         working-directory: docs
-        run: uv run mkdocs build -f ../mkdocs.yml
+        run: uv run mkdocs build --strict -f ../mkdocs.yml

.github/workflows/python_quality.yml

@@ -5,23 +5,25 @@ on:
     branches: ['**']
     paths:
       - "pyproject.toml"
-      - "nifi/requirements.txt"
-      - "nifi/requirements-dev.txt"
+      - "uv.lock"
       - "nifi/**/*.py"
       - "scripts/**/*.py"
       - "typings/**/*.pyi"
       - ".github/workflows/python_quality.yml"
   pull_request:
     paths:
       - "pyproject.toml"
-      - "nifi/requirements.txt"
-      - "nifi/requirements-dev.txt"
+      - "uv.lock"
       - "nifi/**/*.py"
       - "scripts/**/*.py"
       - "typings/**/*.pyi"
       - ".github/workflows/python_quality.yml"
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
 
@@ -39,17 +41,18 @@ jobs:
           python-version: "3.11"
 
       - name: Install uv
-        run: |
-          python -m pip install --upgrade pip uv
+        uses: astral-sh/setup-uv@v10.0.1
+        with:
+          version-file: "pyproject.toml"
 
-      - name: Install requirements with uv
+      - name: Install locked project dependencies
         run: |
-          uv pip install --system -r nifi/requirements.txt -r nifi/requirements-dev.txt
+          uv sync --locked --dev
 
       - name: Ruff (critical rules)
         run: |
-          python -m ruff check scripts nifi/user_python_extensions --select F,E9
+          uv run --locked ruff check scripts nifi/user_python_extensions --select F,E9
 
       - name: MyPy
         run: |
-          python -m mypy --config-file pyproject.toml nifi/user_python_extensions scripts
+          uv run --locked mypy --config-file pyproject.toml nifi/user_python_extensions scripts

.github/workflows/shell_script_enforce_exec_check.yml

@@ -5,6 +5,10 @@ on:
   push:
     branches: ['**']
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
 
@@ -40,3 +44,39 @@ jobs:
             echo "Fix with: git update-index --chmod=+x <file>"
             exit 1
           fi
+
+  shell-quality:
+    name: ShellCheck and syntax
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Check root-owned shell scripts
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          scripts=()
+          while IFS= read -r -d '' file; do
+            scripts+=("$file")
+          done < <(
+            git ls-files -z -- '*.sh' \
+              ':(exclude)services/**' \
+              ':(exclude)deploy/charts/**'
+          )
+
+          if [[ "${#scripts[@]}" -eq 0 ]]; then
+            echo "No root-owned shell scripts found."
+            exit 0
+          fi
+
+          echo "Checking ${#scripts[@]} root-owned shell scripts."
+          docker run --rm \
+            --volume "${GITHUB_WORKSPACE}:/mnt:ro" \
+            --workdir /mnt \
+            koalaman/shellcheck:v0.11.0@sha256:61862eba1fcf09a484ebcc6feea46f1782532571a34ed51fedf90dd25f925a8d \
+            "${scripts[@]}"
+
+          for file in "${scripts[@]}"; do
+            bash -n "$file"
+          done

.github/workflows/smoke_elastic_services.yml

@@ -6,6 +6,10 @@ on:
     branches: ['**']
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
 

.github/workflows/smoke_gitea_service.yml

@@ -6,6 +6,10 @@ on:
     branches: ['**']
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
 
@@ -24,10 +28,11 @@ jobs:
           java-version: '21'
 
       - name: Generate certificates
-        working-directory: security/scripts
         run: |
           set -euo pipefail
-          ./create_root_ca_cert.sh
+          make -C deploy init-security-gitea
+          sudo chown 1000:1000 security/certificates/gitea/gitea.key
+          sudo chmod 600 security/certificates/gitea/gitea.key
 
       - name: Start Gitea service
         run: |