From ac364bde8d75a874c9966e12a6c47df4249e687c Mon Sep 17 00:00:00 2001 From: cdxker Date: Fri, 30 May 2025 10:34:59 -0700 Subject: [PATCH] ci: refactor CI pipeline to use matrix builds with optimized images --- .github/workflows/push-server.yml | 1146 ++--------------------------- 1 file changed, 56 insertions(+), 1090 deletions(-) diff --git a/.github/workflows/push-server.yml b/.github/workflows/push-server.yml index 8ed8d674c0..4a1940cb66 100644 --- a/.github/workflows/push-server.yml +++ b/.github/workflows/push-server.yml @@ -1,8 +1,8 @@ name: Create Docker Images -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref }} - cancel-in-progress: true +# concurrency: +# group: ${{ github.workflow }}-${{ github.head_ref }} +# cancel-in-progress: true on: workflow_dispatch: @@ -14,1095 +14,63 @@ on: jobs: build_server: - name: Push No OCR image - runs-on: ${{ matrix.runner }} + name: Build and Push ${{ matrix.image }} Image + runs-on: blacksmith-32vcpu-ubuntu-2204 timeout-minutes: 60 strategy: matrix: include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - suffix: "" - - runner: blacksmith-8vcpu-ubuntu-2204-arm - platform: linux/arm64 - tag: latest-arm - suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/server - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.server - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - ingest_worker: - name: Push Ingest image - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/ingest - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.ingestion-worker - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - delete_worker: - name: Push Delete worker - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/delete_worker - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.delete-worker - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - csv_jsonl_worker: - name: Push CSV JSONL worker Image - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/csv-jsonl-worker - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.csv-jsonl-worker - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - file_worker: - name: Push File Worker image - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/file_worker - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.file-worker - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - group_worker: - name: Push Group worker - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/group_worker - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.group-worker - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - clone_qdrant: - name: Push Clone Qdrant script - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/clone_qdrant - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.clone-qdrant - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - sync_qdrant: - name: Push Sync Qdrant script - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/sync_qdrant - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.sync-qdrant - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - queue_migration: - name: Push Queue Migration Script - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/queue-migration - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.queue-migration - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - reindex-worker: - name: Push Reindex Worker - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/reindex-worker - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.reindex-worker - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - word-id-cronjob: - name: Push Word ID cronjob - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/word-id-cronjob - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.word-id-cronjob - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - word-worker: - name: Push Word Worker - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/word-worker - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.word-worker - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - bktree-worker: - name: Push BK-Tree Worker - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/bktree-worker - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.bktree-worker - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - crawl-worker: - name: Push Crawl Worker - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/crawl-worker - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.crawl-worker - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - crawl-cron-job: - name: Push Crawl Cron Job - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/crawl-cron-job - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.crawl-cron-job - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - dittofeed-sync-worker: - name: Push Dittofeed Sync Worker - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/dittofeed-sync-worker - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.dittofeed-sync-worker - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - pagefind-worker: - name: Push Pagefind Index worker - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v3 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/pagefind-worker - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.pagefind-worker - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - etl-worker: - name: Push ETL worker - runs-on: ${{ matrix.runner }} - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v3 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/etl-worker - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.etl-worker - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + - file: Dockerfile.server + image: trieve/server + - file: Dockerfile.ingestion-worker + image: trieve/ingest + - file: Dockerfile.delete-worker + image: trieve/delete_worker + - file: Dockerfile.csv-jsonl-worker + image: trieve/csv-jsonl-worker + - file: Dockerfile.file-worker + image: trieve/file_worker + - file: Dockerfile.group-worker + image: trieve/group_worker + - file: Dockerfile.clone-qdrant + image: trieve/clone_qdrant + - file: Dockerfile.sync-qdrant + image: trieve/sync_qdrant + - file: Dockerfile.queue-migration + image: trieve/queue-migration + - file: Dockerfile.reindex-worker + image: trieve/reindex-worker + - file: Dockerfile.word-id-cronjob + image: trieve/word-id-cronjob + - file: Dockerfile.word-worker + image: trieve/word-worker + - file: Dockerfile.bktree-worker + image: trieve/bktree-worker + - file: Dockerfile.crawl-worker + image: trieve/crawl-worker + - file: Dockerfile.crawl-cron-job + image: trieve/crawl-cron-job + - file: Dockerfile.dittofeed-sync-worker + image: trieve/dittofeed-sync-worker + - file: Dockerfile.pagefind-worker + image: trieve/pagefind-worker + - file: Dockerfile.etl-worker + image: trieve/etl-worker + - file: Dockerfile.update-worker + image: trieve/update-worker + - file: Dockerfile.send-stripe-billing + image: trieve/send-stripe-billing + - file: Dockerfile.backfill-qdrant-from-pg + image: trieve/backfill-qdrant + - file: Dockerfile.clone-qdrant-from-offset + image: trieve/clone-qdrant-from-offset + - file: Dockerfile.clone-dataset-worker + image: trieve/clone-dataset-worker - update-worker: - name: Push Update worker - runs-on: ${{ matrix.runner }} - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v3 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/update-worker - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.update-worker - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - send-stripe-billing: - name: Push Send Stripe Billing - runs-on: ${{ matrix.runner }} - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v3 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/send-stripe-billing - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.send-stripe-billing - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - backfill-qdrant-from-pg: - name: Push Backfill Qdrant script - runs-on: ${{ matrix.runner }} - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v3 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/backfill-qdrant - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.backfill-qdrant-from-pg - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - clone-qdrant-from-offset: - name: Push Clone From Offset - runs-on: ${{ matrix.runner }} - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm steps: - name: Checkout the repo uses: actions/checkout@v4 - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v3 - - - name: Setup buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - # list of Docker images to use as base name for tags - images: | - trieve/clone-qdrant-from-offset - tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} - - - name: Build and push Docker image - uses: useblacksmith/build-push-action@v1.0.0-beta - with: - platforms: ${{ matrix.platform }} - context: server/ - file: ./server/Dockerfile.clone-qdrant-from-offset - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - clone-dataset-worker: - name: Push Clone Dataset worker - runs-on: ${{ matrix.runner }} - strategy: - matrix: - include: - - runner: blacksmith-8vcpu-ubuntu-2204 - platform: linux/amd64 - tag: latest - # - runner: blacksmith-8vcpu-ubuntu-2204 - # platform: linux/arm64 - # tag: latest-arm - # suffix: -arm - steps: - - name: Checkout the repo - uses: actions/checkout@v4 - - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v3 - - name: Setup buildx uses: docker/setup-buildx-action@v3 @@ -1116,19 +84,17 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - # list of Docker images to use as base name for tags - images: | - trieve/clone-dataset-worker + images: ${{ matrix.image }} tags: | - type=raw,${{matrix.tag}} - type=sha,suffix=${{matrix.suffix}} + type=raw,latest + type=sha - name: Build and push Docker image uses: useblacksmith/build-push-action@v1.0.0-beta with: - platforms: ${{ matrix.platform }} + platforms: linux/amd64 context: server/ - file: ./server/Dockerfile.clone-dataset-worker + file: ./server/${{ matrix.file }} push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }}