feat(engine+ui): Replace lancedb with postgres (#250) #330
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| push: | |
| branches: ["main"] | |
| paths: | |
| - tracecat/** | |
| - pyproject.toml | |
| - .github/workflows/test.yml | |
| pull_request: | |
| branches: ["main"] | |
| paths: | |
| - tracecat/** | |
| - pyproject.toml | |
| - .github/workflows/test.yml | |
| permissions: | |
| contents: read | |
| packages: write | |
| jobs: | |
| push-to-ghcr: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract Docker metadata | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ghcr.io/TracecatHQ/tracecat | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| push-cli-to-ghcr: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract Docker metadata | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ghcr.io/TracecatHQ/tracecat-cli | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: Dockerfile.cli | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| pytest: | |
| runs-on: ubuntu-latest | |
| needs: [push-to-ghcr, push-cli-to-ghcr] | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: "pip" | |
| cache-dependency-path: pyproject.toml | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Download Temporal CLI | |
| run: | | |
| # Download the Temporal CLI archive | |
| curl -L -o temporal.tar.gz "https://temporal.download/cli/archive/latest?platform=linux&arch=amd64" | |
| # Create a directory for the Temporal CLI | |
| mkdir -p temporal-cli | |
| # Extract the archive | |
| tar -xzf temporal.tar.gz -C temporal-cli | |
| # Add the Temporal CLI binary to the PATH | |
| echo "${GITHUB_WORKSPACE}/temporal-cli" >> $GITHUB_PATH | |
| - name: Verify Temporal CLI installation | |
| run: temporal --version | |
| - name: Run environment setup script | |
| run: bash env.sh | |
| - name: Get image tag | |
| id: set-image-tag | |
| run: | | |
| if [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/main" ]; then | |
| echo "TRACECAT__IMAGE_TAG=main" >> $GITHUB_ENV | |
| else | |
| echo "TRACECAT__IMAGE_TAG=pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV | |
| fi | |
| - name: Start Docker services | |
| run: docker compose up --no-deps api worker postgres_db -d | |
| - name: Verify Tracecat API is running | |
| run: curl -s http://localhost:8000/health | jq -e '.status == "ok"' | |
| - name: pip install Tracecat | |
| run: python -m pip install --upgrade pip && pip install ".[dev]" && pip install ./cli | |
| - name: Start Temporal server | |
| run: nohup temporal server start-dev > temporal.log 2>&1 & | |
| - name: Run tests (headless mode) | |
| env: | |
| TRACECAT__IMAGE_TAG: ${{ env.TRACECAT__IMAGE_TAG }} | |
| LOG_LEVEL: WARNING | |
| run: pytest tests/unit --temporal-no-restart --tracecat-no-restart | |
| playbooks: | |
| runs-on: ubuntu-latest | |
| environment: QA | |
| needs: [push-to-ghcr, push-cli-to-ghcr] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: "pip" | |
| cache-dependency-path: pyproject.toml | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Run environment setup script | |
| run: bash env.sh | |
| - name: Get image tag | |
| id: set-image-tag | |
| run: | | |
| if [ "${{ github.event_name }}" == "push" ] && [ "${{ github.ref }}" == "refs/heads/main" ]; then | |
| echo "TRACECAT__IMAGE_TAG=main" >> $GITHUB_ENV | |
| else | |
| echo "TRACECAT__IMAGE_TAG=pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV | |
| fi | |
| - name: Start Docker services | |
| run: docker compose up --no-deps api worker postgres_db temporal -d | |
| - name: Verify Tracecat API is running | |
| run: curl -s http://localhost:8000/health | jq -e '.status == "ok"' | |
| - name: pip install Tracecat | |
| run: python -m pip install --upgrade pip && pip install ".[dev]" | |
| - name: Install CLI via Docker | |
| env: | |
| TRACECAT__IMAGE_TAG: ${{ env.TRACECAT__IMAGE_TAG }} | |
| run: | | |
| # Pull in CLI | |
| docker pull ghcr.io/tracecathq/tracecat-cli:$TRACECAT__IMAGE_TAG | |
| docker run --network host --rm ghcr.io/tracecathq/tracecat-cli --version | |
| - name: Mask sensitive data | |
| run: | | |
| echo "::add-mask::${{ secrets.INTEGRATION__ABUSECH_API_KEY }}" | |
| echo "::add-mask::${{ secrets.INTEGRATION__ABUSEIPDB_API_KEY }}" | |
| echo "::add-mask::${{ secrets.INTEGRATION__AWS_GUARDDUTY__ACCESS_KEY_ID }}" | |
| echo "::add-mask::${{ secrets.INTEGRATION__AWS_GUARDDUTY__SECRET_ACCESS_KEY }}" | |
| echo "::add-mask::${{ secrets.INTEGRATION__DD_API_KEY }}" | |
| echo "::add-mask::${{ secrets.INTEGRATION__DD_APP_KEY }}" | |
| echo "::add-mask::${{ secrets.INTEGRATION__HYBRID_ANALYSIS_API_KEY }}" | |
| echo "::add-mask::${{ secrets.INTEGRATION__OPENAI_API_KEY }}" | |
| echo "::add-mask::${{ secrets.INTEGRATION__PULSEDRIVE_API_KEY }}" | |
| echo "::add-mask::${{ secrets.INTEGRATION__RESEND_API_KEY }}" | |
| echo "::add-mask::${{ secrets.INTEGRATION__SLACK_BOT_TOKEN }}" | |
| echo "::add-mask::${{ secrets.INTEGRATION__SLACK_CHANNEL }}" | |
| echo "::add-mask::${{ secrets.INTEGRATION__URLSCAN_API_KEY }}" | |
| echo "::add-mask::${{ secrets.INTEGRATION__VT_API_KEY }}" | |
| - name: Run playbooks | |
| env: | |
| TRACECAT__IMAGE_TAG: ${{ env.TRACECAT__IMAGE_TAG }} | |
| ABUSECH_API_KEY: ${{ secrets.INTEGRATION__ABUSECH_API_KEY }} | |
| ABUSEIPDB_API_KEY: ${{ secrets.INTEGRATION__ABUSEIPDB_API_KEY }} | |
| AWS_GUARDDUTY__ACCESS_KEY_ID: ${{ secrets.INTEGRATION__AWS_GUARDDUTY__ACCESS_KEY_ID }} | |
| AWS_GUARDDUTY__SECRET_ACCESS_KEY: ${{ secrets.INTEGRATION__AWS_GUARDDUTY__SECRET_ACCESS_KEY }} | |
| DD_API_KEY: ${{ secrets.INTEGRATION__DD_API_KEY }} | |
| DD_APP_KEY: ${{ secrets.INTEGRATION__DD_APP_KEY }} | |
| HYBRID_ANALYSIS_API_KEY: ${{ secrets.INTEGRATION__HYBRID_ANALYSIS_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.INTEGRATION__OPENAI_API_KEY }} | |
| PULSEDRIVE_API_KEY: ${{ secrets.INTEGRATION__PULSEDRIVE_API_KEY }} | |
| RESEND_API_KEY: ${{ secrets.INTEGRATION__RESEND_API_KEY }} | |
| SLACK_BOT_TOKEN: ${{ secrets.INTEGRATION__SLACK_BOT_TOKEN }} | |
| SLACK_CHANNEL: ${{ secrets.INTEGRATION__SLACK_CHANNEL }} | |
| URLSCAN_API_KEY: ${{ secrets.INTEGRATION__URLSCAN_API_KEY }} | |
| VT_API_KEY: ${{ secrets.INTEGRATION__VT_API_KEY }} | |
| LOG_LEVEL: WARNING | |
| run: pytest -k "test_playbooks" --temporal-no-restart --tracecat-no-restart |