feat(engine+ui): Replace lancedb with postgres #114
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy to AWS | |
| on: | |
| push: | |
| branches: | |
| - "main" | |
| tags: | |
| - "*.*.*" | |
| workflow_dispatch: | |
| pull_request: | |
| branches: ["main"] | |
| paths: | |
| - aws/stack/* | |
| - aws/app.py | |
| - .github/workflows/deploy-to-aws.yml | |
| permissions: | |
| contents: read | |
| packages: write | |
| jobs: | |
| create-secrets: | |
| name: Create AWS Secrets | |
| runs-on: ubuntu-latest | |
| environment: ${{ startsWith(github.ref, 'refs/tags/') && 'production' || 'preview' }} | |
| permissions: | |
| id-token: write # This line allows GitHub Actions to request an OIDC token | |
| contents: read | |
| steps: | |
| - name: Checkout 🛎️ | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: ${{ secrets.AWS_DEFAULT_REGION }} | |
| role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} | |
| role-session-name: GitHubActions | |
| role-duration-seconds: 3600 | |
| - name: Create postgres secrets | |
| continue-on-error: true | |
| run: | | |
| # Create URL-safe postgres passwords | |
| TRACECAT__DB_PASS=$(openssl rand -base64 32 | tr -d '/+' | cut -c1-32) | |
| TEMPORAL__DB_PASS=$(openssl rand -base64 32 | tr -d '/+' | cut -c1-32) | |
| echo "::add-mask::${TRACECAT__DB_PASS}" | |
| echo "::add-mask::${TEMPORAL__DB_PASS}" | |
| # Create postgres secrets | |
| aws secretsmanager create-secret --name ${{ vars.TEMPORAL_DB_PASS_NAME }} --secret-string "${TRACECAT__DB_PASS}" --region ${{ secrets.AWS_DEFAULT_REGION }} | |
| aws secretsmanager create-secret --name ${{ vars.TRACECAT_DB_PASS_NAME }} --secret-string "${TEMPORAL__DB_PASS}" --region ${{ secrets.AWS_DEFAULT_REGION }} | |
| - name: Create encryption keys | |
| continue-on-error: true | |
| run: | | |
| bash env.sh | |
| # Get DB encryption key from .env file | |
| DB_ENCRYPTION_KEY=$(grep TRACECAT__DB_ENCRYPTION_KEY .env | cut -d '=' -f2) | |
| # Get service key | |
| SERVICE_KEY=$(grep TRACECAT__SERVICE_KEY .env | cut -d '=' -f2) | |
| # Get signing secret | |
| SIGNING_SECRET=$(grep TRACECAT__SIGNING_SECRET .env | cut -d '=' -f2) | |
| # Mask secrets | |
| echo "::add-mask::${DB_ENCRYPTION_KEY}" | |
| echo "::add-mask::${SERVICE_KEY}" | |
| echo "::add-mask::${SIGNING_SECRET}" | |
| # Create secrets | |
| aws secretsmanager create-secret --name ${{ vars.DB_ENCRYPTION_KEY_NAME }} --secret-string "${DB_ENCRYPTION_KEY}" --region ${{ secrets.AWS_DEFAULT_REGION }} | |
| aws secretsmanager create-secret --name ${{ vars.SERVICE_KEY_NAME }} --secret-string "${SERVICE_KEY}" --region ${{ secrets.AWS_DEFAULT_REGION }} | |
| aws secretsmanager create-secret --name ${{ vars.SIGNING_SECRET_NAME }} --secret-string "${SIGNING_SECRET}" --region ${{ secrets.AWS_DEFAULT_REGION }} | |
| - name: Create Clerk secrets | |
| continue-on-error: true | |
| run: | | |
| # Create secrets | |
| aws secretsmanager create-secret --name ${{ vars.CLERK_SECRET_KEY_NAME }} --secret-string ${{ secrets.CLERK_SECRET_KEY }} --region ${{ secrets.AWS_DEFAULT_REGION }} | |
| push-ui-to-ecr: | |
| name: Build and push UI Docker image to ECR | |
| runs-on: ubuntu-latest | |
| environment: ${{ startsWith(github.ref, 'refs/tags/') && 'production' || 'preview' }} | |
| permissions: | |
| id-token: write # This line allows GitHub Actions to request an OIDC token | |
| contents: read | |
| steps: | |
| - name: Checkout 🛎️ | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: ${{ secrets.AWS_DEFAULT_REGION }} | |
| role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} | |
| role-session-name: GitHubActionsECR | |
| role-duration-seconds: 3600 | |
| - name: Get SHA | |
| id: get-sha | |
| run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| - name: Build, tag, and push docker image to Amazon ECR | |
| env: | |
| REGISTRY: ${{ steps.login-ecr.outputs.registry }} | |
| REPOSITORY: tracecat-ui | |
| IMAGE_TAG: ${{ steps.get-sha.outputs.sha }} | |
| NEXT_PUBLIC_API_URL: ${{ vars.API_URL }} | |
| NEXT_PUBLIC_APP_ENV: production | |
| NEXT_PUBLIC_APP_URL: ${{ vars.APP_URL }} | |
| NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: ${{ secrets.NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY }} | |
| NEXT_PUBLIC_DISABLE_AUTH: "false" # Enable Clerk | |
| NEXT_SERVER_API_URL: http://api-service:8000 | |
| NODE_ENV: production | |
| run: | | |
| # Mask secrets | |
| echo "::add-mask::${NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY}" | |
| # Docker build and push | |
| docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG -f frontend/Dockerfile.prod frontend \ | |
| --build-arg NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL \ | |
| --build-arg NEXT_PUBLIC_APP_ENV=$NEXT_PUBLIC_APP_ENV \ | |
| --build-arg NEXT_PUBLIC_APP_URL=$NEXT_PUBLIC_APP_URL \ | |
| --build-arg NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=$NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY \ | |
| --build-arg NEXT_PUBLIC_DISABLE_AUTH=$NEXT_PUBLIC_DISABLE_AUTH \ | |
| --build-arg NEXT_SERVER_API_URL=$NEXT_SERVER_API_URL \ | |
| --build-arg NODE_ENV=$NODE_ENV | |
| docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG | |
| deploy: | |
| name: Deploy AWS CDK Stack | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| environment: ${{ startsWith(github.ref, 'refs/tags/') && 'production' || 'preview' }} | |
| needs: [create-secrets, push-ui-to-ecr] | |
| env: | |
| CDK_VERSION: 2.147 | |
| permissions: | |
| id-token: write # This line allows GitHub Actions to request an OIDC token | |
| contents: read | |
| steps: | |
| - name: Checkout 🛎️ | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: ${{ secrets.AWS_DEFAULT_REGION }} | |
| role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} | |
| role-session-name: GitHubActions | |
| role-duration-seconds: 3600 | |
| - name: Install CDK CLI | |
| run: npm install -g aws-cdk@${{ env.CDK_VERSION }} && cdk --version | |
| - name: Install Python CDK | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install aws-cdk-lib==${{ env.CDK_VERSION }} | |
| - name: Get SHA | |
| id: get-sha | |
| run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT | |
| - name: Get image tag | |
| id: set-image-tag | |
| run: | | |
| if [ "${{ github.event_name }}" == "push" ]; then | |
| if [ "${{ github.ref }}" == "refs/heads/main" ]; then | |
| echo "TRACECAT__IMAGE_TAG=main" >> $GITHUB_ENV | |
| elif [[ "${{ github.ref }}" == refs/tags/* ]]; then | |
| version_tag=${{ github.ref }} | |
| version_tag=${version_tag#refs/tags/} | |
| echo "TRACECAT__IMAGE_TAG=${version_tag}" >> $GITHUB_ENV | |
| fi | |
| elif [ "${{ github.event_name }}" == "pull_request" ]; then | |
| echo "TRACECAT__IMAGE_TAG=main" >> $GITHUB_ENV | |
| else | |
| echo "TRACECAT__IMAGE_TAG=main" >> $GITHUB_ENV | |
| fi | |
| - name: CDK synth, bootstrap and deploy | |
| env: | |
| # General | |
| LOG_LEVEL: INFO | |
| AWS_DEFAULT_ACCOUNT: ${{ secrets.AWS_ACCOUNT_ID }} | |
| AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} | |
| # "main" if trigger on main, else use latest tag | |
| TRACECAT__IMAGE_TAG: ${{ env.TRACECAT__IMAGE_TAG}} | |
| # Tag with git SHA | |
| TRACECAT__UI_IMAGE_TAG: ${{ steps.get-sha.outputs.sha }} | |
| # Backend | |
| TRACECAT__ALLOW_ORIGINS: ${{ vars.TRACECAT__ALLOW_ORIGINS }} | |
| TRACECAT__APP_ENV: production | |
| TRACECAT__API_URL: http://api-service:8000 | |
| TRACECAT__APP_URL: ${{ vars.APP_URL }} | |
| TRACECAT__DISABLE_AUTH: "false" | |
| TRACECAT__PUBLIC_RUNNER_URL: ${{ vars.API_URL }} | |
| # Frontend | |
| NEXT_PUBLIC_API_URL: ${{ vars.API_URL }} | |
| NEXT_PUBLIC_APP_ENV: production | |
| NEXT_PUBLIC_APP_URL: ${{ vars.APP_URL }} | |
| NEXT_PUBLIC_DISABLE_AUTH: "false" | |
| NEXT_SERVER_API_URL: http://api-service:8000 | |
| NODE_ENV: production | |
| # Frontend Clerk Auth | |
| NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: ${{ secrets.NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY }} | |
| NEXT_PUBLIC_CLERK_SIGN_IN_URL: /sign-in | |
| NEXT_PUBLIC_CLERK_SIGN_OUT_URL: /sign-out | |
| # Temporal | |
| TEMPORAL__CLUSTER_URL: http://temporal-service:7233 | |
| TEMPORAL__CLUSTER_QUEUE: tracecat-task-queue | |
| TEMPORAL__CLUSTER_NAMESPACE: default | |
| TEMPORAL__VERSION: 1.24.2 | |
| TEMPORAL__UI_VERSION: 2.26.2 | |
| # Posthog | |
| NEXT_PUBLIC_POSTHOG_KEY: ${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }} | |
| NEXT_PUBLIC_POSTHOG_HOST: https://us.i.posthog.com | |
| NEXT_PUBLIC_DISABLE_SESSION_RECORDING: false | |
| # AWS Secret names | |
| DB_ENCRYPTION_KEY_NAME: ${{ vars.DB_ENCRYPTION_KEY_NAME }} | |
| SERVICE_KEY_NAME: ${{ vars.SERVICE_KEY_NAME }} | |
| SIGNING_SECRET_NAME: ${{ vars.SIGNING_SECRET_NAME }} | |
| CLERK_FRONTEND_API_URL: ${{ vars.CLERK_FRONTEND_API_URL }} | |
| CLERK_SECRET_KEY_NAME: ${{ vars.CLERK_SECRET_KEY_NAME }} | |
| TRACECAT_DB_PASS_NAME: ${{ vars.TRACECAT_DB_PASS_NAME }} | |
| TEMPORAL_DB_PASS_NAME: ${{ vars.TEMPORAL_DB_PASS_NAME }} | |
| # DNS settings | |
| CERTIFICATE_ARN: ${{ secrets.CERTIFICATE_ARN }} | |
| API_CERTIFICATE_ARN: ${{ secrets.API_CERTIFICATE_ARN }} | |
| # CPU / RAM settings | |
| TRACECAT__API_CPU: ${{ startsWith(github.ref, 'refs/tags/') && 1024 || 256 }} | |
| TRACECAT__API_RAM: ${{ startsWith(github.ref, 'refs/tags/') && 3072 || 512 }} | |
| TRACECAT__WORKER_CPU: ${{ startsWith(github.ref, 'refs/tags/') && 2048 || 512 }} | |
| TRACECAT__WORKER_RAM: ${{ startsWith(github.ref, 'refs/tags/') && 4096 || 1024 }} | |
| TRACECAT__UI_CPU: ${{ startsWith(github.ref, 'refs/tags/') && 2048 || 512 }} | |
| TRACECAT__UI_RAM: ${{ startsWith(github.ref, 'refs/tags/') && 4096 || 1024 }} | |
| TEMPORAL__SERVER_CPU: ${{ startsWith(github.ref, 'refs/tags/') && 2048 || 512 }} | |
| TEMPORAL__SERVER_RAM: ${{ startsWith(github.ref, 'refs/tags/') && 4096 || 1024 }} | |
| # CIDR blocks (allow all IP addresses) | |
| ALB_ALLOWED_CIDR_BLOCKS: "0.0.0.0/0" | |
| run: | | |
| # Mask secrets | |
| echo "::add-mask::${AWS_DEFAULT_ACCOUNT}" | |
| echo "::add-mask::${AWS_DEFAULT_REGION}" | |
| echo "::add-mask::${NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY}" | |
| echo "::add-mask::${POSTHOG_KEY}" | |
| echo "::add-mask::${DB_ENCRYPTION_KEY_NAME}" | |
| echo "::add-mask::${SERVICE_KEY_NAME}" | |
| echo "::add-mask::${SIGNING_SECRET_NAME}" | |
| echo "::add-mask::${CLERK_SECRET_KEY_NAME}" | |
| echo "::add-mask::${TRACECAT_DB_PASS_NAME}" | |
| echo "::add-mask::${TEMPORAL_DB_PASS_NAME}" | |
| echo "::add-mask::${CERTIFICATE_ARN}" | |
| echo "::add-mask::${API_CERTIFICATE_ARN}" | |
| cd aws | |
| pip install . | |
| cdk synth --app "python3 app.py" | |
| cdk bootstrap --app cdk.out | |
| cdk deploy --app cdk.out --all --require-approval never |