Revert CI workflow (1 of 2)

.github/workflows/ci.yml

@@ -3,29 +3,12 @@ on:
   push:
     branches:
       - master
-    tags:
-      - '*'
   pull_request_target:
     branches:
       - master
 env:
-  CYPRESS_COVERAGE: "true"
   NODE_VERSION: 18
   YARN_VERSION: 1.22.22
-  REDASH_COOKIE_SECRET: 2H9gNG9obnAQ9qnR9BDTQUph6CbXKCzF
-  REDASH_SECRET_KEY: 2H9gNG9obnAQ9qnR9BDTQUph6CbXKCzF
-  COMPOSE_DOCKER_CLI_BUILD: 1
-  DOCKER_BUILDKIT: 1
-  FRONTEND_BUILD_MODE: 1
-  INSTALL_GROUPS: main,all_ds,dev
-  PERCY_BRANCH: ${{github.head_ref || github.ref_name}}
-  PERCY_COMMIT: ${{github.sha}}
-  PERCY_PULL_REQUEST: ${{github.event.number}}
-  COMMIT_INFO_BRANCH: ${{github.head_ref || github.ref_name}}
-  COMMIT_INFO_MESSAGE: ${{github.event.head_commit.message}}
-  COMMIT_INFO_AUTHOR: ${{github.event.pull_request.user.login}}
-  COMMIT_INFO_SHA: ${{github.sha}}
-  COMMIT_INFO_REMOTE: ${{github.server_url}}/${{github.repository}}
 jobs:
   backend-lint:
     runs-on: ubuntu-22.04
@@ -40,15 +23,18 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: '3.8'
-      - run: sudo pip install black==24.3.0 ruff==0.1.9
+      - run: sudo pip install black==23.1.0 ruff==0.0.287
       - run: ruff check .
       - run: black --check .
 
   backend-unit-tests:
     runs-on: ubuntu-22.04
     needs: backend-lint
     env:
-      FRONTEND_BUILD_MODE: 0
+      COMPOSE_FILE: .ci/compose.ci.yaml
+      COMPOSE_PROJECT_NAME: redash
+      COMPOSE_DOCKER_CLI_BUILD: 1
+      DOCKER_BUILDKIT: 1
     steps:
       - if: github.event.pull_request.mergeable == 'false'
         name: Exit if PR is not mergeable
@@ -60,34 +46,31 @@ jobs:
       - name: Build Docker Images
         run: |
           set -x
-          touch .env
-          docker compose build
+          docker compose build --build-arg install_groups="main,all_ds,dev" --build-arg skip_frontend_build=true
           docker compose up -d
           sleep 10
       - name: Create Test Database
-        run: docker compose run --rm postgres psql -h postgres -U postgres -c "create database tests;"
+        run: docker compose -p redash run --rm postgres psql -h postgres -U postgres -c "create database tests;"
       - name: List Enabled Query Runners
-        run: docker compose run --rm server manage ds list_types
+        run: docker compose -p redash run --rm redash manage ds list_types
       - name: Run Tests
-        run: docker compose run --name tests server tests --junitxml=junit.xml --cov-report=xml --cov=redash --cov-config=.coveragerc tests/
+        run: docker compose -p redash run --name tests redash tests --junitxml=junit.xml --cov-report=xml --cov=redash --cov-config=.coveragerc tests/
       - name: Copy Test Results
         run: |
           mkdir -p /tmp/test-results/unit-tests
           docker cp tests:/app/coverage.xml ./coverage.xml
           docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
       - name: Store Test Results
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
-          name: backend-test-results
+          name: test-results
           path: /tmp/test-results
       - name: Store Coverage Results
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
-          name: backend-coverage
+          name: coverage
           path: coverage.xml
 
   frontend-lint:
@@ -107,14 +90,13 @@ jobs:
       - name: Install Dependencies
         run: |
           npm install --global --force yarn@$YARN_VERSION
-          yarn cache clean
-          yarn --frozen-lockfile --network-concurrency 1
+          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
       - name: Run Lint
         run: yarn lint:ci
       - name: Store Test Results
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
-          name: frontend-test-results
+          name: test-results
           path: /tmp/test-results
 
   frontend-unit-tests:
@@ -135,24 +117,21 @@ jobs:
       - name: Install Dependencies
         run: |
           npm install --global --force yarn@$YARN_VERSION
-          yarn cache clean
-          yarn --frozen-lockfile --network-concurrency 1
+          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
       - name: Run App Tests
         run: yarn test
       - name: Run Visualizations Tests
-        run: |
-          cd viz-lib
-          yarn test
+        run: cd viz-lib && yarn test
       - run: yarn lint
 
   frontend-e2e-tests:
     runs-on: ubuntu-22.04
     needs: frontend-lint
     env:
+      COMPOSE_FILE: .ci/compose.cypress.yaml
+      COMPOSE_PROJECT_NAME: cypress
       CYPRESS_INSTALL_BINARY: 0
       PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1
-      INSTALL_GROUPS: main
-      COMPOSE_PROFILES: e2e
       PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
       CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
       CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
@@ -168,16 +147,17 @@ jobs:
         with:
           node-version: ${{ env.NODE_VERSION }}
           cache: 'yarn'
+      - name: Enable Code Coverage Report For Master Branch
+        if: endsWith(github.ref, '/master')
+        run: |
+          echo "CODE_COVERAGE=true" >> "$GITHUB_ENV"
       - name: Install Dependencies
         run: |
           npm install --global --force yarn@$YARN_VERSION
-          yarn cache clean
-          yarn --frozen-lockfile --network-concurrency 1
+          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
       - name: Setup Redash Server
         run: |
           set -x
-          touch .env
-          yarn build
           yarn cypress build
           yarn cypress start -- --skip-db-seed
           docker compose run cypress yarn cypress db-seed
@@ -189,12 +169,7 @@ jobs:
       - name: Copy Code Coverage Results
         run: docker cp cypress:/usr/src/app/coverage ./coverage || true
       - name: Store Coverage Results
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
-          name: frontend-coverage
+          name: coverage
           path: coverage
-      - uses: actions/upload-artifact@v4
-        with:
-          name: frontend
-          path: client/dist
-          retention-days: 1

.github/workflows/preview-image.yml

@@ -1,20 +1,15 @@
 name: Preview Image
 on:
-  workflow_run:
-    workflows: 
-      - Tests
-    types:
-      - completed
-    branches:
-      - master
+  push:
+    tags:
+      - '*-dev'
 
 env:
-  DOCKER_REPO: redash
+  NODE_VERSION: 18
 
 jobs:
   build-skip-check:
     runs-on: ubuntu-22.04
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     outputs:
       skip: ${{ steps.skip-check.outputs.skip }}
     steps:
@@ -37,118 +32,56 @@ jobs:
     runs-on: ubuntu-22.04
     needs:
       - build-skip-check
-    outputs:
-      version: ${{ steps.version.outputs.VERSION_TAG }}
-      repo: ${{ steps.version.outputs.DOCKER_REPO }}
     if: needs.build-skip-check.outputs.skip == 'false'
-    strategy:
-      fail-fast: false
-      matrix:
-        platform:
-          - linux/amd64
-          - linux/arm64
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 1
           ref: ${{ github.event.push.after }}
-      - uses: dawidd6/action-download-artifact@v3
+
+      - uses: actions/setup-node@v4
         with:
-          name: frontend
-          workflow: ci.yml
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          run_id: ${{ github.event.workflow_run.id }}
-          path: client/dist
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'yarn'
+
+      - name: Install Dependencies
+        run: |
+          npm install --global --force yarn@1.22.22
+          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+
       - name: Login to DockerHub
         uses: docker/login-action@v3
         with:
           username: ${{ vars.DOCKER_USER }}
           password: ${{ secrets.DOCKER_PASS }}
+
       - name: Set version
         id: version
         run: |
           set -x
-          VERSION=$(jq -r .version package.json)
-          FULL_VERSION=${VERSION}-b${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER}
-          sed -ri "s/^__version__ = ([A-Za-z0-9.-]*)'/__version__ = '${FULL_VERSION}'/" redash/__init__.py
-          sed -i "s/dev/${GITHUB_SHA}/" client/app/version.json
-          echo "VERSION_TAG=$FULL_VERSION" >> "$GITHUB_OUTPUT"
-          platform=${{ matrix.platform }}
-          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
-          echo "SCOPE=${platform//\//-}" >> $GITHUB_ENV
-          if [[ "${{ vars.DOCKER_REPO }}" != "" ]]; then
-            echo "DOCKER_REPO=${{ vars.DOCKER_REPO }}" >> $GITHUB_ENV
-            echo "DOCKER_REPO=${{ vars.DOCKER_REPO }}" >> $GITHUB_OUTPUT
-          else
-            echo "DOCKER_REPO=${DOCKER_REPO}" >> $GITHUB_ENV
-            echo "DOCKER_REPO=${DOCKER_REPO}" >> $GITHUB_OUTPUT
-          fi
+          .ci/update_version
+          VERSION_TAG=$(jq -r .version package.json)
+          echo "VERSION_TAG=$VERSION_TAG" >> "$GITHUB_OUTPUT"
+
       - name: Build and push preview image to Docker Hub
-        uses: docker/build-push-action@v5
-        id: build
+        uses: docker/build-push-action@v4
         with:
           push: true
+          tags: |
+            redash/redash:preview
+            redash/preview:${{ steps.version.outputs.VERSION_TAG }}
           context: .
-          cache-from: type=gha,scope=${{ env.SCOPE }}
-          cache-to: type=gha,mode=max,scope=${{ env.SCOPE }}
-          platforms: ${{ matrix.platform }}
-          outputs: type=image,name=${{ env.DOCKER_REPO }}/redash,push-by-digest=true,name-canonical=true,push=true
           build-args: |
-            FRONTEND_BUILD_MODE=1
+            test_all_deps=true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          platforms: linux/amd64
         env:
           DOCKER_CONTENT_TRUST: true
-      - name: Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-      - name: Upload digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: digests-${{ env.PLATFORM_PAIR }}
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
 
-  publish-docker-manifest:
-    runs-on: ubuntu-22.04
-    needs:
-      - build-skip-check
-      - build-docker-image
-    if: needs.build-skip-check.outputs.skip == 'false'
-    steps:
-      - name: Download digests
-        uses: actions/download-artifact@v4
-        with:
-          pattern: digests-*
-          path: /tmp/digests
-          merge-multiple: true
-      - name: Setup Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ needs.build-docker-image.outputs.repo }}/redash
-          tags: preview
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ vars.DOCKER_USER }}
-          password: ${{ secrets.DOCKER_PASS }}
-      - name: Create manifest list and push
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-          $(printf '${{ needs.build-docker-image.outputs.repo }}/redash@sha256:%s ' *)
-      - name: Inspect image
-        run: |
-          REDASH_IMAGE="${{ needs.build-docker-image.outputs.repo }}/redash:${{ steps.meta.outputs.version }}"
-          docker buildx imagetools inspect $REDASH_IMAGE
-      - name: Push image ${{ needs.build-docker-image.outputs.repo }}/preview image
-        run: |
-          REDASH_IMAGE="${{ needs.build-docker-image.outputs.repo }}/redash:preview"
-          PREVIEW_IMAGE="${{ needs.build-docker-image.outputs.repo }}/preview:${{ needs.build-docker-image.outputs.version }}"
-          docker buildx imagetools create --tag $PREVIEW_IMAGE $REDASH_IMAGE
+      - name: "Failure: output container logs to console"
+        if: failure()
+        run: docker compose logs