chore: realtime dashboard performance improvement #5356
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: dist scans | |
| on: | |
| pull_request: {} | |
| workflow_dispatch: {} | |
| merge_group: {} | |
| jobs: | |
| build-dist: | |
| runs-on: ${{ vars.LARGE_RUNNER_S || 'ubuntu-latest' }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| env: | |
| iam_role_to_assume: ${{ secrets.ROLE_ARN }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Configure AWS Credentials | |
| if: ${{ env.iam_role_to_assume != '' }} | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: ${{ env.iam_role_to_assume }} | |
| aws-region: us-east-1 | |
| - name: Login to Amazon ECR Public | |
| if: ${{ env.iam_role_to_assume != '' }} | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| with: | |
| registry-type: public | |
| - name: Test build dist | |
| run: |- | |
| chmod +x ./deployment/test-build-dist-1.sh | |
| ./deployment/test-build-dist-1.sh | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: templates-${{ github.event.pull_request.number }} | |
| path: | | |
| .github/ | |
| src/**/* | |
| test/** | |
| frontend/**/* | |
| sonar-project.properties | |
| deployment/global-s3-assets | |
| !deployment/global-s3-assets/**/asset.* | |
| cfn-lint-scan: | |
| name: cfn-lint scan | |
| needs: build-dist | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: templates-${{ github.event.pull_request.number }} | |
| path: templates | |
| - name: Setup Cloud Formation Linter with Latest Version | |
| uses: scottbrenner/cfn-lint-action@v2 | |
| - name: Print the Cloud Formation Linter Version & run Linter. | |
| id: cfn-lint | |
| run: | | |
| cfn-lint --version | |
| TEMPLATE_ROOT=templates/deployment/global-s3-assets/default | |
| cfn-lint -i W3005 W2001 W3045 W8003 -e -r us-east-1,ap-northeast-1 -t $TEMPLATE_ROOT/*.template.json | |
| cfn-lint -i W3005 W2001 W3045 W8003 -e -r ap-east-1 --ignore-templates $TEMPLATE_ROOT/data-reporting-quicksight-stack.template.json --ignore-templates $TEMPLATE_ROOT/*NewServerlessRedshift*.nested.template.json --ignore-templates $TEMPLATE_ROOT/data-pipeline-stack.template.json --ignore-templates $TEMPLATE_ROOT/datapipeline*.nested.template.json --ignore-templates $TEMPLATE_ROOT/cloudfront-s3-control-plane-stack-global* --ignore-templates $TEMPLATE_ROOT/*cognito-control-plane-stack.template.json --ignore-templates $TEMPLATE_ROOT/public-exist-vpc-custom-domain-control-plane-stack.template.json -t $TEMPLATE_ROOT/*.template.json | |
| TEMPLATE_ROOT=templates/deployment/global-s3-assets/cn | |
| cfn-lint -i W3005 W2001 W3045 W8003 -e -r cn-north-1,cn-northwest-1 --ignore-templates $TEMPLATE_ROOT/data-reporting-quicksight-stack.template.json --ignore-templates $TEMPLATE_ROOT/*NewServerlessRedshift*.nested.template.json --ignore-templates $TEMPLATE_ROOT/data-pipeline-stack.template.json --ignore-templates $TEMPLATE_ROOT/datapipeline*.nested.template.json --ignore-templates $TEMPLATE_ROOT/cloudfront-s3-control-plane-stack-global*.json --ignore-templates $TEMPLATE_ROOT/*cognito-control-plane-stack.template.json --ignore-templates $TEMPLATE_ROOT/public-exist-vpc-custom-domain-control-plane-stack.template.json --ignore-templates $TEMPLATE_ROOT/ingestionserver*.nested.template.json -t $TEMPLATE_ROOT/*.template.json | |
| cfn-nag-scan: | |
| name: cfn-nag scan | |
| needs: build-dist | |
| runs-on: ubuntu-latest | |
| container: | |
| image: stelligent/cfn_nag | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: templates-${{ github.event.pull_request.number }} | |
| path: templates | |
| - name: Run cfn-nag scan | |
| run: | | |
| set -xeuo pipefail | |
| cfn_nag -f -b .cfnnag_global_suppress_list templates/deployment/global-s3-assets/**/*.template.json | |
| post-build-scan: | |
| name: post-build scan | |
| needs: build-dist | |
| runs-on: ubuntu-latest | |
| env: | |
| SOLUTION_NAME: clickstream-analytics-on-aws | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: "8.15.3" | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: templates-${{ github.event.pull_request.number }} | |
| path: deployment/global-s3-assets | |
| - name: build package | |
| run: | | |
| cd deployment/ | |
| ./build-open-source-dist.sh $SOLUTION_NAME | |
| - name: install dependencies | |
| run: | | |
| pnpm install --frozen-lockfile | |
| pnpm projen | |
| pnpm install --frozen-lockfile --dir frontend | |
| - name: Post-build scan | |
| env: | |
| VIPERLIGHT_SCAN_STRATEGY: Enforce | |
| VIPERLIGHT_PUBCHECK_STRATEGY: Monitor | |
| run: |- | |
| curl https://viperlight-scanner.s3.us-east-1.amazonaws.com/latest/.viperlightrc -o .viperlightrc | |
| curl https://viperlight-scanner.s3.us-east-1.amazonaws.com/latest/viperlight.zip -o viperlight.zip | |
| unzip -q viperlight.zip -d ../viperlight && rm viperlight.zip | |
| cp ../viperlight/scripts/*.sh . | |
| default_scan="./codescan-postbuild-default.sh" | |
| if [ -f "$default_scan" ]; then | |
| chmod +x $default_scan | |
| else | |
| default_scn="viperlight scan" | |
| fi | |
| custom_scancmd="./codescan-postbuild-custom.sh" | |
| if [ -f "$custom_scancmd" ]; then | |
| chmod +x $custom_scancmd | |
| $custom_scancmd | |
| else | |
| $default_scan | |
| fi | |
| # force check notice-js | |
| ../viperlight/bin/viperlight pubcheck -m notice-js | |
| sonarqube: | |
| name: sonarqube scan | |
| if: ${{ github.event.pull_request.number }} | |
| runs-on: ${{ vars.LARGE_RUNNER_S || 'ubuntu-latest' }} | |
| permissions: | |
| id-token: write | |
| pull-requests: write | |
| contents: read | |
| services: | |
| sonarqube: | |
| image: public.ecr.aws/docker/library/sonarqube:9.9.0-community | |
| ports: | |
| - 9000:9000 | |
| env: | |
| iam_role_to_assume: ${{ secrets.ROLE_ARN }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: "8.15.3" | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Configure AWS Credentials | |
| if: ${{ env.iam_role_to_assume != '' }} | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: ${{ env.iam_role_to_assume }} | |
| aws-region: us-east-1 | |
| - name: Login to Amazon ECR Public | |
| if: ${{ env.iam_role_to_assume != '' }} | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| with: | |
| registry-type: public | |
| - name: Install dependencies | |
| run: pnpm i --no-frozen-lockfile | |
| - name: Run build and unit tests | |
| env: | |
| JEST_MAX_WORKERS: ${{ vars.JEST_MAX_WORKERS || 'auto' }} | |
| LOCAL_TESTING: true # speed up the tests for infra | |
| run: | | |
| bash deployment/run-unit-tests.sh | |
| - name: Configure sonarqube | |
| env: | |
| SONARQUBE_URL: http://localhost:9000 | |
| SONARQUBE_ADMIN_PASSWORD: ${{ secrets.SONARQUBE_ADMIN_PASSWORD }} | |
| run: | | |
| bash .github/workflows/sonarqube/sonar-configure.sh | |
| - name: SonarQube Scan | |
| uses: SonarSource/sonarqube-scan-action@v2.1.0 | |
| env: | |
| SONAR_HOST_URL: http://sonarqube:9000 | |
| SONAR_TOKEN: ${{ env.SONARQUBE_TOKEN }} | |
| with: | |
| args: > | |
| -Dsonar.projectKey=pr-${{ github.event.pull_request.number }} | |
| # Check the Quality Gate status. | |
| - name: SonarQube Quality Gate check | |
| id: sonarqube-quality-gate-check | |
| uses: SonarSource/sonarqube-quality-gate-action@v1.1.0 | |
| # Force to fail step after specific time. | |
| timeout-minutes: 5 | |
| env: | |
| SONAR_TOKEN: ${{ env.SONARQUBE_TOKEN }} | |
| SONAR_HOST_URL: http://localhost:9000 | |
| - uses: phwt/sonarqube-quality-gate-action@v1 | |
| id: quality-gate-check | |
| if: always() | |
| with: | |
| sonar-project-key: pr-${{ github.event.pull_request.number }} | |
| sonar-host-url: http://sonarqube:9000 | |
| sonar-token: ${{ env.SONARQUBE_TOKEN }} | |
| github-token: ${{ secrets.PROJEN_GITHUB_TOKEN }} | |
| - name: Post Sonar Quality results to PR | |
| uses: zxkane/sonar-quality-gate@master | |
| if: always() | |
| env: | |
| DEBUG: true | |
| GITHUB_TOKEN: ${{ secrets.PROJEN_GITHUB_TOKEN }} | |
| GIT_URL: "https://api.github.com" | |
| GIT_TOKEN: ${{ secrets.PROJEN_GITHUB_TOKEN }} | |
| SONAR_URL: http://sonarqube:9000 | |
| SONAR_TOKEN: ${{ env.SONARQUBE_TOKEN }} | |
| SONAR_PROJECT_KEY: pr-${{ github.event.pull_request.number }} | |
| with: | |
| login: ${{ env.SONARQUBE_TOKEN }} | |
| skipScanner: true |