docker-unified.yml

name: Unified Docker Workflow
on:
  push:
    branches:
      - master
    paths-ignore:
      - 'docs/**'
      - '**.md'
  pull_request:
    branches:
      - master
    paths-ignore:
      - 'docs/**'
      - '**.md'
  release:
    types: [published, edited]

concurrency:
  # Using `github.run_id` (unique val) instead of `github.ref` here
  # because we don't want to cancel this workflow on master only for PRs
  #   as that makes reproducing issues easier
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

env:
  DATAHUB_GMS_IMAGE: 'linkedin/datahub-gms'
  DATAHUB_FRONTEND_IMAGE: 'linkedin/datahub-frontend-react'
  DATAHUB_MAE_CONSUMER_IMAGE: 'linkedin/datahub-mae-consumer'
  DATAHUB_MCE_CONSUMER_IMAGE: 'linkedin/datahub-mce-consumer'
  DATAHUB_KAFKA_SETUP_IMAGE: 'linkedin/datahub-kafka-setup'
  DATAHUB_ELASTIC_SETUP_IMAGE: 'linkedin/datahub-elasticsearch-setup'
  DATAHUB_MYSQL_SETUP_IMAGE: 'acryldata/datahub-mysql-setup'
  DATAHUB_UPGRADE_IMAGE: 'acryldata/datahub-upgrade'
  

jobs:
  setup:
    runs-on: ubuntu-latest
    outputs:
      tag: ${{ steps.tag.outputs.tag }}
      unique_tag: ${{ steps.tag.outputs.unique_tag }}
      publish: ${{ steps.publish.outputs.publish }}
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Compute Tag
        id: tag
        run: |
          echo "GITHUB_REF: $GITHUB_REF"
          SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
          TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
          UNIQUE_TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
          echo "tag=$TAG"
          echo "unique_tag=$UNIQUE_TAG"
          echo "::set-output name=tag::$TAG"
          echo "::set-output name=unique_tag::$UNIQUE_TAG"
      - name: Check whether publishing enabled
        id: publish
        env:
          ENABLE_PUBLISH: ${{ secrets.DOCKER_PASSWORD }}
        run: |
          echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}"
          echo "::set-output name=publish::${{ env.ENABLE_PUBLISH != '' }}"

  gms_build:
    name: Build and Push DataHub GMS Docker Image
    runs-on: ubuntu-latest
    needs: setup
    outputs:
      image_tag: ${{ steps.docker_meta.outputs.tags }}
      image_name: ${{ env.DATAHUB_GMS_IMAGE }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_GMS_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-gms/Dockerfile
          platforms: linux/amd64,linux/arm64
  gms_scan:
    name: "[Monitoring] Scan GMS images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup, gms_build]
    steps:
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
          format: 'table'
          ignore-unfixed: true
          vuln-type: 'os,library'
          severity: 'CRITICAL,HIGH'

  mae_consumer_build:
    name: Build and Push DataHub MAE Consumer Docker Image
    runs-on: ubuntu-latest
    needs: setup
    outputs:
      image_tag: ${{ steps.docker_meta.outputs.tags }}
      image_name: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-mae-consumer/Dockerfile
          platforms: linux/amd64
  mae_consumer_scan:
    name: "[Monitoring] Scan MAE consumer images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup,mae_consumer_build]
    steps:
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
          format: 'table'
          ignore-unfixed: true
          vuln-type: 'os,library'
          severity: 'CRITICAL,HIGH'

  mce_consumer_build:
    name: Build and Push DataHub MCE Consumer Docker Image
    runs-on: ubuntu-latest
    needs: setup
    outputs:
      image_tag: ${{ steps.docker_meta.outputs.tags }}
      image_name: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-mce-consumer/Dockerfile
          platforms: linux/amd64
  mce_consumer_scan:
    name: "[Monitoring] Scan MCE consumer images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup,mce_consumer_build]
    steps:
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
          format: 'table'
          ignore-unfixed: true
          vuln-type: 'os,library'
          severity: 'CRITICAL,HIGH'

  datahub_upgrade_build:
    name: Build and Push DataHub Upgrade Docker Image
    runs-on: ubuntu-latest
    needs: setup
    outputs:
      image_tag: ${{ steps.docker_meta.outputs.tags }}
      image_name: ${{ env.DATAHUB_UPGRADE_IMAGE }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_UPGRADE_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-upgrade/Dockerfile
          platforms: linux/amd64,linux/arm64
  datahub_upgrade_scan:
    name: "[Monitoring] Scan DataHub Upgrade images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup, datahub_upgrade_build]
    steps:
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
          format: 'table'
          ignore-unfixed: true
          vuln-type: 'os,library'
          severity: 'CRITICAL,HIGH'

  frontend_build:
    name: Build and Push DataHub Frontend Docker Image
    runs-on: ubuntu-latest
    needs: setup
    outputs:
      image_tag: ${{ steps.docker_meta.outputs.tags }}
      image_name: ${{ env.DATAHUB_FRONTEND_IMAGE }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_FRONTEND_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/datahub-frontend/Dockerfile
          platforms: linux/amd64,linux/arm64
  frontend_scan:
    name: "[Monitoring] Scan Frontend images for vulnerabilities"
    runs-on: ubuntu-latest
    needs: [setup, frontend_build]
    steps:
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
          format: 'table'
          ignore-unfixed: true
          vuln-type: 'os,library'
          severity: 'CRITICAL,HIGH'

  kafka_setup_build:
    name: Build and Push DataHub Kafka Setup Docker Image
    runs-on: ubuntu-latest
    needs: setup
    outputs:
      image_tag: ${{ steps.docker_meta.outputs.tags }}
      image_name: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: ./docker/kafka-setup
          file: ./docker/kafka-setup/Dockerfile
          platforms: linux/amd64,linux/arm64

  mysql_setup_build:
    name: Build and Push DataHub MySQL Setup Docker Image
    runs-on: ubuntu-latest
    needs: setup
    outputs:
      image_tag: ${{ steps.docker_meta.outputs.tags }}
      image_name: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/mysql-setup/Dockerfile
          platforms: linux/amd64,linux/arm64

  elasticsearch_setup_build:
    name: Build and Push DataHub Elasticsearch Setup Docker Image
    runs-on: ubuntu-latest
    needs: setup
    outputs:
      image_tag: ${{ steps.docker_meta.outputs.tags }}
      image_name: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish }}
          context: .
          file: ./docker/elasticsearch-setup/Dockerfile
          platforms: linux/amd64,linux/arm64

  smoke_test:
    name: Run Smoke Tests
    runs-on: ubuntu-latest
    needs: [setup,gms_build,frontend_build,kafka_setup_build,mysql_setup_build,elasticsearch_setup_build]
    steps:
      - name: Check out the repo
        uses: actions/checkout@v2
      - name: Set up JDK 1.8
        uses: actions/setup-java@v1
        with:
          java-version: 1.8
      - uses: actions/setup-python@v2
        with:
          python-version: "3.7"
      - name: Install dependencies
        run: ./metadata-ingestion/scripts/install_deps.sh
      - name: Build datahub cli
        run: |
          ./gradlew :metadata-ingestion:install
      - name: Download GMS image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download Frontend image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image:  ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download Kafka Setup image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download Mysql Setup image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Download Elastic Setup image
        uses: ishworkh/docker-image-artifact-download@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
      - name: Smoke test
        env:
          DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
        run: |
          echo "$DATAHUB_VERSION"
          ./smoke-test/smoke.sh
      - name: store logs
        if: failure()
        run: |
          docker ps -a
          docker logs datahub-gms >& gms.log
      - name: Upload logs
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: docker logs
          path: "*.log"
      - name: Upload screenshots
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: cypress-snapshots
          path: smoke-test/tests/cypress/cypress/screenshots/
      - uses: actions/upload-artifact@v2
        if: always()
        with:
          name: Test Results (smoke tests)
          path: |
            **/build/reports/tests/test/**
            **/build/test-results/test/**
            **/junit.*.xml
      - name: Slack failure notification
        if: failure() && github.event_name == 'push'
        uses: kpritam/slack-job-status-action@v1
        with:
          job-status: ${{ job.status }}
          slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
          channel: github-activities