build-and-release-docker.yml

name: Build and Release Docker

on:
  workflow_call:
    inputs:
      enabled:
        required: true
        type: boolean
      release_tag:
        required: true
        type: string
      elyra_branch:
        required: false
        default: "master"
        type: string
      unskript_branch:
        required: false
        default: "master"
        type: string
      celltoolbar_branch:
        required: false
        default: "master"
        type: string
      snippet_branch:
        required: false
        default: "master"
        type: string
      awesome_branch:
        required: false
        default: "master"
        type: string
      devops_branch:
        required: false
        default: "master"
        type: string
      build_number:
        required: true
        type: string
      latest:
        required: false
        default: true
        type: boolean

  workflow_dispatch:
    inputs:
      enabled:
        description: 'Workflow Enable Flag'
        required: true
        default: false
        type: boolean
      elyra_branch:
        description: 'Elyra Branch name'
        required: true
        default: master
        type: string
      unskript_branch:
        description: 'unSkript Branch name'
        required: true
        default: master
        type: string
      celltoolbar_branch:
        description: 'Celltoolbar Branch name'
        required: true
        default: master
        type: string
      snippet_branch:
        description: 'Code Snippets Branch name'
        required: true
        default: master
        type: string
      awesome_branch:
        description: 'unSkript submodule awesome Branch name'
        required: true
        default: master
        type: string
      devops_branch:
        description: 'Devops Branch name'
        required: false
        default: master
        type: string
      build_number:
        description: 'Docker build number'
        required: true
        type: string
      latest:
        description: 'Docker Latest tag Branch name'
        default: false
        type: boolean

concurrency:
  group: ${{ github.ref }}-full-build
  cancel-in-progress: true

env:
  DOCKER_REGISTRY: docker.io
  DOCKER_IMAGE: unskript/awesome-runbooks
  DOCKER_USERNAME: ${{ secrets.DOCKER_USER }}
  DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
  USERNAME: ${{ secrets.BUILD_USER }}
  DOCKER_TARGET: linux/amd64, linux/arm64

permissions:
  contents: read

jobs:
  build-elyra:
    runs-on: ubuntu-latest
    if: ${{ inputs.enabled }}
    strategy:
      fail-fast: false

    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
        with:
          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs

      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.0
      - name: Get current date
        id: date
        run: echo "::set-output name=date::$(date +'%Y-%m-%d-%s')"
      - name: Install system dependencies
        run: |
          pip install shyaml

      - name: Update Spy2 to workaround for repo timeout issue
        run: |
           sudo gem install apt-spy2
           sudo apt-spy2 fix --commit --launchpad --country=US
           sudo apt-get update

      - name: Set up Python 3.x
        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # v2.3.3
        with:
          python-version: '3.7'

      - name: Checkout & Build Code
        run: |
          cd $HOME
          git clone https://${{ env.USERNAME }}:${{ secrets.BUILDER_PAT }}@github.com/unskript/elyra.git elyra
          cd elyra
          git checkout ${{ inputs.elyra_branch }}
          python3 ./setup.py sdist
          mv dist/elyra*.tar.gz /tmp

      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
        with:
          name: elyra-${{ github.run_id }}
          path: /tmp/elyra-3.0.0.dev0.tar.gz

  build-unskript:
    runs-on: ubuntu-latest
    if: ${{ inputs.enabled }}
    needs: [build-elyra]
    strategy:
      fail-fast: false

    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
        with:
          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs

      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.0
      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: elyra-${{ github.run_id }}
          path: /tmp/elyra
      - name: Get current date
        id: date
        run: echo "::set-output name=date::$(date +'%Y-%m-%d-%s')"

      - name: Update Spy2 to workaround for repo timeout issue
        run: |
           sudo gem install apt-spy2
           sudo apt-spy2 fix --commit --launchpad --country=US
           sudo apt-get update


      - name: Set up Python 3.x
        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # v2.3.3
        with:
          # Restricting python version to 3.9
          python-version: '3.9'

      - name: Install system dependencies
        run: |
          pip install shyaml
          sudo apt-get install -y --allow-downgrades unixodbc-dev=2.3.7 unixodbc=2.3.7 odbcinst1debian2=2.3.7 odbcinst=2.3.7


      - name: Checkout Code
        run: |
          wget -O /tmp/pandas-2.0.1.tar.gz https://files.pythonhosted.org/packages/6c/e0/73987b6ecc7246e02ab557240843f93fd5adf45d1355abb458aa1f2a0932/pandas-2.0.1.tar.gz
          sudo pip install /tmp/pandas-2.0.1.tar.gz
          cd $HOME
          git clone https://${{ env.USERNAME }}:${{ secrets.BUILDER_PAT }}@github.com/unskript/unskript.git unskript
          cd unskript
          git checkout ${{ inputs.unskript_branch }}
          /usr/bin/env python -m pip install /tmp/elyra/*.tar.gz
          # We use the --upgrade-strage only-if-needed and --use-deprecated=legacy-resolver to avoid
          # PIP dependency loop. Since we are asking the git runner to be of ubuntu-latest, it becomes
          # a moving target for us hence we want to restrict the packages that are needed for unskript
          # to compile to be fixed.
          /usr/bin/env python -m pip install -r ./requirements.txt --upgrade --upgrade-strategy only-if-needed --use-deprecated=legacy-resolver
          /usr/bin/env python -m pip install --upgrade protobuf
          # We need to restrict URLLIB3 to 1.26.6 because Version 2.x.y onwards the DEFAULT_CIPHERS
          # variable is deprecrated. Unfortunately our boto3 and botocore that is needed for our
          # unskript package does not work with the latest version of URLLIB3. Hence we need to
          # restrict it to a fixed version of 1.26.6
          /usr/bin/env python -m pip install --upgrade urllib3==1.26.6
          /usr/bin/env python -m pip install --upgrade types-urllib3==1.26.13
          /usr/bin/env python -m pip install google-api-python-client==2.77.0
          make awesome-submodule
          cd awesome
          git checkout ${{ inputs.awesome_branch }}
          cd ..
          make legoschema
          [ -f "setup-full.py" ] &&  cp "setup-full.py" ./setup.py
          /usr/bin/env python ./setup.py bdist_wheel
          mv dist/code*tar /tmp
          mv dist/unskript-0.1.0-py2.py3-none-any.whl /tmp

      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
        with:
          name: schema-${{ github.run_id }}
          path: /tmp/code_snippet_schemas.tar

      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
        with:
          name: unskript-${{ github.run_id }}
          path: /tmp/unskript-0.1.0-py2.py3-none-any.whl


  build-jlab-celltoolbar:
    runs-on: ubuntu-20.04
    if: ${{ inputs.enabled }}
    strategy:
      fail-fast: false

    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
        with:
          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs

      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.0
      - name: Get current date
        id: date
        run: echo "::set-output name=date::$(date +'%Y-%m-%d-%s')"

      - name: Update Spy2 to workaround for repo timeout issue
        run: |
           sudo gem install apt-spy2
           sudo apt-spy2 fix --commit --launchpad --country=US
           sudo apt-get update

      - name: Set up Python 3.x
        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # v2.3.3
        with:
          python-version: '3.7'

      - name: Install system dependencies
        run: |
          pip install shyaml
          sudo apt install nodejs npm
          pip install jupyterlab

      - name: Checkout Code
        run: |
          cd $HOME
          export NODE_OPTIONS=--openssl-legacy-provider
          git clone https://${{ env.USERNAME }}:${{ secrets.BUILDER_PAT }}@github.com/unskript/jlab-enhanced-cell-toolbar.git jlab-enhanced-cell-toolbar
          cd jlab-enhanced-cell-toolbar
          git checkout ${{ inputs.celltoolbar_branch }}
          /usr/bin/env python -m pip install jupyter-packaging
          /usr/bin/env python -m pip install --upgrade markupsafe
          /usr/bin/env python -m pip install --upgrade jinja2
          jlpm install
          /usr/bin/env python ./setup.py bdist_wheel
          mv dist/*.whl /tmp


      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
        with:
          name: toolbar-${{ github.run_id }}
          path: /tmp/jlab_enhanced_cell_toolbar-3.4.0-py3-none-any.whl

  build-code-snippets:
    runs-on: ubuntu-20.04
    if: ${{ inputs.enabled }}
    strategy:
      fail-fast: false

    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
        with:
          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs

      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.0
      - name: Get current date
        id: date
        run: echo "::set-output name=date::$(date +'%Y-%m-%d-%s')"

      - name: Update Spy2 to workaround for repo timeout issue
        run: |
           sudo gem install apt-spy2
           sudo apt-spy2 fix --commit --launchpad --country=US
           sudo apt-get update

      - name: Set up Python 3.x
        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # v2.3.3
        with:
          python-version: '3.7'

      - name: Install system dependencies
        run: |
          pip install shyaml
          sudo apt install nodejs npm
          pip install jupyterlab

      - name: Checkout Code
        run: |
          cd $HOME
          git clone https://${{ env.USERNAME }}:${{ secrets.BUILDER_PAT }}@github.com/unskript/jupyterlab-code-snippets.git jupyterlab-code-snippets
          cd jupyterlab-code-snippets
          git checkout ${{ inputs.snippet_branch }}
          /usr/bin/env python -m pip install jupyter-packaging
          /usr/bin/env python -m pip install --upgrade markupsafe
          /usr/bin/env python -m pip install --upgrade jinja2
          jlpm install
          /usr/bin/env python ./setup.py bdist_wheel
          mv dist/*.whl /tmp


      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
        with:
          name: code-snippets-${{ github.run_id }}
          path: /tmp/jupyterlab_code_snippets-2.1.1-py3-none-any.whl

  build-docker:
    runs-on: ubuntu-latest
    if: ${{ inputs.enabled }}
    needs: [build-unskript, build-elyra, build-jlab-celltoolbar, build-code-snippets]
    strategy:
      fail-fast: false

    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
        with:
          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs

      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.0

      - name: Update Spy2 to workaround for repo timeout issue
        run: |
           sudo gem install apt-spy2
           sudo apt-spy2 fix --commit --launchpad --country=US
           sudo apt-get update


      - name: Set up Python 3.x
        uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # v2.3.3
        with:
          python-version: '3.7'

      - name: Install system dependencies
        run: |
          sudo apt update --fix-missing
          pip install shyaml

      - name: Setup Docker Buildx
        uses: crazy-max/ghaction-docker-buildx@126d331dc69f4a1aa02452e374835e6a5d565613 # v3.3.1
        with:
          version: latest
          config: .github/buildkitd.toml

      - name: Prepare Docker Buildx
        id: prepare
        run: |
          echo ::set-output name=docker_platform::${DOCKER_TARGET}
          echo ::set-output name=docker_image::${DOCKER_REGISTRY}/${DOCKER_IMAGE}
          echo ::set-output name=version::${GITHUB_RUN_NUMBER}

      - name: Docker Login
        run: |
          echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin


      - name: Checkout Code
        run: |
          cd $HOME
          pwd
          git clone https://${{ env.USERNAME }}:${{ secrets.BUILDER_PAT }}@github.com/unskript/devops.git devops

      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: code-snippets-${{ github.run_id }}
          path: /tmp/code_snippet

      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: toolbar-${{ github.run_id }}
          path: /tmp/jlab_enhanced_cell_toolbar

      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: schema-${{ github.run_id }}
          path: /tmp/unskript

      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: unskript-${{ github.run_id }}
          path: /tmp/unskript

      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: elyra-${{ github.run_id }}
          path: /tmp/elyra

      - name: Prepare to Build
        run: |
          cd $HOME/devops/dockers/jupyterlab/oss_docker

          ls -l /tmp/unskript/
          ls -l /tmp/elyra/
          ls -l /tmp/jlab_enhanced_cell_toolbar/
          ls -l /tmp/code_snippet/
          tar xf /tmp/unskript/code_snippet_schemas.tar
          cd downloads
          mv /tmp/unskript/*.whl .
          mv /tmp/code_snippet/*.whl .
          mv /tmp/jlab_enhanced_cell_toolbar/*.whl .
          mv /tmp/elyra/elyra*tar.gz .
          if [[ ${{ inputs.latest }} == "true" ]]; then
              bt="${{ env.DOCKER_IMAGE }}:${{ inputs.build_number }} -t ${{ env.DOCKER_IMAGE }}:latest"
          else
              bt="${{ env.DOCKER_IMAGE }}:${{ inputs.build_number }}"
          fi
          echo "BUILD_TAGS=$bt" >> $GITHUB_ENV

      - uses: geekyeggo/delete-artifact@54ab544f12cdb7b71613a16a2b5a37a9ade990af # v2.0.0
        with:
          name: |
            elyra-${{ github.run_id }}
            toolbar-${{ github.run_id }}
            code-snippets-${{ github.run_id }}
            unskript-${{ github.run_id }}
            schema-${{ github.run_id }}

      - name: Build & Push
        run: |
          docker buildx create --name mybuilder
          docker buildx use mybuilder
          docker buildx inspect --bootstrap
          docker buildx ls
          cd $HOME/devops/dockers/jupyterlab/oss_docker
          git checkout ${{ inputs.devops_branch }}
          export BUILD_NUMBER=${{ inputs.build_number }}
          make copy
          cd $HOME/devops/dockers/jupyterlab/oss_docker/
          git clone https://${{ env.USERNAME }}:${{ secrets.BUILDER_PAT }}@github.com/unskript/unskript.git unskript
          cd unskript
          git checkout ${{ inputs.unskript_branch }}
          make awesome-submodule
          cd awesome
          git checkout ${{ inputs.awesome_branch }}
          cd ..
          make syncrunbooks
          cp -Rf awesome/unskript-ctl/* $HOME/devops/dockers/jupyterlab/oss_docker/
          cp -Rf awesome/bin/* $HOME/devops/dockers/jupyterlab/oss_docker/
          cp awesome/build/templates/Welcome_template.ipynb $HOME/devops/dockers/jupyterlab/oss_docker/runbooks/template.ipynb
          cp awesome/build/templates/Welcome.ipynb $HOME/devops/dockers/jupyterlab/oss_docker/runbooks/Welcome.ipynb
          cd $HOME/devops/dockers/jupyterlab/oss_docker/
          cp $HOME/devops/dockers/jupyterlab/common/install_utils.sh .
          docker buildx build --platform linux/amd64,linux/arm64 --push -t ${{ env.BUILD_TAGS }} .

      - name: Docker Scout
        id: docker-scout
        uses: docker/scout-action@v1
        with:
          command: cves
          image: ${{ env.BUILD_TAGS }}
          only-severities: critical,high
          exit-code: true
      
      - name: Validate result of Docker Scout
        if: failure()
        run: |
            sudo apt install -y jq curl
            TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username": "${{ env.DOCKER_USERNAME }}", "password": "${{ env.DOCKER_PASSWORD }}"}' https://hub.docker.com/v2/users/login/ | jq -r .token)
            REPO_LIST=$(curl -s -H "Authorization: JWT ${TOKEN}" https://hub.docker.com/v2/repositories/unskript/?page_size=200 | jq -r '.results | if . then .[] | .name else empty end')
            for i in ${REPO_LIST}
            do
                curl -X DELETE -s -H "Authorization: JWT ${TOKEN}" https://hub.docker.com/v2/repositories/unskript/${i}/tags/${{ inputs.build_number }}/
                echo "DELETED IMAGE THAT FAILED DOCKER SCOUT TEST"
            done
            
  cleanup:
    runs-on: ubuntu-latest
    if: ${{ inputs.enabled }}
    needs: [build-docker]
    strategy:
      fail-fast: false

    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
        with:
          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs

      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.0
      - name: Install system dependencies
        run: |
          pip install shyaml


      - uses: geekyeggo/delete-artifact@54ab544f12cdb7b71613a16a2b5a37a9ade990af # v2.0.0
        with:
          name: |
            elyra-${{ github.run_id }}
            toolbar-${{ github.run_id }}
            code-snippets-${{ github.run_id }}
            unskript-${{ github.run_id }}
            schema-${{ github.run_id }}