ci_module.yml

name: Automated CI testing
# This workflow run automatically for every commit on github it checks the syntax and launch the tests.
# | grep . | uniq -c filters out empty lines and then groups consecutive lines together with the number of occurrences
on:
  workflow_call:
    secrets:
      SONAR_TOKEN:
        description: 'A sonar access token passed from the caller workflow'
        required: false
    inputs:
      SONAR_PROJECT_KEY:
        description: 'SonarQube Project Key'
        required: false
        type: string
      SONAR_ORGANIZATION:
        description: 'SonarQube Organization'
        required: false
        type: string
      SONAR_PROJECT_NAME:
        description: 'SonarQube Project Name'
        required: false
        type: string
      SONAR_PROJECT_VERSION:
        description: 'SonarQube Project Version'
        required: false
        type: string
      SONAR_EXCLUSIONS:
        description: 'SonarQube Project Exclusions'
        required: false
        type: string
      SONAR_SOURCES:
        description: 'SonarQube Project Sources'
        required: false
        type: string

jobs:
  build_backend:
    name: Build Backend
    runs-on: ubuntu-20.04
    steps:
      - name: Set up Python 3.8
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Pull openIMIS Backend
        run: |
          rm ./openimis -rf
          git clone --depth=1 --branch=develop https://github.com/openimis/openimis-be_py.git ./openimis

      - name: Copy Current branch
        uses: actions/checkout@v2
        with:
          # Disabling shallow clone is recommended for improving relevancy of reporting
          fetch-depth: 0
          path: './current-module'

      - name: Cache dependencies
        uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
            ${{ runner.os }}-pip-
            ${{ runner.os }}-

      - name: Update the configuration
        working-directory: ./openimis
        run: |
          sudo apt-get update
          sudo apt-get install jq

          export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')"
          
          echo "MODULE_NAME=$MODULE_NAME" >> $GITHUB_ENV
          
          # Add or replace MODULE_NAME module in openimis.json (local version)
          echo $(jq --arg name "$MODULE_NAME" 'if [.modules[].name  ==  ($name)]| max then (.modules[]  | select(.name == ($name)) | .pip)|="-e ../current-module" else .modules |= .+ [{name:($name), pip:"../current-module"}] end' openimis.json)  > openimis.json
          cat openimis.json

      - name: Install Python dependencies
        working-directory: ./openimis
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt
          python modules-requirements.py openimis.json > modules-requirements.txt
          cat modules-requirements.txt
          pip install --no-cache-dir -r modules-requirements.txt

      - name: Environment info
        run: |
          pip list

      - name: Locate site-packages and upload
        run: |
          SITE_PACKAGES=$(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
          echo "SITE_PACKAGES=$SITE_PACKAGES" >> $GITHUB_ENV
        id: locate

      - name: Compress IMIS artifacts
        run: |
          tar -czf site-packages.tar.gz -C ${{ env.SITE_PACKAGES }} .
          tar -czf openimis.tar.gz ./openimis ./current-module

      - name: Upload compressed site-packages as artifact
        uses: actions/upload-artifact@v2
        with:
          name: site-packages
          path: site-packages.tar.gz

      - name: Upload build as artifact
        uses: actions/upload-artifact@v2
        with:
          name: code-artifacts
          path: openimis.tar.gz
  ci_module_mssql_tests:
    name: Run All Tests (MSSQL)
    runs-on: ubuntu-20.04
    needs: build_backend
    services:
      mssql:
        image: mcr.microsoft.com/mssql/server:2017-latest
        env:
          ACCEPT_EULA: Y
          SA_PASSWORD: GitHub999
        ports:
          - 1433:1433
        # this health-cmd needed because the mssql container does not provide a health check
        options: >-
          --health-interval=10s
          --health-timeout=5s
          --health-start-period=10s
          --health-retries=10
          --health-cmd="/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P ${SA_PASSWORD} -Q 'SELECT 1' || exit 1"
    steps:
      - name: Set up Python 3.8
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Download site-packages artifact
        uses: actions/download-artifact@v2
        with:
          name: site-packages
          path: /tmp/

      - name: Locate site-packages
        run: |
          SITE_PACKAGES=$(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
          echo "SITE_PACKAGES=$SITE_PACKAGES" >> $GITHUB_ENV

      - name: Download site-packages artifact
        uses: actions/download-artifact@v2
        with:
          name: code-artifacts
          path: /tmp/

      - name: Decompress artifacts
        run: |
          sudo tar -xzf /tmp/site-packages.tar.gz -C ${{ env.SITE_PACKAGES }}
          sudo tar -xzf /tmp/openimis.tar.gz -C .
      - name: Initialize MSSQL
        run: |
          export DB_NAME_TEST="test_$DB_NAME"
          
          if [ ${GITHUB_REF##*/} = "develop" ]; then export DBBRANCH="main"; else export DBBRANCH="develop"; fi
          echo "Branch ${GITHUB_REF##*/}, usign ${DBBRANCH} branch for database"
          git clone --depth=1 --branch=$DBBRANCH https://github.com/openimis/database_ms_sqlserver.git ./sql_mssql
          cd sql_mssql/ && bash concatenate_files.sh && cd ..

          curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
          curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
          sudo apt-get update
          sudo apt-get install mssql-tools unixodbc-dev
          
          #Main db has to exists for tests to run correctly, psql container creates empty main db by default
          /opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "DROP DATABASE IF EXISTS $DB_NAME"
          /opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "CREATE DATABASE $DB_NAME"

          /opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "DROP DATABASE IF EXISTS $DB_NAME_TEST"
          /opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "CREATE DATABASE $DB_NAME_TEST"
          /opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -d $DB_NAME_TEST -i ./sql_mssql/output/fullDemoDatabase.sql | grep . | uniq -c
        env:
          DB_HOST: localhost
          DB_PORT: 1433
          DB_NAME: imis
          DB_USER: sa
          DB_PASSWORD: GitHub999
      - name: Cache dependencies
        uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
            ${{ runner.os }}-pip-
            ${{ runner.os }}-
      - name: Django tests MSSQL
        working-directory: ./openimis/openIMIS
        # Run the tests regardless if previous steps failed (if setup fails the tests should crash instantly)
        if: success() || failure()
        run: |
          python -V
          export MODULES=$(jq -r '(.modules[].name)' ../openimis.json | xargs)
          echo $MODULES
          python manage.py test --debug-mode --timing --keepdb $MODULES
        env:
          SECRET_KEY: secret
          DEBUG: true
          DB_ENGINE: mssql
          #DJANGO_SETTINGS_MODULE: hat.settings
          DB_HOST: localhost
          DB_PORT: 1433
          DB_NAME: imis
          DB_USER: sa
          DB_PASSWORD: GitHub999
          #DEV_SERVER: true
          SITE_ROOT: api
          #REMOTE_USER_AUTHENTICATION: False
  ci_module_psql_test:
    name: Run All Tests (PSQL)
    runs-on: ubuntu-20.04
    needs: build_backend
    services:
      pgsql:
        image: postgres
        env:
          DB_HOST: localhost
          DB_PORT: 5432
          POSTGRES_DB: imis
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: GitHub999
        ports:
          - 5432:5432
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 10
    steps:
      - name: Set up Python 3.8
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Download site-packages artifact
        uses: actions/download-artifact@v2
        with:
          name: site-packages
          path: /tmp/

      - name: Locate site-packages
        run: |
          SITE_PACKAGES=$(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
          echo "SITE_PACKAGES=$SITE_PACKAGES" >> $GITHUB_ENV

      - name: Download site-packages artifact
        uses: actions/download-artifact@v2
        with:
          name: code-artifacts
          path: /tmp/

      - name: Decompress artifacts
        run: |
          sudo tar -xzf /tmp/site-packages.tar.gz -C ${{ env.SITE_PACKAGES }}
          sudo tar -xzf /tmp/openimis.tar.gz -C .
      - name: Initialize PSQL
        run: |
          export DB_NAME_TEST="test_$DB_NAME"
          
          if [ ${GITHUB_REF##*/} = "develop" ]; then export DBBRANCH="main"; else export DBBRANCH="develop"; fi
          echo "Branch ${GITHUB_REF##*/}, usign ${DBBRANCH} branch for database"
          git clone  --depth=1 --branch=$DBBRANCH https://github.com/openimis/database_postgresql.git ./sql_psql
          
          echo 'set search_path to public' >> ~/.psqlrc
          PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -c "DROP DATABASE IF EXISTS \"$DB_NAME_TEST\";"
          PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -c "CREATE DATABASE \"$DB_NAME_TEST\";"
          
          PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f  ./sql_psql/database\ scripts/00_dump.sql | grep . | uniq -c
          PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f  ./sql_psql/database\ scripts/02_aux_functions.sql | grep . | uniq -c
          PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f  ./sql_psql/database\ scripts/03_views.sql | grep . | uniq -c
          PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f  ./sql_psql/database\ scripts/04_functions.sql | grep . | uniq -c
          PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f  ./sql_psql/database\ scripts/05_stored_procs.sql | grep . | uniq -c
          PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f  ./sql_psql/database\ scripts/demo_db.sql | grep . | uniq -c

        env:
          DB_HOST: localhost
          DB_PORT: 5432
          DB_NAME: imis
          DB_USER: postgres
          DB_PASSWORD: GitHub999
      - name: Django tests PSQL
        working-directory: ./openimis/openIMIS
        # Run the tests regardless if previous steps failed (if setup fails the tests should crash instantly)
        if: success() || failure()
        run: |
          python -V
          export MODULES=$(jq -r '(.modules[].name)' ../openimis.json | xargs)
          echo $MODULES
          # Generate sonar report 
          export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')"
          
          # This has to exit 0, or report is not generated 
          python -m coverage run --source=$MODULE_NAME --omit=*/migrations/* ./manage.py test --keepdb $MODULES
        env:
          SECRET_KEY: secret
          DEBUG: true
          DB_ENGINE: django.db.backends.postgresql
          #DJANGO_SETTINGS_MODULE: hat.settings
          DB_HOST: localhost
          DB_PORT: 5432
          DB_NAME: imis
          DB_USER: postgres
          DB_PASSWORD: GitHub999
          #DEV_SERVER: true
          SITE_ROOT: api
      - name: Generate Coverage Report
        if: always()
        working-directory: ./openimis/openIMIS
        run: |
          python -m coverage xml -o coverage.xml
          # Only leave actual code part, skip absuolute directory like 
          # /home/runner/work/openimis-be-invoice_py/openimis-be-invoice_py/current-module/
          # It's required by circle-ci setup
          mkdir coverage
          sed -E 's|/home/runner/work/(.*?)/(.*?)/current-module/|./current-module/|' coverage.xml >> coverage/coverage1.xml
          sed -E 's|/home/runner/work/(.*?)/(.*?)/current-module/|./|' coverage.xml >> coverage/coverage2.xml
          
          cat coverage.xml
      - name: Coverage results
        if: success() || failure()
        uses: actions/upload-artifact@v3
        with:
          name: coverage.xml
          path: ./openimis/openIMIS/coverage
  flake-8-linter:
    name: Run flake8 check
    runs-on: ubuntu-20.04
    needs: build_backend
    steps:
      - name: Set up Python 3.8
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - name: Download site-packages artifact
        uses: actions/download-artifact@v2
        with:
          name: site-packages
          path: /tmp/

      - name: Locate site-packages
        run: |
          SITE_PACKAGES=$(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
          echo "SITE_PACKAGES=$SITE_PACKAGES" >> $GITHUB_ENV

      - name: Download code artifact
        uses: actions/download-artifact@v2
        with:
          name: code-artifacts
          path: /tmp/

      - name: Decompress artifacts
        run: |
          sudo tar -xzf /tmp/site-packages.tar.gz -C ${{ env.SITE_PACKAGES }}
          sudo tar -xzf /tmp/openimis.tar.gz -C .
      - name: Flake8 Style Check
        working-directory: ./openimis/
        # Run the tests regardless if previous steps failed (if setup fails the tests should crash instantly)
        if: success() || failure()
        run: |
          # Flake8 Report
          export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')"
          MOD_DIR="../current-module/$MODULE_NAME"
          python -m flake8 --output-file=flake8-report.txt $MOD_DIR --exit-zero
          sed -i 's|\.\./current-module/||g' flake8-report.txt
          
          python -m flake8 $MOD_DIR
      - name: Flake8 results upload
        if: success() || failure()
        uses: actions/upload-artifact@v3
        with:
          name: flake8-report.txt
          path: ./openimis/flake8-report.txt
  sonar_scan:
    name: Run Sonar Scan
    runs-on: ubuntu-20.04
    needs: [ci_module_psql_test, flake-8-linter]
    if: always()
    env:
      SONAR_PROJECT_KEY: ${{ inputs.SONAR_PROJECT_KEY }}
      SONAR_ORGANIZATION: ${{ inputs.SONAR_ORGANIZATION }}
      SONAR_PROJECT_NAME: ${{ inputs.SONAR_PROJECT_NAME }}
      SONAR_PROJECT_VERSION: ${{ inputs.SONAR_PROJECT_VERSION }}
      SONAR_PYTHON_VERSION: 3.8
      SONAR_FLAKE8_REPORT_PATHS: ../report/flake8-report.txt
      SONAR_SOURCES: ${{ inputs.SONAR_SOURCES }}
    steps:
      - name: Set environment variables with default values
        run: |
          if [ -z "${{ inputs.SONAR_PROJECT_VERSION }}" ]; then
            echo "SONAR_PROJECT_VERSION=1.0" >> $GITHUB_ENV
          else
            echo "SONAR_PROJECT_VERSION=${{ inputs.SONAR_PROJECT_VERSION }}" >> $GITHUB_ENV
          fi

          if [ -z "${{ inputs.SONAR_EXCLUSIONS }}" ]; then
            echo "SONAR_EXCLUSIONS='**/migrations/**,**/static/**,**/media/**,**/tests/**'" >> $GITHUB_ENV
          else
            echo "SONAR_EXCLUSIONS=${{ inputs.SONAR_EXCLUSIONS }}" >> $GITHUB_ENV
          fi
      - name: Download coverage artifact
        uses: actions/download-artifact@v2
        with:
          name: coverage.xml
          path: report
      - name: Download flake8 artifact
        uses: actions/download-artifact@v2
        with:
          name: flake8-report.txt
          path: report
      - name: Download code artifact
        uses: actions/download-artifact@v2
        with:
          name: code-artifacts
          path: /tmp/
      - name: Decompress artifacts
        run: |
          # This path is required by the structure of coverage.xml paths
          sudo tar -xzf /tmp/openimis.tar.gz -C .
      - name: Debug
        run: |
          ls -a
          echo "---"
          ls report
          echo "---"
          ls current-module -a

      - name: SonarCloud Scan
        uses: SonarSource/sonarcloud-github-action@master
        with:
          args: >
            -Dsonar.projectKey=${{ inputs.SONAR_PROJECT_KEY }}
            -Dsonar.organization=${{ inputs.SONAR_ORGANIZATION }}
            -Dsonar.projectName=${{ inputs.SONAR_PROJECT_NAME }}
            -Dsonar.projectVersion=${{ env.SONAR_PROJECT_VERSION }}
            -Dsonar.python.version=3.8
            -Dsonar.sources=current-module
            -Dsonar.python.coverage.reportPaths=report/coverage1.xml,report/coverage2.xml
            -Dsonar.python.flake8.reportPaths=report/flake8-report.txt
            -Dsonar.exclusions=${{ env.SONAR_EXCLUSIONS }}
            -X
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}