Skip to content

update ci module to split module and full test #8

update ci module to split module and full test

update ci module to split module and full test #8

Workflow file for this run

---
name: Automated CI testing
on:
workflow_call:
secrets:
SONAR_TOKEN:
description: A sonar access token passed from the caller workflow
required: false
inputs:
SONAR_PROJECT_KEY:
description: SonarQube Project Key
required: false
type: string
SONAR_ORGANIZATION:
description: SonarQube Organization
required: false
type: string
SONAR_PROJECT_NAME:
description: SonarQube Project Name
required: false
type: string
SONAR_PROJECT_VERSION:
description: SonarQube Project Version
required: false
type: string
SONAR_EXCLUSIONS:
description: SonarQube Project Exclusions
required: false
type: string
SONAR_SOURCES:
description: SonarQube Project Sources
required: false
type: string
jobs:
build_backend:
name: Build Backend
runs-on: ubuntu-20.04
steps:
- name: Set up Python 3.10
uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Pull openIMIS Backend
run: >
rm ./openimis -rf
git clone --depth=1 --branch=develop https://github.com/openimis/openimis-be_py.git ./openimis
- name: Copy Current branch
uses: actions/checkout@v2
with:
fetch-depth: 0
path: ./current-module
- name: Cache dependencies
uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
${{ runner.os }}-pip-
${{ runner.os }}-
- name: Update the configuration
working-directory: ./openimis
run: >
sudo apt-get update
sudo apt-get install jq
export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')"
echo "MODULE_NAME=$MODULE_NAME" >> $GITHUB_ENV
export MODULES=$(jq -r '(.modules[].name)' ./openimis.json | xargs)
# Remove some module from MODULES
MODULES=$(echo "$MODULES" | sed -E "s/\b$(echo "${CI_EXCLUDED_MODULE// /\\b|\\b}")\b/ /g" | xargs)
echo "MODULES=$MODULES" >> $GITHUB_ENV
# Add or replace MODULE_NAME module in openimis.json (local version)
echo $(jq --arg name "$MODULE_NAME" 'if [.modules[].name == ($name)]| max then (.modules[] | select(.name == ($name)) | .pip)|="-e ../current-module" else .modules |= .+ [{name:($name), pip:"../current-module"}] end' openimis.json) > openimis.json
cat openimis.json
- name: Install Python dependencies3.10
working-directory: ./openimis
run: >
python -m pip install --upgrade pip
pip install -r requirements.txt
python modules-requirements.py openimis.json > modules-requirements.txt
cat modules-requirements.txt
pip install --no-cache-dir -r modules-requirements.txt
- name: Environment info
run: |
pip list
- name: Locate site-packages and upload
run: >
SITE_PACKAGES=$(python -c "from distutils.sysconfig import get_python_lib;
print(get_python_lib())")
echo "SITE_PACKAGES=$SITE_PACKAGES" >> $GITHUB_ENV
id: locate
- name: Compress IMIS artifacts
run: |
tar -czf site-packages.tar.gz -C ${{ env.SITE_PACKAGES }} .
tar -czf openimis.tar.gz ./openimis ./current-module
- name: Upload compressed site-packages as artifact
uses: actions/upload-artifact@v2
with:
name: site-packages
path: site-packages.tar.gz
- name: Upload build as artifact
uses: actions/upload-artifact@v2
with:
name: code-artifacts
path: openimis.tar.gz
ci_module_mssql_tests:
name: Run module Tests (MSSQL)
runs-on: ubuntu-20.04
needs: build_backend
services:
mssql:
image: mcr.microsoft.com/mssql/server:2017-latest
env:
ACCEPT_EULA: Y
SA_PASSWORD: GitHub999
ports:
- 1433:1433
options: --health-interval=10s --health-timeout=5s --health-start-period=10s
--health-retries=10 --health-cmd="/opt/mssql-tools/bin/sqlcmd -S
localhost -U sa -P ${SA_PASSWORD} -Q 'SELECT 1' || exit 1"
steps:
- name: Set up Python 3.10
uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Download site-packages artifact
uses: actions/download-artifact@v2
with:
name: site-packages
path: /tmp/
- name: Locate site-packages
run: >
SITE_PACKAGES=$(python -c "from distutils.sysconfig import get_python_lib;
print(get_python_lib())")
echo "SITE_PACKAGES=$SITE_PACKAGES" >> $GITHUB_ENV
export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')"
echo "MODULE_NAME=$MODULE_NAME" >> $GITHUB_ENV
- name: Download site-packages artifact
uses: actions/download-artifact@v2
with:
name: code-artifacts
path: /tmp/
- name: Decompress artifacts
run: |
sudo tar -xzf /tmp/site-packages.tar.gz -C ${{ env.SITE_PACKAGES }}
sudo tar -xzf /tmp/openimis.tar.gz -C .
- name: Initialize MSSQL
run: >
export DB_NAME_TEST="test_$DB_NAME"
if [ ${GITHUB_REF##*/} = "main" ]; then export DBBRANCH="main"; else export DBBRANCH="develop"; fi
echo "Branch ${GITHUB_REF##*/}, usign ${DBBRANCH} branch for database"
git clone --depth=1 --branch=$DBBRANCH https://github.com/openimis/database_ms_sqlserver.git ./sql_mssql
cd sql_mssql/ && bash concatenate_files.sh && cd ..
curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
sudo apt-get update
sudo apt-get install mssql-tools unixodbc-dev
#Main db has to exists for tests to run correctly, psql container creates empty main db by default
/opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "DROP DATABASE IF EXISTS $DB_NAME"
/opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "CREATE DATABASE $DB_NAME"
/opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "DROP DATABASE IF EXISTS $DB_NAME_TEST"
/opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -Q "CREATE DATABASE $DB_NAME_TEST"
/opt/mssql-tools/bin/sqlcmd -S $DB_HOST -U $DB_USER -P $DB_PASSWORD -d $DB_NAME_TEST -i ./sql_mssql/output/fullDemoDatabase.sql | grep . | uniq -c
env:
DB_HOST: localhost
DB_PORT: 1433
DB_NAME: imis
DB_USER: sa
DB_PASSWORD: GitHub999
- name: Cache dependencies
uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
${{ runner.os }}-pip-
${{ runner.os }}-
- name: Django tests MSSQL
working-directory: ./openimis/openIMIS
if: success() || failure()
run: |
python -V
python manage.py test --debug-mode --timing --keepdb $MODULE_NAME
env:
SECRET_KEY: secret
MODE: DEV
DB_ENGINE: mssql
DB_HOST: localhost
DB_PORT: 1433
DB_NAME: imis
DB_USER: sa
DB_PASSWORD: GitHub999
SITE_ROOT: api
CI_EXCLUDED_MODULE: ${{ vars.CI_EXCLUDED_MODULE }}
ci_module_psql_test:
name: Run module Tests (PSQL)
runs-on: ubuntu-20.04
needs: build_backend
services:
pgsql:
image: postgres
env:
DB_HOST: localhost
DB_PORT: 5432
POSTGRES_DB: imis
POSTGRES_USER: postgres
POSTGRES_PASSWORD: GitHub999
ports:
- 5432:5432
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s
--health-retries 10
steps:
- name: Set up Python 3.10
uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Download site-packages artifact
uses: actions/download-artifact@v2
with:
name: site-packages
path: /tmp/
- name: Locate site-packages
run: >
SITE_PACKAGES=$(python -c "from distutils.sysconfig import get_python_lib;
print(get_python_lib())")
echo "SITE_PACKAGES=$SITE_PACKAGES" >> $GITHUB_ENV
export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')"
echo "MODULE_NAME=$MODULE_NAME" >> $GITHUB_ENV
- name: Download site-packages artifact
uses: actions/download-artifact@v2
with:
name: code-artifacts
path: /tmp/
- name: Decompress artifacts
run: |
sudo tar -xzf /tmp/site-packages.tar.gz -C ${{ env.SITE_PACKAGES }}
sudo tar -xzf /tmp/openimis.tar.gz -C .
- name: Initialize PSQL
run: >
export DB_NAME_TEST="test_$DB_NAME"
if [ ${GITHUB_REF##*/} = "main" ]; then export DBBRANCH="main"; else export DBBRANCH="develop"; fi
echo "Branch ${GITHUB_REF##*/}, usign ${DBBRANCH} branch for database"
git clone --depth=1 --branch=$DBBRANCH https://github.com/openimis/database_postgresql.git ./sql_psql
echo 'set search_path to public' >> ~/.psqlrc
PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -c "DROP DATABASE IF EXISTS \"$DB_NAME_TEST\";"
PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -c "CREATE DATABASE \"$DB_NAME_TEST\";"
PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/00_dump.sql | grep . | uniq -c
PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/02_aux_functions.sql | grep . | uniq -c
PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/03_views.sql | grep . | uniq -c
PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/04_functions.sql | grep . | uniq -c
PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/05_stored_procs.sql | grep . | uniq -c
PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/demo_db.sql | grep . | uniq -c
env:
DB_HOST: localhost
DB_PORT: 5432
DB_NAME: imis
DB_USER: postgres
DB_PASSWORD: GitHub999
- name: Django tests PSQL
working-directory: ./openimis/openIMIS
id: django-test-psql
if: success() || failure()
run: >
python -V
# This has to exit 0, or report is not generated
python -m coverage run --source=$MODULE_NAME --omit=*/migrations/* ./manage.py test --keepdb $MODULE_NAME
env:
SECRET_KEY: secret
MODE: DEV
DB_ENGINE: django.db.backends.postgresql
DB_HOST: localhost
DB_PORT: 5432
DB_NAME: imis
DB_USER: postgres
DB_PASSWORD: GitHub999
SITE_ROOT: api
CI_EXCLUDED_MODULE: ${{ vars.CI_EXCLUDED_MODULE }}
- name: Django full PSQL
working-directory: ./openimis/openIMIS
id: django-full-psql
run: >
export MODULES=$(jq -r '(.modules[].name)' ../openimis.json | xargs)
# Remove some module from MODULES
MODULES=$(echo "$MODULES" | sed -E "s/\b$(echo "${CI_EXCLUDED_MODULE// /\\b|\\b}")\b/ /g" | xargs)
echo "MODULES=$MODULES" >> $GITHUB_ENV
python manage.py test --debug-mode --timing --keepdb $MODULES
continue-on-error: true
env:
SECRET_KEY: secret
MODE: DEV
DB_ENGINE: django.db.backends.postgresql
DB_HOST: localhost
DB_PORT: 5432
DB_NAME: imis
DB_USER: postgres
DB_PASSWORD: GitHub999
SITE_ROOT: api
CI_EXCLUDED_MODULE: ${{ vars.CI_EXCLUDED_MODULE }}
- name: Generate Coverage Report
if: always()
working-directory: ./openimis/openIMIS
run: >
python -m coverage xml -o coverage.xml
# Only leave actual code part, skip absuolute directory like
# /home/runner/work/openimis-be-invoice_py/openimis-be-invoice_py/current-module/
# It's required by circle-ci setup
mkdir coverage
sed -E 's|/home/runner/work/(.*?)/(.*?)/current-module/|./current-module/|' coverage.xml >> coverage/coverage1.xml
sed -E 's|/home/runner/work/(.*?)/(.*?)/current-module/|./|' coverage.xml >> coverage/coverage2.xml
cat coverage.xml
- name: Coverage results
if: success() || failure()
uses: actions/upload-artifact@v3
with:
name: coverage.xml
path: ./openimis/openIMIS/coverage
ci_full_psql_test:
name: Run All Tests (PSQL)
runs-on: ubuntu-20.04
needs: build_backend
services:
pgsql:
image: postgres
env:
DB_HOST: localhost
DB_PORT: 5432
POSTGRES_DB: imis
POSTGRES_USER: postgres
POSTGRES_PASSWORD: GitHub999
ports:
- 5432:5432
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s
--health-retries 10
steps:
- name: Set up Python 3.10
uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Download site-packages artifact
uses: actions/download-artifact@v2
with:
name: site-packages
path: /tmp/
- name: Locate site-packages
run: >
SITE_PACKAGES=$(python -c "from distutils.sysconfig import get_python_lib;
print(get_python_lib())")
echo "SITE_PACKAGES=$SITE_PACKAGES" >> $GITHUB_ENV
export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')"
echo "MODULE_NAME=$MODULE_NAME" >> $GITHUB_ENV
- name: Download site-packages artifact
uses: actions/download-artifact@v2
with:
name: code-artifacts
path: /tmp/
- name: Decompress artifacts
run: |
sudo tar -xzf /tmp/site-packages.tar.gz -C ${{ env.SITE_PACKAGES }}
sudo tar -xzf /tmp/openimis.tar.gz -C .
- name: Initialize PSQL
run: >
export DB_NAME_TEST="test_$DB_NAME"
if [ ${GITHUB_REF##*/} = "main" ]; then export DBBRANCH="main"; else export DBBRANCH="develop"; fi
echo "Branch ${GITHUB_REF##*/}, usign ${DBBRANCH} branch for database"
git clone --depth=1 --branch=$DBBRANCH https://github.com/openimis/database_postgresql.git ./sql_psql
echo 'set search_path to public' >> ~/.psqlrc
PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -c "DROP DATABASE IF EXISTS \"$DB_NAME_TEST\";"
PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -c "CREATE DATABASE \"$DB_NAME_TEST\";"
PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/00_dump.sql | grep . | uniq -c
PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/02_aux_functions.sql | grep . | uniq -c
PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/03_views.sql | grep . | uniq -c
PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/04_functions.sql | grep . | uniq -c
PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/05_stored_procs.sql | grep . | uniq -c
PGPASSWORD=GitHub999 psql -U $DB_USER -h $DB_HOST -d $DB_NAME_TEST -f ./sql_psql/database\ scripts/demo_db.sql | grep . | uniq -c
env:
DB_HOST: localhost
DB_PORT: 5432
DB_NAME: imis
DB_USER: postgres
DB_PASSWORD: GitHub999
- name: Django full PSQL
working-directory: ./openimis/openIMIS
id: django-full-psql
run: >
export MODULES=$(jq -r '(.modules[].name)' ../openimis.json | xargs)
# Remove some module from MODULES
MODULES=$(echo "$MODULES" | sed -E "s/\b$(echo "${CI_EXCLUDED_MODULE// /\\b|\\b}")\b/ /g" | xargs)
echo "MODULES=$MODULES" >> $GITHUB_ENV
python manage.py test --debug-mode --timing --keepdb $MODULES
env:
SECRET_KEY: secret
MODE: DEV
DB_ENGINE: django.db.backends.postgresql
DB_HOST: localhost
DB_PORT: 5432
DB_NAME: imis
DB_USER: postgres
DB_PASSWORD: GitHub999
SITE_ROOT: api
CI_EXCLUDED_MODULE: ${{ vars.CI_EXCLUDED_MODULE }}
- name: change outcome
run:
flake-8-linter:
name: Run flake8 check
runs-on: ubuntu-20.04
needs: build_backend
steps:
- name: Set up Python 3.10
uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Download site-packages artifact
uses: actions/download-artifact@v2
with:
name: site-packages
path: /tmp/
- name: Locate site-packages
run: >
SITE_PACKAGES=$(python -c "from distutils.sysconfig import get_python_lib;
print(get_python_lib())")
echo "SITE_PACKAGES=$SITE_PACKAGES" >> $GITHUB_ENV
- name: Download code artifact
uses: actions/download-artifact@v2
with:
name: code-artifacts
path: /tmp/
- name: Decompress artifacts
run: |
sudo tar -xzf /tmp/site-packages.tar.gz -C ${{ env.SITE_PACKAGES }}
sudo tar -xzf /tmp/openimis.tar.gz -C .
- name: Flake8 Style Check
working-directory: ./openimis/
if: success() || failure()
run: |
# Flake8 Report
MOD_DIR="../current-module/$MODULE_NAME"
python -m flake8 --output-file=flake8-report.txt $MOD_DIR --exit-zero
sed -i 's|\.\./current-module/||g' flake8-report.txt
python -m flake8 $MOD_DIR
- name: Flake8 results upload
if: success() || failure()
uses: actions/upload-artifact@v3
with:
name: flake8-report.txt
path: ./openimis/flake8-report.txt
sonar_scan:
name: Run Sonar Scan
runs-on: ubuntu-20.04
needs:
- ci_module_psql_test
- flake-8-linter
if: always()
env:
SONAR_PROJECT_KEY: ${{ inputs.SONAR_PROJECT_KEY }}
SONAR_ORGANIZATION: ${{ inputs.SONAR_ORGANIZATION }}
SONAR_PROJECT_NAME: ${{ inputs.SONAR_PROJECT_NAME }}
SONAR_PROJECT_VERSION: ${{ inputs.SONAR_PROJECT_VERSION }}
SONAR_PYTHON_VERSION: "3.10"
SONAR_FLAKE8_REPORT_PATHS: ../report/flake8-report.txt
SONAR_SOURCES: ${{ inputs.SONAR_SOURCES }}
steps:
- name: Set environment variables with default values
run: >
if [ -z "${{ inputs.SONAR_PROJECT_VERSION }}" ]; then
echo "SONAR_PROJECT_VERSION=1.0" >> $GITHUB_ENV
else
echo "SONAR_PROJECT_VERSION=${{ inputs.SONAR_PROJECT_VERSION }}" >> $GITHUB_ENV
fi
if [ -z "${{ inputs.SONAR_EXCLUSIONS }}" ]; then
echo "SONAR_EXCLUSIONS='**/migrations/**,**/static/**,**/media/**,**/tests/**'" >> $GITHUB_ENV
else
echo "SONAR_EXCLUSIONS=${{ inputs.SONAR_EXCLUSIONS }}" >> $GITHUB_ENV
fi
- name: Download coverage artifact
uses: actions/download-artifact@v2
with:
name: coverage.xml
path: report
- name: Download flake8 artifact
uses: actions/download-artifact@v2
with:
name: flake8-report.txt
path: report
- name: Download code artifact
uses: actions/download-artifact@v2
with:
name: code-artifacts
path: /tmp/
- name: Decompress artifacts
run: |
# This path is required by the structure of coverage.xml paths
sudo tar -xzf /tmp/openimis.tar.gz -C .
- name: Debug
run: |
ls -a
echo "---"
ls report
echo "---"
ls current-module -a
- name: SonarCloud Scan
uses: SonarSource/sonarcloud-github-action@master
with:
args: >
-Dsonar.projectKey=${{ inputs.SONAR_PROJECT_KEY }}
-Dsonar.organization=${{ inputs.SONAR_ORGANIZATION }}
-Dsonar.projectName=${{ inputs.SONAR_PROJECT_NAME }}
-Dsonar.projectVersion=${{ env.SONAR_PROJECT_VERSION }}
-Dsonar.python.version="3.10" -Dsonar.sources=current-module
-Dsonar.python.coverage.reportPaths=report/coverage1.xml,report/coverage2.xml
-Dsonar.python.flake8.reportPaths=report/flake8-report.txt
-Dsonar.exclusions=${{ env.SONAR_EXCLUSIONS }} -X
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}