_python.yml

name: Reusable python linter workflow
on:
  workflow_call:
    inputs:
      python_versions:
        description: Python versions to use
        type: string
        required: true
      working_directory:
        description: Directory that must be run against the linters
        type: string
        required: true

      use_autoflake:
        description: Use autoflake linter
        default: false
        type: boolean
        required: false
      use_black:
        description: Use black formatter
        default: false
        type: boolean
        required: false
      use_isort:
        description: Use isort formatter
        default: false
        type: boolean
        required: false
      use_flake8:
        description: Use flake8 linter
        default: false
        type: boolean
        required: false
      use_pylint:
        description: Use pylint linter
        default: false
        type: boolean
        required: false
      use_bandit:
        description: Use bandit linter
        default: false
        type: boolean
        required: false

      run_codeql:
        description: Run codeql
        default: false
        type: boolean
        required: false

      requirements_path:
        description: Path to the requirements.txt file
        type: string
        required: true

      install_from:
        description: Directory that must be used to install the packages
        type: string
        required: false
        default: .

      packages_path:
        description: Path to the packages.txt file
        type: string
        required: false

      custom_command:
        description: String of custom command to run
        type: string
        required: false
      django_settings_module:
        description: Path to the django settings file
        type: string
        required: false
        default: ''

      check_migrations:
        description: Check if migrations are valid. Require django_settings_module to be set.
        type: boolean
        required: false
        default: false
      check_requirements_licenses:
        description: Check if requirements have a valid license. Require django_settings_module to be set.
        type: boolean
        required: false
        default: true
      ignore_requirements_licenses_regex:
        description: Regex of repositories of which ignore license
        type: string
        required: false
        default: uWSGI.*|lunardate.*|.*QuokkaClient.*|pyquokka.*

      check_docs_directory:
        description: Check docs using rstcheck inside this directory
        type: string
        required: false
        default: ''
      check_dockerfile:
        description: Check dockerfile build. WARNING action total time may increase significantly
        type: string
        required: false
        default: ''

      use_postgres:
        description: Use postgres service
        default: false
        type: boolean
        required: false
      use_elastic_search:
        description: Use elastic_search service
        default: false
        type: boolean
        required: false
      use_memcached:
        description: Use memcached service
        default: false
        type: boolean
        required: false
      use_redis:
        description: Use redis service
        default: false
        type: boolean
        required: false
      use_rabbitmq:
        description: Use rabbitmq service
        default: false
        type: boolean
        required: false
      use_mongo:
        description: Use mongo service
        default: false
        type: boolean
        required: false
      use_celery:
        description: Create a celery worker
        default: false
        type: boolean
        required: false

      use_coverage:
        description: Use coverage.py.
        default: false
        type: boolean
        required: false
      coverage_config_path:
        description: Path to the coverage.py config file
        type: string
        required: false
        default: .coveragerc
      upload_coverage:
        description: Upload coverage.py report to github
        default: false
        type: boolean
        required: false

      tags_for_slow_tests:
        description: Tags for tests that will be run only on master/main branch, space separated. Can be used only for django projects.
        default: slow
        type: string
        required: false
      tags_for_manual_tests:
        description: Tags for tests that will not be run on the CI, space separated. Can be used only for django projects.
        default: manual
        type: string
        required: false

      postgres_db:
        description: Postgres service db. Requires use_postgres to be true
        type: string
        required: false
        default: db
      postgres_user:
        description: Postgres service user. Requires use_postgres to be true
        type: string
        required: false
        default: user
      postgres_password:
        description: Postgres service password. Requires use_postgres to be true
        type: string
        required: false
        default: password
      postgres_version:
        description: Postgres service version. Requires use_postgres to be true
        type: string
        required: false
        default: latest

      mongo_version:
        description: Mongo container version
        type: string
        required: false
        default: latest
      elasticsearch_version:
        description: Elasticsearch container version
        type: string
        required: false
        default: 8.11.1
      elasticsearch_port:
        description: Elasticsearch container port
        type: string
        required: false
        default: 9200
      memcached_version:
        description: Memcached alpine container version
        type: string
        required: false
        default: latest
      redis_version:
        description: Redis alpine container version
        type: string
        required: false
        default: latest
      rabbitmq_version:
        description: RabbitMQ management-alpine container version
        type: string
        required: false
        default: 3

      celery_app:
        description: Celery app name. Requires use_celery to be true
        type: string
        required: false

      celery_queues:
        description: Celery queues separated by ,. Requires use_celery to be true
        type: string
        required: false
        default: default

      env:
        description: Environment variables to set
        type: string
        required: false
        default: >-
          {}

      max_timeout:
        description: Max time that the CI can be run
        type: number
        required: false
        default: 30

      ubuntu_version:
        description: Ubuntu version to use
        type: string
        default: latest
        required: false

jobs:
  python:
    name: Run python test suite
    runs-on: ubuntu-${{ inputs.ubuntu_version }}
    timeout-minutes: ${{ inputs.max_timeout }}
    strategy:
      matrix:
        python_version: ${{ fromJson(inputs.python_versions) }}
        language: ['python']
    env: ${{ fromJson(inputs.env) }}
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python_version }}

      - name: Inject stuff to environment
        run: |
          if [[ '${{ inputs.django_settings_module}}' != 'false' ]]; then
            echo "PYTHONPATH=${{ github.workspace }}/${{ inputs.working_directory }}" >> $GITHUB_ENV
            echo "DJANGO_SETTINGS_MODULE=${{ inputs.django_settings_module }}" >> $GITHUB_ENV
          fi
          if [[ '${{ inputs.run_codeql}}' != 'false' ]]; then
            echo "CODEQL_PYTHON=$(which python)" >> $GITHUB_ENV
          fi
        shell:
          bash

      - name: Install apt requirements
        if: inputs.packages_path
        uses: ./.github/actions/apt_requirements
        with:
          requirements_file: ${{ inputs.packages_path }}

      - name: Check requirements licenses
        if: inputs.check_requirements_licenses && steps.cache-virtualenv.outputs.cache-hit != 'true'
        id: license_check_report
        continue-on-error: true
        uses: pilosus/action-pip-license-checker@v2
        with:
          requirements: ${{ inputs.install_from }}/${{ inputs.requirements_path }}
          exclude: ${{ inputs.ignore_requirements_licenses_regex }}
          table-headers: true
          fail: 'StrongCopyleft,NetworkCopyleft,Error'
          fails-only: true

      - name: Print wrong licenses
        if: steps.license_check_report.outcome == 'failure'
        run: |
          echo "License check failed"
          echo "===================="
          echo "${{ steps.license_check_report.outputs.report }}"
          echo "===================="
          exit 1
        shell: bash

      # not the best solution because i do not think that dependabot supports this
      - name: Create requirements-linters.txt
        run: |
          echo > requirements-linters.txt

          if [[ '${{ inputs.use_black}}' != 'false' ]]; then
            echo "black==23.11.0" >> requirements-linters.txt
          fi

          if [[ '${{ inputs.use_isort}}' != 'false' ]]; then
            echo "isort==5.12.0" >> requirements-linters.txt
          fi

          if [[ '${{ inputs.use_flake8}}' != 'false' ]]; then
            echo "flake8==6.1.0" >> requirements-linters.txt
            if [[ -n '${{ inputs.django_settings_module }}' ]]; then
              echo "flake8-django==1.4" >> requirements-linters.txt
            fi
          fi

          if [[ '${{ inputs.use_pylint}}' != 'false' ]]; then
            echo "pylint==2.17.7" >> requirements-linters.txt
            if [[ -n '${{ inputs.django_settings_module }}' ]]; then
              echo "pylint-django==2.5.5" >> requirements-linters.txt
            fi
          fi

          if [[ '${{ inputs.use_bandit}}' != 'false' ]]; then
            echo "bandit==1.7.5" >> requirements-linters.txt
          fi
          if [[ '${{ inputs.use_autoflake}}' != 'false' ]]; then
            echo "autoflake==2.2.1" >> requirements-linters.txt
          fi
          cat $(echo ${{ inputs.requirements_path }} | sed -e 's/.txt/-linter.txt/') >> requirements-linters.txt 2>/dev/null || exit 0
        shell: bash
        working-directory: ${{ inputs.install_from }}

      - name: Create requirements-dev.txt
        run: |
          echo > requirements-dev.txt
          if [[ '${{ inputs.use_coverage }}' != 'false' ]]; then
            echo "coverage>=7.3.2" >> requirements-dev.txt
          fi
          cat $(echo ${{ inputs.requirements_path }} | sed -e 's/.txt/-dev.txt/') >> requirements-dev.txt 2>/dev/null || exit 0
        shell: bash
        working-directory: ${{ inputs.install_from }}

      - name: Create requirements-docs.txt
        run: |
          echo > requirements-docs.txt
          if [[ -n '${{ inputs.check_docs_directory }}' ]]; then
            echo "rstcheck[sphinx]" >> requirements-docs.txt 
            echo "sphinx==7.2.6" >> requirements-docs.txt
            echo "sphinx_rtd_theme==1.3.0" >> requirements-docs.txt
            echo "sphinxcontrib-spelling==8.0.0" >> requirements-docs.txt
            if [[ -n '${{ inputs.django_settings_module }}' ]]; then
              echo "sphinxcontrib-django2==1.9" >> requirements-docs.txt
            fi
            cat $(echo ${{ inputs.requirements_path }} | sed -e 's/.txt/-docs.txt/') >> requirements-docs.txt 2>/dev/null || exit 0 
          fi
        shell: bash
        working-directory: ${{ inputs.install_from }}

      - name: Check virtualenv cache
        uses: syphar/restore-virtualenv@v1
        id: cache-virtualenv
        with:
          requirement_files: |
            ${{ inputs.install_from }}/${{ inputs.requirements_path }}
            ${{ inputs.install_from }}/requirements-dev.txt
            ${{ inputs.install_from }}/requirements-linters.txt
            ${{ inputs.install_from }}/requirements-docs.txt

      - name: Check pip cache
        uses: syphar/restore-pip-download-cache@v1
        if: steps.cache-virtualenv.outputs.cache-hit != 'true'
        with:
          requirement_files: |
            ${{ inputs.install_from }}/${{ inputs.requirements_path }}
            ${{ inputs.install_from }}/requirements-dev.txt
            ${{ inputs.install_from }}/requirements-linters.txt
            ${{ inputs.install_from }}/requirements-docs.txt

      - name: Install requirements
        if: steps.cache-virtualenv.outputs.cache-hit != 'true'
        run: |
          pip install -r ${{ inputs.requirements_path }}
          pip install -r requirements-dev.txt
          pip install -r requirements-linters.txt
          pip install -r requirements-docs.txt
        shell: bash
        working-directory: ${{ inputs.install_from }}

      - name: Run linters
        uses: ./.github/actions/python_linter
        if: inputs.use_black || inputs.use_isort || inputs.use_flake8 || inputs.use_pylint || inputs.use_bandit || inputs.use_autoflake
        with:
          working_directory: ${{ inputs.working_directory }}
          use_black: ${{ inputs.use_black }}
          use_isort: ${{ inputs.use_isort }}
          use_flake8: ${{ inputs.use_flake8 }}
          use_pylint: ${{ inputs.use_pylint }}
          use_bandit: ${{ inputs.use_bandit }}
          use_autoflake: ${{ inputs.use_autoflake }}

      - name: Run CodeQL
        if: inputs.run_codeql
        uses: ./.github/actions/codeql
        with:
          language: python
          working_directory: ${{ inputs.working_directory }}

      - name: Build Docs
        if: inputs.check_docs_directory
        run: |
          rstcheck -r  --report-level error --ignore-directives automodule,autoclass .
          python3 -m sphinx -a -T -E -b html -d  _build/doctrees -D language=en . _build/html
        shell: bash
        working-directory: ${{ inputs.check_docs_directory }}

      - name: Build DockerFile
        if: inputs.check_dockerfile
        run: |
           docker build -f ${{ inputs.check_dockerfile }} .
        working-directory: ${{ inputs.working_directory }}

      - name: Start services
        uses: ./.github/actions/services
        if: inputs.use_postgres || inputs.use_elastic_search || inputs.use_memcached ||  inputs.use_redis || inputs.use_rabbitmq || inputs.use_mongo
        with:
          use_mongo: ${{ inputs.use_mongo }}
          use_redis: ${{ inputs.use_redis }}
          use_postgres: ${{ inputs.use_postgres }}
          use_elastic_search: ${{ inputs.use_elastic_search }}
          use_memcached: ${{ inputs.use_memcached }}
          use_rabbitmq: ${{ inputs.use_rabbitmq }}
          postgres_db: ${{ inputs.postgres_db }}
          postgres_user: ${{ inputs.postgres_user }}
          postgres_password: ${{ inputs.postgres_password }}
          postgres_version: ${{ inputs.postgres_version }}
          mongo_version: ${{ inputs.mongo_version }}
          elasticsearch_version:  ${{ inputs.elasticsearch_version }}
          elasticsearch_port: ${{ inputs.elasticsearch_port }}
          memcached_version:  ${{ inputs.memcached_version }}
          redis_version:  ${{ inputs.redis_version }}
          rabbitmq_version:  ${{ inputs.rabbitmq_version }}

      - name: Start celery worker
        if: inputs.use_celery
        run: |
          celery -A ${{ inputs.celery_app }} worker -E -c 1 -n worker -Q ${{ inputs.celery_queues }} --detach
        working-directory: ${{ inputs.working_directory }}

      - name: Run custom command
        if: inputs.custom_command
        run: |
          COMMAND='${{ inputs.custom_command }}'
          echo "Running command: $COMMAND"
          eval $COMMAND
        working-directory: ${{ inputs.working_directory }}
        shell: bash

      - name: Check migrations
        if: inputs.check_migrations && inputs.django_settings_module
        run: |
          django-admin makemigrations --dry-run >/dev/null
        working-directory: ${{ inputs.working_directory }}
        shell: bash

      - name: Run unittest
        run: |
          CMD="python"
          if [[ '${{ inputs.use_coverage }}' != 'false' ]]; then
              CMD="coverage run --rcfile=${{ inputs.coverage_config_path }}"
          
              if [[ -n '${{ inputs.django_settings_module }}' ]]; then
                CMD="${CMD} -a manage.py test"
                # manage manual tags
                if [[ -n '${{ inputs.tags_for_manual_tests }}' ]]; then
                  for tag in ${{ inputs.tags_for_manual_tests }}; do
                    CMD="${CMD} --exclude-tag=$tag"
                  done
                fi
                # manage slow tags
                if [[ '${{ github.base_ref }}' != 'master' && '${{ github.base_ref }}' != 'main' ]]; then
                  if [[ -n '${{ inputs.tags_for_slow_tests }}' ]]; then
                    for tag in ${{ inputs.tags_for_slow_tests }}; do
                      CMD="${CMD} --exclude-tag=$tag"
                    done
                  fi
                fi
          
              else
                CMD="${CMD} -m unittest discover"
              fi

          else
            CMD="python -m unittest discover"
          fi
          CMD="${CMD} --failfast"
          echo "Running command: ${CMD}"
          $CMD
        working-directory: ${{ inputs.working_directory }}
        env: ${{ secrets }}
        shell: bash

      - name: Create coverage output
        if: inputs.use_coverage && inputs.upload_coverage
        id: coverage-output
        run: |
          echo "Uploading coverage.py report"
          echo "## Coverage.py report" >> $GITHUB_STEP_SUMMARY
          echo "$(coverage report -m --format=markdown)" >> $GITHUB_STEP_SUMMARY
        working-directory: ${{ inputs.working_directory }}