test Tdd py3 PR

.github/workflows/main.yml

@@ -1,50 +1,118 @@
+# Can be run locally using act and docker, or in Fedora 39 also with podman, using:
+# act --bind --container-daemon-socket $XDG_RUNTIME_DIR/podman/podman.sock -W .github/workflows/main.yml
+
 name: Unit tests
 
-on: [push, pull_request]
+on: pull_request
+env:
+  PYTHONWARNINGS: "ignore"
 
 jobs:
-  test_py2:
-    runs-on: ubuntu-20.04
-
+  test:
+    strategy:
+      max-parallel: 1
+      matrix:
+        include:
+        - python-version: '3.11'
+          os: ubuntu-latest
+        - python-version: '2.7'
+          os: ubuntu-20.04
+        - python-version: '3.10'
+          os: ubuntu-latest
+        - python-version: '3.9'
+          os: ubuntu-latest
+        - python-version: '3.8'
+          os: ubuntu-latest
+        - python-version: '3.7'
+          os: ubuntu-latest
+        - python-version: '3.6'
+          os: ubuntu-20.04
+    runs-on: ${{ matrix.os }}
+    permissions:
+      # Gives the action the necessary permissions for publishing new
+      # comments in pull requests.
+      pull-requests: write
+      # Gives the action the necessary permissions for pushing data to the
+      # python-coverage-comment-action branch, and for editing existing
+      # comments (to avoid publishing multiple comments in the same PR)
+      contents: write
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
-          fetch-depth: 0
-      - name: Set up Python 2.7
-        uses: actions/setup-python@v2
+          # ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0  # To get changed lines for coverage-diff: origin/master..HEAD
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
         with:
-          python-version: '2.7'
+          python-version: ${{ matrix.python-version }}
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements-dev.txt
-          # FIXME: branding.py still has no permanent home
-          curl https://gist.github.com/ydirson/3c36a7e19d762cc529a6c82340894ccc/raw/5ca39f621b1feab813e171f535c1aad1bd483f1d/branding.py -O -L
-          pip install pyliblzma
-          pip install -e .
-          command -v xz
-
-      - name: Test
+          python -m pip install --quiet --upgrade pip
+          pip install --quiet -r requirements-dev.txt coveralls # FIXME:
+          if [ ${{ matrix.python-version }} = 2.7 ]; then python2 setup.py install --user;
+          else pip install --quiet -e .;fi
+
+      - name: Post a pylint Report on the Action's summary page
+        if: ${{ matrix.python-version == '3.9' }}
+        run: |
+          pip install --quiet pandas tabulate
+          tool=github-action-pylint-report.py
+          repo=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY
+          tree=$repo/tree/$GITHUB_REF_NAME
+          ./github-action-pylint-report.py xcp pylint-report.md $tree
+          message_count=`tail -1 pylint-report.md|cut -d'|' -f2`
+          if [ -e $GITHUB_STEP_SUMMARY ]; then  #  The "act" local runner does not have this:
+              echo -n "### [$tool]($tree/$tool) on **xcp/\\*\\*/*.py**" >>$GITHUB_STEP_SUMMARY
+              echo ": #Total Pylint messages: $message_count :rocket: " >>$GITHUB_STEP_SUMMARY
+              cat pylint-report.md                                      >>$GITHUB_STEP_SUMMARY
+          fi
+
+      - name: Run pytest with --cov and generate HTML coverage reports
+        id: coverage
+        if: ${{ matrix.python-version == '3.11' }}
         run: |
-          pytest --cov -rP
+          set -x;PYTHONPATH=$PWD pytest --cov
           coverage xml
-          coverage html
-          coverage html -d htmlcov-tests --include="tests/*"
-          diff-cover --html-report coverage-diff.html coverage.xml
+          coveralls --submit=coverage.xml
+          coverage html -d htmlcov-${{ matrix.python-version }}
+          coverage html -d htmlcov-tests-${{ matrix.python-version }} --include="tests/*"
+          diff-cover --compare-branch=origin/master --html-report coverage-diff-${{ matrix.python-version }}.html coverage.xml
+        env:
+          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+
+      - name: "Run tox for ${{ matrix.python-version }} (and also mypy if 3.11)"
+        if: ${{ matrix.python-version != '2.7' }}  # On Ubuntu 20.04 Python2.7, tox is older
+        run: |
+          set -ex;pip install --quiet --user --upgrade poetry tox tox-gh-actions
+          ~/.local/bin/tox  # Runs pytest & mypy in defined environments, simple to run locally
 
       - name: Pylint
         run: |
           pylint --version
           pylint --exit-zero xcp/ tests/ setup.py
           pylint --exit-zero --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" xcp/ tests/ setup.py > pylint.txt
-          diff-quality --violations=pylint --html-report pylint-diff.html pylint.txt
+          diff-quality --compare-branch=origin/master --violations=pylint --html-report pylint-diff-${{ matrix.python-version }}.html pylint.txt
+
+      - name: Get Cover
+        if: github.event_name == 'pull_request' && steps.coverage.outcome == 'success'
+        uses: orgoro/coverage@v3
+        with:
+          # https://github.com/marketplace/actions/python-coverage
+          coverageFile: coverage.xml
+          thresholdNew: 0.1
+          thresholdAll: 0.4
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        if: github.event_name == 'pull_request' && steps.coverage.outcome == 'success'
 
       - uses: actions/upload-artifact@v3
         with:
           name: Coverage and pylint reports
           path: |
-            coverage-diff.html
-            pylint-diff.html
-            htmlcov
-            htmlcov-tests
+            coverage-diff-${{ matrix.python-version }}.html
+            pylint-diff-${{ matrix.python-version }}.html
+            htmlcov-${{ matrix.python-version }}
+            htmlcov-tests-${{ matrix.python-version }}

branding.py

@@ -0,0 +1,36 @@
+BRAND_CONSOLE_URL = "https://xcp-ng.org"
+BRAND_CONSOLE = "XCP-ng Center"
+BRAND_GUEST_SHORT = "VM"
+BRAND_GUESTS_SHORT = "VMs"
+BRAND_GUESTS = "Virtual Machines"
+BRAND_GUEST = "Virtual Machine"
+BRAND_SERVERS = "XCP-ng Hosts"
+BRAND_SERVER = "XCP-ng Host"
+BRAND_VDI = ""
+COMPANY_DOMAIN = "xcp-ng.org"
+COMPANY_NAME_LEGAL = "Open Source"
+COMPANY_NAME = "Open Source"
+COMPANY_NAME_SHORT = "Open Source"
+COMPANY = "Open Source"
+COMPANY_PRODUCT_BRAND = "XCP-ng"
+COMPANY_WEBSITE = "https://xcp-ng.org"
+COPYRIGHT_YEARS = "2018-2022"
+ISO_PV_TOOLS_COPYRIGHT = "XCP-ng"
+ISO_PV_TOOLS_LABEL = "XCP-ng VM Tools"
+ISO_PV_TOOLS_PUBLISHER = "XCP-ng"
+PLATFORM_MAJOR_VERSION = "3"
+PLATFORM_MICRO_VERSION = "1"
+PLATFORM_MINOR_VERSION = "2"
+PLATFORM_NAME = "XCP"
+PLATFORM_ORGANISATION = "xen.org"
+PLATFORM_VERSION = "3.2.1"
+PLATFORM_WEBSITE = "www.xen.org"
+PRODUCT_BRAND = "XCP-ng"
+PRODUCT_BRAND_DASHED = "XCP-ng"
+PRODUCT_MAJOR_VERSION = "8"
+PRODUCT_MICRO_VERSION = "1"
+PRODUCT_MINOR_VERSION = "2"
+PRODUCT_NAME = "xenenterprise"
+PRODUCT_VERSION = "8.2.1"
+PRODUCT_VERSION_TEXT = "8.2"
+PRODUCT_VERSION_TEXT_SHORT = "8.2"

github-action-pylint-report.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+from pylint.reporters import JSONReporter
+from pylint.lint import Run
+from glob import glob
+from io import StringIO
+import pandas as pd
+import json
+import os
+import sys
+
+
+def del_dict_keys(r, *args: str):
+    for arg in args:
+        r.pop(arg, None)
+
+
+def pylint_project(module_path: str, branch_url: str):
+    pylint_options = []
+    pylint_overview = []
+    pylint_results = []
+    pylint_messages = {}
+    glob_pattern = os.path.join(module_path, "**", "*.py")
+    score_sum = 0.0
+    smell_sum = 0
+    for filepath in glob(glob_pattern, recursive=True):
+        if filepath[:11] == "__init__.py":
+            continue
+        reporter_buffer = StringIO()
+        results = Run(
+            [filepath] + pylint_options,
+            reporter=JSONReporter(reporter_buffer),
+            do_exit=False,
+        )
+        score = results.linter.stats.global_note
+        file_results = json.loads(reporter_buffer.getvalue())
+        if not file_results:
+            continue
+        for r in file_results:
+            sym = r["symbol"]
+            typ = r["type"]
+            if sym in [
+                "duplicate-except",
+                "super-init-not-called",
+                "attribute-defined-outside-init",
+            ]:
+                typ = "notice"
+            else:
+                pylint_messages[sym] = 0
+            filepath = r["path"]
+            print(
+                f"::{typ} file={filepath},line={r['line']},endLine={r['endLine']},title={sym}::{r['message']}"
+            )
+            try:
+                slashpos = filepath.rindex("/") + 1
+            except ValueError:
+                slashpos = 0
+            file = filepath[slashpos:]
+            linktext = file.removesuffix(".py")
+            r["path"] = f"[{linktext}]({branch_url}/{filepath}#L{r['line']})"
+            del_dict_keys(
+                r,
+                "module",
+                "column",
+                "endColumn",
+                "message-id",
+                "endLine",
+                "type",
+                "line",
+            )
+            try:
+                dotpos = r["obj"].rindex(".") + 1
+            except ValueError:
+                dotpos = 0
+            r["obj"] = r["obj"][dotpos:][:16]
+            r["symbol"] = sym[:32]
+            r["message"] = r["message"][:96]
+        pylint_results.extend(file_results)
+        smells = len(file_results)
+        smell_sum += smells
+        score_sum += score
+
+        pylint_overview.append(
+            {
+                "filepath": filepath,
+                "smells": smells,
+                "symbols": " ".join(pylint_messages.keys()),
+                "score": float("{:0.3f}".format(score)),
+            }
+        )
+    avg_score = score_sum / len(pylint_overview)
+    pylint_overview.append(
+        {
+            "filepath": "total",
+            "smells": smell_sum,
+            "symbols": "",
+            "score": float("{:0.3f}".format(avg_score)),
+        }
+    )
+    return pd.DataFrame(pylint_overview), pd.DataFrame(pylint_results)  # , avg_score
+
+
+def main(module_dir, output_file, branch_url):
+    panda_overview, panda_results = pylint_project(module_dir, branch_url)
+    print("### Overview")
+    print(panda_overview.to_markdown())
+    print("\n### Results")
+    print(panda_results.to_markdown())
+
+    summary_file = output_file or os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary_file:
+        with open(summary_file, "a", encoding="utf-8") as fp:
+            fp.write(panda_results.to_markdown())
+
+
+if __name__ == "__main__":
+    main(sys.argv[1], sys.argv[2], sys.argv[3])