Fix click compatibility and update python support

.github/workflows/pypi-release.yml

@@ -71,6 +71,9 @@ jobs:
     needs:
       - create-gh-release
     runs-on: ubuntu-24.04
+    environment: pypi-publish
+    permissions:
+      id-token: write
 
     steps:
       - name: Download built archives

CHANGELOG.rst

@@ -1,7 +1,19 @@
 Release notes
 =============
 
-Version 32.3.0 - (2025-03-06)
+Version 32.4.0 - (2025-10-22)
+-----------------------------
+
+- Fix ``click`` 8.3.0 compatibililty issues.
+  https://github.com/aboutcode-org/commoncode/pull/92
+
+- Drop python 3.9 support and add python 3.14
+  https://github.com/aboutcode-org/commoncode/pull/92
+
+- Handle paths with non-utf-8 bytes
+  https://github.com/aboutcode-org/commoncode/pull/91
+
+Version 32.3.0 - (2025-06-11)
 -----------------------------
 
 - Fix ``click`` compatibililty issues.

azure-pipelines.yml

@@ -13,47 +13,47 @@ jobs:
     parameters:
       job_name: ubuntu22_cpython
       image_name: ubuntu-22.04
-      python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+      python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"]
       test_suites:
         all: venv/bin/pytest -n 2 -vvs
 
   - template: etc/ci/azure-posix.yml
     parameters:
       job_name: ubuntu24_cpython
       image_name: ubuntu-24.04
-      python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+      python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"]
       test_suites:
         all: venv/bin/pytest -n 2 -vvs
 
   - template: etc/ci/azure-posix.yml
     parameters:
       job_name: macos13_cpython
       image_name: macOS-13
-      python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+      python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"]
       test_suites:
         all: venv/bin/pytest -n 2 -vvs
 
   - template: etc/ci/azure-posix.yml
     parameters:
         job_name: macos14_cpython
         image_name: macOS-14
-        python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13']
+        python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"]
         test_suites:
             all: venv/bin/pytest -n 2 -vvs
 
   - template: etc/ci/azure-win.yml
     parameters:
       job_name: win2025_cpython
       image_name: windows-2025
-      python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+      python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"]
       test_suites:
         all: venv\Scripts\pytest -n 2 -vvs
 
   - template: etc/ci/azure-win.yml
     parameters:
       job_name: win2022_cpython
       image_name: windows-2022
-      python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+      python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"]
       test_suites:
         all: venv\Scripts\pytest -n 2 -vvs
 
@@ -65,10 +65,10 @@ jobs:
     parameters:
       job_name: ubuntu24_test_all_supported_click_versions
       image_name: ubuntu-24.04
-      python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+      python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"]
       test_suites:
         click_versions: |
-          for clk_ver in 8.2.0 8.2.1 8.1.8 8.1.7 8.1.6 8.1.5 8.1.4 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7;
+          for clk_ver in 8.3.0 8.2.0 8.2.1 8.1.8 8.1.7 8.1.6 8.1.5 8.1.4 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7;
             do
               venv/bin/pip install click==$clk_ver;
               venv/bin/pytest -vvs tests/test_cliutils_progressbar.py;
@@ -82,7 +82,7 @@ jobs:
     parameters:
       job_name: ubuntu24_cpython_latest_from_pip
       image_name: ubuntu-24.04
-      python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+      python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"]
       test_suites:
         all: |
           venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .
@@ -93,7 +93,7 @@ jobs:
     parameters:
       job_name: win2022_cpython_latest_from_pip
       image_name: windows-2022
-      python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+      python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"]
       test_suites:
         all: |
           venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e .

configure

@@ -110,7 +110,7 @@ create_virtualenv() {
         fi
 
         $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \
-            --wheel embed --pip embed --setuptools embed \
+            --pip embed --setuptools embed \
             --seeder pip \
             --never-download \
             --no-periodic-update \

configure.bat

@@ -110,7 +110,7 @@ if not exist "%CFG_BIN_DIR%\python.exe" (
 
     if exist "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" (
         %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ^
-            --wheel embed --pip embed --setuptools embed ^
+            --pip embed --setuptools embed ^
             --seeder pip ^
             --never-download ^
             --no-periodic-update ^
@@ -126,7 +126,7 @@ if not exist "%CFG_BIN_DIR%\python.exe" (
             )
         )
         %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ^
-            --wheel embed --pip embed --setuptools embed ^
+            --pip embed --setuptools embed ^
             --seeder pip ^
             --never-download ^
             --no-periodic-update ^

etc/scripts/utils_requirements.py

@@ -153,7 +153,7 @@ def split_req(req):
     if not req:
         raise ValueError("req is required")
     # do not allow multiple constraints and tags
-    if not any(c in req for c in ",;"):
+    if any(c in req for c in ",;"):
         raise Exception(f"complex requirements with : or ; not supported: {req}")
     req = "".join(req.split())
     if not any(c in req for c in comparators):

requirements-dev.txt

@@ -2,7 +2,7 @@ aboutcode-toolkit==11.1.1
 black==23.1.0
 bleach==6.0.0
 boolean.py==4.0
-cffi==1.15.1
+cffi==2.0.0
 cryptography==39.0.1
 docutils==0.19
 et-xmlfile==1.1.0
@@ -27,11 +27,11 @@ packaging==23.0
 pathspec==0.11.0
 pkginfo==1.9.6
 platformdirs==3.0.0
-pluggy==1.0.0
+pluggy==1.6.0
 pycodestyle==2.13.0
 pycparser==2.21
 pygments==2.14.0
-pytest==7.2.1
+pytest==8.4.2
 pytest-xdist==3.2.0
 readme-renderer==37.3
 requests-toolbelt==0.10.1

requirements.txt

@@ -1,18 +1,17 @@
-attrs==25.3.0
-beautifulsoup4==4.13.4
-certifi==2025.4.26
+attrs==25.4.0
+beautifulsoup4==4.14.2
+certifi==2025.10.5
 chardet==5.2.0
-charset-normalizer==3.4.2
-click==8.2.1;python_version>='3.10'
-click==8.1.8;python_version<'3.10'
+charset-normalizer==3.4.4
+click==8.3.0
 idna==3.10
 pip==25.1.1
 PyYAML==6.0.2
 requests==2.32.2
 saneyaml==0.6.1
-setuptools==80.3.1
-soupsieve==2.7
+setuptools==80.9.0
+soupsieve==2.8
 text-unidecode==1.3
-typing_extensions==4.13.2
-urllib3==1.26.20
+typing_extensions==4.15.0
+urllib3==2.5.0
 wheel==0.38.4

setup.cfg

@@ -31,7 +31,7 @@ license_files =
     commoncode.ABOUT
 
 [options]
-python_requires = >=3.9
+python_requires = >=3.10
 package_dir =
     =src
 packages = find:
@@ -44,8 +44,7 @@ install_requires =
     attrs >= 18.1,!=20.1.0;python_version<'3.11'
     attrs >= 22.1.0;python_version>='3.11'
     Beautifulsoup4[chardet] >= 4.13.0
-    click >= 6.7, !=7.0;python_version<'3.10'
-    click >= 8.2.0;python_version>='3.10'
+    click >= 8.3.0
     requests[use_chardet_on_py3] >= 2.7.0
     saneyaml >= 0.5.2
     text_unidecode >= 1.0

src/commoncode/cliutils.py

@@ -22,6 +22,19 @@
 # Tracing flags
 TRACE = False
 
+try:
+    # Introduced in click 8.3.0 to have a sentinel value
+    # (https://peps.python.org/pep-0661/) for flag values
+    # and default values instead of None to differentiate
+    # between explicitly setting a `None` value and
+    # not setting and value.
+    # See https://github.com/pallets/click/pull/3030 and
+    # https://github.com/pallets/click/releases/tag/8.3.0
+    from click.core import UNSET
+except ImportError:
+    # to maintain compatibility with click < 8.3.0
+    UNSET = None
+
 
 def logger_debug(*args):
     pass
@@ -165,7 +178,7 @@ def format_options(self, ctx, formatter):
 
 
 class CompatProgressBar(ProgressBar):
-    # TODO Remove when dropping support for Python 3.9 or Click 8.1.
+    # TODO Remove when dropping support for Click 8.1.
     @property
     def is_hidden(self) -> bool:
         return self.hidden
@@ -188,7 +201,7 @@ def make_step(self, n_steps):
     # overriden and copied from Click to work around Click woes for
     # https://github.com/aboutcode-org/scancode-toolkit/issues/2583
     def generator(self):
-        if self.is_hidden:
+        if self.hidden:
             yield from self.iter
         else:
             for rv in self.iter:
@@ -207,7 +220,7 @@ class EnhancedProgressBar(DebuggedProgressBar):
     """
 
     def render_progress(self):
-        if not self.is_hidden:
+        if not self.hidden:
             return super(EnhancedProgressBar, self).render_progress()
 
 
@@ -228,7 +241,7 @@ class ProgressLogger(CompatProgressBar):
 
     def __init__(self, *args, **kwargs):
         super(ProgressLogger, self).__init__(*args, **kwargs)
-        self.is_hidden = False
+        self.hidden = False
 
     def render_progress(self):
         line = self.format_progress_line()
@@ -429,7 +442,7 @@ def __init__(
         confirmation_prompt=False,
         hide_input=False,
         is_flag=None,
-        flag_value=None,
+        flag_value=UNSET,
         multiple=False,
         count=False,
         allow_from_autoenv=True,

tests/test_filetype.py

@@ -61,7 +61,6 @@ def test_get_type(self):
         if on_posix:
             expected += [
                 ("2-SYMTYPE", "l"),
-                ("6-FIFOTYPE", "s"),
             ]
 
         try: