Compile dependencies per platform

.github/dependabot.yml

@@ -48,7 +48,7 @@ updates:
           - "version-update:semver-minor"
 
   - package-ecosystem: "pip"
-    directory: "/"
+    directory: "/requirements/"
     labels:
       - "dependencies"
       - "python"

.github/workflows/fork_pr_benchmarks_run.yml

@@ -39,19 +39,13 @@ jobs:
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
           # The cache key depends on requirements.txt
-          key: ${{ matrix.platform.runner }}-pip-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('requirements.txt') }}
+          key: ${{ matrix.platform.runner }}-pip-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('requirements/dev_Linux_*_requirements.txt') }}
       # Build a virtualenv, but only if it doesn't already exist
-      - name: Populate pip cache
-        # requirements.txt is not reliable since across different platforms and
-        # their versions the pip package versions might vary. We regenerate it
-        # again from pyproject.toml every time when pyproject.toml or
-        # requirements.txt changes. The pinned versions in requirements.txt are
-        # tested by coverage since that is running on ubuntu which is also used
-        # to produce the main requirements.txt file.
+      - name: Find out Python version
+        run: echo "PYVERSION=$(python3 tools/py_version.py)" >> $GITHUB_ENV
+      - name: Install dependencies
         run: |
-          pip install pip-tools
-          pip-compile --generate-hashes --extra dev pyproject.toml > requirements.txt
-          pip install -r requirements.txt
+          pip install --require-hashes --no-deps -r requirements/dev_Linux_py${{ env.PYVERSION }}_requirements.txt
         if: steps.cache.outputs.cache-hit != 'true'
       - name: Save cache
         id: cache-save
@@ -61,7 +55,7 @@ jobs:
           key: ${{ steps.cache.outputs.cache-primary-key }}
         if: steps.cache.outputs.cache-hit != 'true'
       - name: Install sedpack locally
-        run: pip install --editable .
+        run: pip install --editable '.[dev]'
       # end: Use source sedpack package
 
       - name: Prepare benchmarking data

.github/workflows/maturin_ci.yml

@@ -17,7 +17,7 @@ on:
       - '**/*.py'
       - '**/*.rs'
       - 'pyproject.toml'
-      - 'requirements.txt'
+      - 'requirements/*.txt'
       - 'rust/Cargo.toml'
       - 'rust/Cargo.lock'
   merge_group:  # Just to be sure

.github/workflows/mypy.yml

@@ -36,14 +36,16 @@ jobs:
         id: cache
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
-          # The cache key depends on requirements.txt
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}-${{ hashFiles('pyproject.toml') }}
+          # The cache key depends on requirements file.
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/dev_Linux_*_requirements.txt') }}-${{ hashFiles('pyproject.toml') }}
           restore-keys: |
             ${{ runner.os }}-pip-
+      - name: Find out Python version
+        run: echo "PYVERSION=$(python3 tools/py_version.py)" >> $GITHUB_ENV
       # Build a virtualenv, but only if it doesn't already exist
       - name: Populate pip cache
         run: |
-          pip install --require-hashes --no-deps -r requirements.txt
+          pip install --require-hashes --no-deps -r requirements/dev_Linux_py${{ env.PYVERSION }}_requirements.txt
           pip install --editable ".[dev]"
       - name: Save cache
         id: cache-save

.github/workflows/pylint.yml

@@ -27,19 +27,19 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: ['3.10', '3.11', '3.12']
+        python-version: ['3.10', '3.11', '3.12', '3.13']
     steps:
       - uses: actions/checkout@v5
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
           cache: 'pip'
+      - name: Find out Python version
+        run: echo "PYVERSION=$(python3 tools/py_version.py)" >> $GITHUB_ENV
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip setuptools wheel
-          pip install --require-hashes --no-deps -r requirements.txt
-          pip install --upgrade pylint
+          pip install --require-hashes --no-deps -r requirements/dev_Linux_py${{ env.PYVERSION }}_requirements.txt
       - name: Register matcher
         run: echo ::add-matcher::./.github/python_matcher.json
       - name: Test code with pylint

.github/workflows/pytest.yml

@@ -27,8 +27,6 @@ jobs:
         platform:
           - runner: ubuntu-latest  # x64
           - runner: windows-latest  # x64
-          - runner: macos-14  # arm64
-          - runner: macos-15-intel  # Intel
           - runner: macos-latest  # arm64
     if: github.event_name != 'schedule'
     steps:
@@ -50,18 +48,12 @@ jobs:
           path: ${{ steps.pip-cache.outputs.dir }}
           key: ${{ matrix.platform.runner }}-pip-${{ hashFiles('pyproject.toml') }}
       # Build a virtualenv, but only if it doesn't already exist
-      - name: Populate pip cache
-        # requirements.txt is not reliable since across different platforms and
-        # their versions the pip package versions might vary. We regenerate it
-        # again from pyproject.toml every time when pyproject.toml or
-        # requirements.txt changes. The pinned versions in requirements.txt are
-        # tested by coverage since that is running on ubuntu which is also used
-        # to produce the main requirements.txt file.
+      - name: Find out Python version
+        shell: bash
+        run: echo "PYVERSION=$(python3 tools/py_version.py)" >> $GITHUB_ENV
+      - name: Install dependencies
         run: |
-          pip install pip==25.2 # TODO(remove the pinning) pip-tools issue 2252
-          pip install pip-tools
-          pip-compile --generate-hashes --extra dev pyproject.toml > dev_requirements.txt
-          pip install -r dev_requirements.txt
+          pip install --require-hashes --no-deps -r requirements/dev_${{ runner.os }}_py${{ env.PYVERSION }}_requirements.txt
         if: steps.cache.outputs.cache-hit != 'true'
       - name: Save cache
         id: cache-save
@@ -95,11 +87,14 @@ jobs:
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
           # The cache key depends on requirements.txt
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/dev_Linux_*_requirements.txt') }}
       # Build a virtualenv, but only if it doesn't already exist
-      - name: Populate pip cache
+      - name: Find out Python version
+        shell: bash
+        run: echo "PYVERSION=$(python3 tools/py_version.py)" >> $GITHUB_ENV
+      - name: Install dependencies
         run: |
-          python -m pip install --require-hashes --no-deps -r requirements.txt
+          pip install --require-hashes --no-deps -r requirements/dev_${{ runner.os }}_py${{ env.PYVERSION }}_requirements.txt
         if: steps.cache.outputs.cache-hit != 'true'
       - name: Save cache
         id: cache-save

README.md

@@ -27,7 +27,7 @@ Development dependencies:
 #### Development install
 
 1.  Clone the repository: `git clone https://github.com/google/sedpack`
-2.  Install dependencies: `python3 -m pip install --require-hashes -r requirements.txt`
+2.  Install dependencies: `python3 -m pip install --require-hashes -r requirements/Linux_py3.13_requirements.txt`
 3.  Install the package in development mode: `python3 -m pip install --editable
     .` (short `pip install -e .` or legacy `python setup.py develop`)
 
@@ -41,12 +41,7 @@ Development dependencies:
 
 ### Update dependencies
 
-Make sure to have: `sudo apt install python3 python3-pip python3-venv` and
-activated the virtual environment.
-
-Install requirements: `pip install --require-hashes -r base-tooling-requirements.txt`
-
-Update: `pip-compile pyproject.toml --generate-hashes --upgrade` and commit requirements.txt.
+Run `tools/pip_compile.sh` (which will install [uv](https://docs.astral.sh/uv/guides/install-python/)).
 
 #### Package install
 

project-words.txt

@@ -81,4 +81,5 @@ tinyaes
 tobytes
 uoffset
 xored
+ziglang
 zstd