sped up c.item(..., default=...) for python 3.10+ by a c-extension

westandskif · Jul 14, 2024 · ed4e8b0 · ed4e8b0
1 parent 8940873
commit ed4e8b0
Show file tree

Hide file tree

Showing 23 changed files with 647 additions and 287 deletions.
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -1,41 +1,146 @@
 name: tests
 
-on: [push, pull_request]
+on:
+  pull_request:
+  push:
+  workflow_dispatch:
 
-jobs:
-  build:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
-    runs-on: ubuntu-20.04
+jobs:
+  run_tests:
     strategy:
+      fail-fast: true
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
-
+        os-python-env:
+          - [ubuntu-20.04, "3.6",  "CONVTOOLS_CEXT_DISABLED=1"]
+          - [ubuntu-20.04, "3.7",  "CONVTOOLS_CEXT_DISABLED=1"]
+          - [ubuntu-20.04, "3.8",  "CONVTOOLS_CEXT_DISABLED=1"]
+          - [ubuntu-20.04, "3.9",  "CONVTOOLS_CEXT_DISABLED=1"]
+          - [ubuntu-20.04, "3.10", "CONVTOOLS_CEXT_DISABLED=1"]
+          - [ubuntu-20.04, "3.11", "CONVTOOLS_CEXT_DISABLED=1"]
+          - [ubuntu-20.04, "3.12", "CONVTOOLS_CEXT_DISABLED=1"]
+          - [ubuntu-20.04, "3.10", "CONVTOOLS_CEXT_DISABLED=0"]
+    runs-on: ${{ matrix.os-python-env[0] }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.os-python-env[1] }}
 
-      - name: main python version - 3.9
-        if: ${{ matrix.python-version == '3.9' }}
+      - name: run tests
+        if: ${{ matrix.os-python-env[1] != '3.10' }}
         run: |
-          pip install -r ci-requirements/requirements3.9.out
-          pip install -e .
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 src tests --count --select=E9,F63,F7,F82 --show-source --statistics
-          # stop the build if there is something black can fix
+          pip install -r ci-requirements/requirements${{ matrix.os-python-env[1] }}.out
+          ${{ matrix.os-python-env[2] }} pip install -e .
+          pytest --benchmark-skip --no-cov
+
+      - name: run tests & lint
+        if: ${{ matrix.os-python-env[1] == '3.10' }}
+        run: |
+          pip install -r ci-requirements/requirements${{ matrix.os-python-env[1] }}.out
           black src tests --check
           isort src tests --check
+          ruff check src
           mypy --check-untyped-defs src
-          ruff src
+          ${{ matrix.os-python-env[2] }} pip install -e .
+          pytest --benchmark-skip --no-cov
+
+  build_sdist:
+    name: Build source distribution
+    if: startsWith(github.ref, 'refs/tags/')
+    needs:
+      - run_tests
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.10
+
+      - name: upload coverage
+        run: |
+          pip install -r ci-requirements/requirements3.10.out
+          find src/convtools -name "*.so" -delete
+          CONVTOOLS_CEXT_DISABLED=1 pip install -e .
           pytest --benchmark-skip
+          CONVTOOLS_CEXT_DISABLED=0 pip install -e .
+          pytest --benchmark-skip --cov-append
           bash <(curl -s https://codecov.io/bash) -t ${{ secrets.CODECOV_TOKEN }}
 
-      - name: other python versions
-        if: ${{ matrix.python-version != '3.9' }}
+      - name: build sdist
         run: |
-          pip install -r ci-requirements/requirements${{ matrix.python-version }}.out
-          pip install -e .
-          pytest --benchmark-skip
+          pip install -U pip build
+          python -m build --sdist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: artifact-sdist
+          path: ./dist/*.tar.gz
+
+  build_wheels:
+    name: Build wheels
+    if: startsWith(github.ref, 'refs/tags/')
+    needs:
+      - run_tests
+    strategy:
+      matrix:
+        buildplat:
+          - [ubuntu-20.04, manylinux_x86_64]
+          - [ubuntu-20.04, musllinux_x86_64]
+          - [macos-13, macosx_x86_64]
+          - [macos-14, macosx_arm64]
+          - [windows-2019, win_amd64]
+          - [windows-2019, win_arm64]
+        python: ["cp310"]
+    runs-on: ${{ matrix.buildplat[0] }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.10
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2
+        env:
+          CIBW_BUILD: ${{ matrix.python }}-${{ matrix.buildplat[1] }}
+          CIBW_BEFORE_TEST: pip install -r ci-requirements/requirements3.10.out
+          CIBW_TEST_COMMAND: pytest --benchmark-skip --no-cov
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: artifact-${{ matrix.python }}-${{ matrix.buildplat[1] }}
+          path: ./wheelhouse/*.whl
+
+  pypi-publish:
+    name: Upload release to PyPI
+    if: startsWith(github.ref, 'refs/tags/')
+    needs:
+      - build_sdist
+      - build_wheels
+    runs-on: ubuntu-20.04
+    environment:
+      name: pypi
+      url: https://pypi.org/p/convtools
+    permissions:
+      id-token: write
+    steps:
+      - name: Download all the dists
+        uses: actions/download-artifact@v3
+        with:
+          path: dist/
+          pattern: artifact-*
+
+      - name: Release
+        uses: softprops/action-gh-release@v2
+
+      - name: Publish distribution 📦 to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/.gitignore b/.gitignore
@@ -53,3 +53,5 @@ MANIFEST
 src/convtools/mergers.p
 tmp/
 tests/csvs/out.csv
+
+.ccls-cache/
diff --git a/.ignore b/.ignore
@@ -5,6 +5,6 @@ src/convtools.egg-info/
 tmp/
 .git/
 .mypy_cache/
-ci-requirements/
 .pytest_cache/
 *.patch
+.ccls-cache/
diff --git a/Makefile b/Makefile
@@ -30,7 +30,7 @@ checks:
 	isort src tests
 	pylint src
 	mypy --check-untyped-defs src
-	ruff src
+	ruff check src
 
 bash-py%:
 	docker run --rm -it -v $$PWD:/mnt/convtools -w /mnt/convtools python:$* bash
@@ -48,3 +48,10 @@ test-py%:
 		"pip install -e . && pytest"
 
 test: test-py3.6 test-py3.7 test-py3.8 test-py3.9 test-py3.10 test-py3.11 test-py3.12
+
+# find src/convtools -name "*.so" -delete && CONVTOOLS_CEXT_DISABLED=1 python setup.py build_ext --build-lib src
+# find src/convtools -name "*.so" -delete && CONVTOOLS_CEXT_DISABLED=0 python setup.py build_ext --build-lib src
+# pytest --pdb -k test_cext
+# python -m build --sdist
+# python -m build --wheel
+# pytest -k 'not test_window' -k 'not test_date and not test_window'
diff --git a/benchmarks/benchmark_results_V1.csv b/benchmarks/benchmark_results_V1.csv
@@ -3,15 +3,18 @@ Darwin,arm64,3.10,3.10.9,CPython,Clang 14.0.0 (clang-1400.0.29.201),1.5.0,GroupB
 Darwin,arm64,3.10,3.10.9,CPython,Clang 14.0.0 (clang-1400.0.29.201),1.5.0,GroupBy1 - MANY_GROUPS,0.944842675212749
 Darwin,arm64,3.10,3.10.9,CPython,Clang 14.0.0 (clang-1400.0.29.201),1.5.0,Aggregate1,1.275670737228574
 Darwin,arm64,3.10,3.10.9,CPython,Clang 14.0.0 (clang-1400.0.29.201),1.5.0,DatetimeFormat2,1.384244545416736
+Darwin,arm64,3.10,3.10.9,CPython,Clang 14.0.0 (clang-1400.0.29.201),1.11.0,GetDefaultPositive,1.3871398683645213
 Darwin,arm64,3.10,3.10.9,CPython,Clang 14.0.0 (clang-1400.0.29.201),1.5.0,DatetimeFormat1,1.563485977949039
 Darwin,arm64,3.10,3.10.9,CPython,Clang 14.0.0 (clang-1400.0.29.201),1.5.0,IterOfIter1,1.9784229104298519
 Darwin,arm64,3.10,3.10.9,CPython,Clang 14.0.0 (clang-1400.0.29.201),1.5.0,DateFormat2,2.4857102895297283
+Darwin,arm64,3.10,3.10.9,CPython,Clang 14.0.0 (clang-1400.0.29.201),1.11.0,GetDefaultNegative,2.568550428138011
 Darwin,arm64,3.10,3.10.9,CPython,Clang 14.0.0 (clang-1400.0.29.201),1.5.0,TableDictReader,2.648612201665222
 Darwin,arm64,3.10,3.10.9,CPython,Clang 14.0.0 (clang-1400.0.29.201),1.5.0,DatetimeParse2,3.1014797033834314
 Darwin,arm64,3.10,3.10.9,CPython,Clang 14.0.0 (clang-1400.0.29.201),1.5.0,DatetimeParse1,3.5290177348090914
 Darwin,arm64,3.10,3.10.9,CPython,Clang 14.0.0 (clang-1400.0.29.201),1.5.0,DateParse1,3.6199648147122168
 Darwin,arm64,3.10,3.10.9,CPython,Clang 14.0.0 (clang-1400.0.29.201),1.5.0,DateParse2,3.637901388647322
 Darwin,arm64,3.10,3.10.9,CPython,Clang 14.0.0 (clang-1400.0.29.201),1.5.0,DateFormat1,5.900623431731855
+Darwin,arm64,3.11,3.11.4,CPython,Clang 14.0.3 (clang-1403.0.22.14.1),1.11.0,GetDefaultPositive,1.260915182429801
 Darwin,arm64,3.11,3.11.4,CPython,Clang 14.0.3 (clang-1403.0.22.14.1),1.5.0,Aggregate1,1.304136594230012
 Darwin,arm64,3.11,3.11.4,CPython,Clang 14.0.3 (clang-1403.0.22.14.1),1.5.0,DatetimeFormat2,1.3352037974418336
 Darwin,arm64,3.11,3.11.4,CPython,Clang 14.0.3 (clang-1403.0.22.14.1),1.5.0,GroupBy1 - MANY_GROUPS,1.3534988732037592
@@ -21,16 +24,19 @@ Darwin,arm64,3.11,3.11.4,CPython,Clang 14.0.3 (clang-1403.0.22.14.1),1.5.0,IterO
 Darwin,arm64,3.11,3.11.4,CPython,Clang 14.0.3 (clang-1403.0.22.14.1),1.5.0,DateFormat2,2.3978751310243416
 Darwin,arm64,3.11,3.11.4,CPython,Clang 14.0.3 (clang-1403.0.22.14.1),1.5.0,TableDictReader,2.613231834836861
 Darwin,arm64,3.11,3.11.4,CPython,Clang 14.0.3 (clang-1403.0.22.14.1),1.5.0,DatetimeParse2,3.106492966996272
+Darwin,arm64,3.11,3.11.4,CPython,Clang 14.0.3 (clang-1403.0.22.14.1),1.11.0,GetDefaultNegative,3.337299194764925
 Darwin,arm64,3.11,3.11.4,CPython,Clang 14.0.3 (clang-1403.0.22.14.1),1.5.0,DatetimeParse1,3.45907332726145
 Darwin,arm64,3.11,3.11.4,CPython,Clang 14.0.3 (clang-1403.0.22.14.1),1.5.0,DateParse1,3.5413213419123815
 Darwin,arm64,3.11,3.11.4,CPython,Clang 14.0.3 (clang-1403.0.22.14.1),1.5.0,DateParse2,3.622774755235592
 Darwin,arm64,3.11,3.11.4,CPython,Clang 14.0.3 (clang-1403.0.22.14.1),1.5.0,DateFormat1,6.163392245468712
+Darwin,arm64,3.12,3.12.1,CPython,Clang 15.0.0 (clang-1500.1.0.2.5),1.11.0,GetDefaultPositive,1.231069958275713
 Darwin,arm64,3.12,3.12.0,CPython,Clang 15.0.0 (clang-1500.0.40.1),1.5.0,Aggregate1,1.2840447782891051
 Darwin,arm64,3.12,3.12.0,CPython,Clang 15.0.0 (clang-1500.0.40.1),1.5.0,GroupBy1 - MANY_GROUPS,1.298331539739505
 Darwin,arm64,3.12,3.12.0,CPython,Clang 15.0.0 (clang-1500.0.40.1),1.5.0,DatetimeFormat2,1.4948781898859296
 Darwin,arm64,3.12,3.12.0,CPython,Clang 15.0.0 (clang-1500.0.40.1),1.5.0,GroupBy1 - FEW_GROUPS,1.7039472888506901
 Darwin,arm64,3.12,3.12.0,CPython,Clang 15.0.0 (clang-1500.0.40.1),1.5.0,DatetimeFormat1,1.7510580501523285
 Darwin,arm64,3.12,3.12.0,CPython,Clang 15.0.0 (clang-1500.0.40.1),1.5.0,IterOfIter1,1.907398707755145
+Darwin,arm64,3.12,3.12.1,CPython,Clang 15.0.0 (clang-1500.1.0.2.5),1.11.0,GetDefaultNegative,1.9462888690937525
 Darwin,arm64,3.12,3.12.0,CPython,Clang 15.0.0 (clang-1500.0.40.1),1.5.0,TableDictReader,2.2277725862485283
 Darwin,arm64,3.12,3.12.0,CPython,Clang 15.0.0 (clang-1500.0.40.1),1.5.0,DateFormat2,2.7831509034675364
 Darwin,arm64,3.12,3.12.0,CPython,Clang 15.0.0 (clang-1500.0.40.1),1.5.0,DatetimeParse2,3.0539091296415206

diff --git a/benchmarks/benchmarks.py b/benchmarks/benchmarks.py
@@ -95,9 +95,9 @@ def f(data):
             return [
                 {
                     "name": name,
-                    "avg": value["sum"] / value["count"]
-                    if value["count"]
-                    else 0,
+                    "avg": (
+                        value["sum"] / value["count"] if value["count"] else 0
+                    ),
                     "min": value["min"],
                     "max": value["max"],
                 }
@@ -247,3 +247,25 @@ def gen_data(self):
         return [
             (dt + timedelta(days=i)).strftime(self.FMT) for i in range(1000)
         ]
+
+
+class GetDefault(BaseBenchmark):
+    POSITIVE = True
+
+    def gen_converter(self):
+        return c.item(0, default=-1).gen_converter()
+
+    def gen_naive_implementations(self):
+        def f(data):
+            try:
+                return data[0]
+            except (TypeError, KeyError, IndexError):
+                return -1
+
+        yield c.call_func(f, c.this).gen_converter()
+
+    def gen_data(self):
+        if self.POSITIVE:
+            return {0: 1}
+        else:
+            return {10: 1}
diff --git a/ci-requirements/requirements3.10.in b/ci-requirements/requirements3.10.in
@@ -1,3 +1,22 @@
 pytest
 pytest-cov
 pytest-benchmark
+
+black
+isort
+mypy
+pylint
+ruff
+
+build
+
+markdown-include
+mdx-truly-sane-lists
+mkdocs
+mkdocs-exclude
+mkdocs-material
+pygments
+pymdown-extensions
+
+pydantic
+tabulate
diff --git a/ci-requirements/requirements3.10.out b/ci-requirements/requirements3.10.out
@@ -1,10 +1,58 @@
-coverage==7.4.1
-exceptiongroup==1.2.0
+annotated-types==0.7.0
+astroid==3.2.3
+Babel==2.15.0
+black==24.4.2
+build==1.2.1
+certifi==2024.7.4
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+coverage==7.6.0
+dill==0.3.8
+exceptiongroup==1.2.2
+ghp-import==2.1.0
+idna==3.7
 iniconfig==2.0.0
-packaging==23.2
-pluggy==1.4.0
+isort==5.13.2
+Jinja2==3.1.4
+Markdown==3.6
+markdown-include==0.8.1
+MarkupSafe==2.1.5
+mccabe==0.7.0
+mdx-truly-sane-lists==1.3
+mergedeep==1.3.4
+mkdocs==1.6.0
+mkdocs-exclude==1.0.2
+mkdocs-get-deps==0.2.0
+mkdocs-material==9.5.29
+mkdocs-material-extensions==1.3.1
+mypy==1.10.1
+mypy-extensions==1.0.0
+packaging==24.1
+paginate==0.5.6
+pathspec==0.12.1
+platformdirs==4.2.2
+pluggy==1.5.0
 py-cpuinfo==9.0.0
-pytest==8.0.0
+pydantic==2.8.2
+pydantic_core==2.20.1
+Pygments==2.18.0
+pylint==3.2.5
+pymdown-extensions==10.8.1
+pyproject_hooks==1.1.0
+pytest==8.2.2
 pytest-benchmark==4.0.0
-pytest-cov==4.1.0
+pytest-cov==5.0.0
+python-dateutil==2.9.0.post0
+PyYAML==6.0.1
+pyyaml_env_tag==0.1
+regex==2024.5.15
+requests==2.32.3
+ruff==0.5.2
+six==1.16.0
+tabulate==0.9.0
 tomli==2.0.1
+tomlkit==0.13.0
+typing_extensions==4.12.2
+urllib3==2.2.2
+watchdog==4.0.1
diff --git a/ci-requirements/requirements3.9.in b/ci-requirements/requirements3.9.in
@@ -1,23 +1,3 @@
 pytest
 pytest-cov
 pytest-benchmark
-
-black
-flake8
-isort
-mypy
-pylint
-ruff
-
-hatch
-
-markdown-include
-mdx-truly-sane-lists
-mkdocs
-mkdocs-exclude
-mkdocs-material
-pygments
-pymdown-extensions
-
-pydantic
-tabulate