Merge branch 'main' of https://github.com/pandas-dev/pandas into perf-p…

…andas-devgh-46470-slow-performance-of-to-dict
RogerThomas · Nov 22, 2022 · 6487cc9 · 6487cc9
2 parents ecfffa6 + 556848e
commit 6487cc9
Show file tree

Hide file tree

Showing 514 changed files with 7,975 additions and 11,423 deletions.
diff --git a/.github/workflows/32-bit-linux.yml b/.github/workflows/32-bit-linux.yml
@@ -5,12 +5,10 @@ on:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
   pull_request:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
     paths-ignore:
       - "doc/**"
 

diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
@@ -5,12 +5,10 @@ on:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
   pull_request:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
 
 env:
   ENV_FILE: environment.yml
@@ -88,7 +86,7 @@ jobs:
         echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV
       if: ${{ steps.build.outcome == 'success' && always() }}
 
-    - name: Typing
+    - name: Typing + pylint
       uses: pre-commit/action@v2.0.3
       with:
         extra_args: --hook-stage manual --all-files

diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml
@@ -5,14 +5,12 @@ on:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
     tags:
       - '*'
   pull_request:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
 
 env:
   ENV_FILE: environment.yml
@@ -66,22 +64,22 @@ jobs:
         mkdir -m 700 -p ~/.ssh
         echo "${{ secrets.server_ssh_key }}" > ~/.ssh/id_rsa
         chmod 600 ~/.ssh/id_rsa
-        echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts
+        echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFjYkJBk7sos+r7yATODogQc3jUdW1aascGpyOD4bohj8dWjzwLJv/OJ/fyOQ5lmj81WKDk67tGtqNJYGL9acII=" > ~/.ssh/known_hosts
       if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
 
     - name: Copy cheatsheets into site directory
       run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/
 
     - name: Upload web
-      run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas
+      run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ web@${{ secrets.server_ip }}:/var/www/html
       if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Upload dev docs
-      run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev
+      run: rsync -az --delete doc/build/html/ web@${{ secrets.server_ip }}:/var/www/html/pandas-docs/dev
       if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Upload prod docs
-      run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/version/${GITHUB_REF_NAME:1}
+      run: rsync -az --delete doc/build/html/ web@${{ secrets.server_ip }}:/var/www/html/pandas-docs/version/${GITHUB_REF_NAME:1}
       if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
 
     - name: Move docs into site directory

diff --git a/.github/workflows/macos-windows.yml b/.github/workflows/macos-windows.yml
@@ -5,12 +5,10 @@ on:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
   pull_request:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
     paths-ignore:
       - "doc/**"
 

diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml
@@ -0,0 +1,56 @@
+name: Package Checks
+
+on:
+  push:
+    branches:
+      - main
+      - 1.5.x
+  pull_request:
+    branches:
+      - main
+      - 1.5.x
+
+permissions:
+  contents: read
+
+jobs:
+  pip:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        extra: ["test", "performance", "timezone", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output_formatting", "clipboard", "compression", "all"]
+      fail-fast: false
+    name: Install Extras - ${{ matrix.extra }}
+    concurrency:
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pip-extras-${{ matrix.extra }}
+      cancel-in-progress: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        id: setup_python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.8'
+
+      # Hacky patch to disable building cython extensions.
+      # This job should only check that the extras successfully install.
+      - name: Disable building ext_modules
+        run: |
+          sed -i '/ext_modules=/d' setup.py
+        shell: bash -el {0}
+
+      - name: Install required dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools wheel python-dateutil pytz numpy cython
+        shell: bash -el {0}
+
+      - name: Pip install with extra
+        run: |
+          python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation
+        shell: bash -el {0}
diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml
@@ -25,12 +25,10 @@ on:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
   pull_request:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
     paths-ignore:
       - "doc/**"
 

diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml
@@ -5,12 +5,10 @@ on:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
   pull_request:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
     types: [labeled, opened, synchronize, reopened]
     paths-ignore:
       - "doc/**"
@@ -30,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
       group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{matrix.python-version}}-sdist
@@ -42,7 +40,7 @@ jobs:
         fetch-depth: 0
 
     - name: Set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -86,6 +84,8 @@ jobs:
           pip install numpy==1.20.3 ;;
         3.10)
           pip install numpy==1.21.2 ;;
+        3.11)
+          pip install numpy==1.23.2 ;;
         esac
 
     - name: Import pandas

diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -5,12 +5,10 @@ on:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
   pull_request:
     branches:
       - main
       - 1.5.x
-      - 1.4.x
     paths-ignore:
       - "doc/**"
 

diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -72,14 +72,22 @@ jobs:
         env:
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
 
-      # Used to test the built wheels
-      - uses: actions/setup-python@v4
+      # Used to test(Windows-only) and push the built wheels
+      # You might need to use setup-python separately
+      # if the new Python-dev version
+      # is unavailable on conda-forge.
+      - uses: conda-incubator/setup-miniconda@v2
         with:
+          auto-update-conda: true
           python-version: ${{ matrix.python[1] }}
+          activate-environment: test
+          channels: conda-forge, anaconda
+          channel-priority: true
+          mamba-version: "*"
 
       - name: Test wheels (Windows 64-bit only)
         if: ${{ matrix.buildplat[1] == 'win_amd64' }}
-        shell: cmd
+        shell: cmd /C CALL {0}
         run: |
           python ci/test_wheels.py wheelhouse
 
@@ -88,26 +96,15 @@ jobs:
           name: ${{ matrix.python[0] }}-${{ startsWith(matrix.buildplat[1], 'macosx') && 'macosx' || matrix.buildplat[1] }}
           path: ./wheelhouse/*.whl
 
-      # Used to push the built wheels
-      # TODO: once Python 3.11 is available on conda, de-dup with
-      # setup python above
-      - uses: conda-incubator/setup-miniconda@v2
-        with:
-          auto-update-conda: true
-          # Really doesn't matter what version we upload with
-          # just the version we test with
-          python-version: '3.8'
-          channels: conda-forge
-          channel-priority: true
-          mamba-version: "*"
 
       - name: Install anaconda client
         if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }}
+        shell: bash -el {0}
         run: conda install -q -y anaconda-client
 
 
       - name: Upload wheels
-        if: success()
+        if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }}
         shell: bash -el {0}
         env:
           PANDAS_STAGING_UPLOAD_TOKEN: ${{ secrets.PANDAS_STAGING_UPLOAD_TOKEN }}
@@ -180,11 +177,12 @@ jobs:
 
       - name: Install anaconda client
         if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }}
+        shell: bash -el {0}
         run: |
           conda install -q -y anaconda-client
 
       - name: Upload sdist
-        if: success()
+        if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }}
         shell: bash -el {0}
         env:
           PANDAS_STAGING_UPLOAD_TOKEN: ${{ secrets.PANDAS_STAGING_UPLOAD_TOKEN }}

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -18,16 +18,16 @@ repos:
         pass_filenames: true
         require_serial: false
 -   repo: https://github.com/python/black
-    rev: 22.8.0
+    rev: 22.10.0
     hooks:
     -   id: black
 -   repo: https://github.com/codespell-project/codespell
-    rev: v2.2.1
+    rev: v2.2.2
     hooks:
     -   id: codespell
         types_or: [python, rst, markdown]
 -   repo: https://github.com/MarcoGorelli/cython-lint
-    rev: v0.1.8
+    rev: v0.2.1
     hooks:
     -   id: cython-lint
 -   repo: https://github.com/pre-commit/pre-commit-hooks
@@ -60,15 +60,16 @@ repos:
         - flake8-bugbear==22.7.1
         - pandas-dev-flaker==0.5.0
 -   repo: https://github.com/pycqa/pylint
-    rev: v2.15.3
+    rev: v2.15.5
     hooks:
     -   id: pylint
+        stages: [manual]
 -   repo: https://github.com/PyCQA/isort
     rev: 5.10.1
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.38.2
+    rev: v3.2.0
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -83,7 +84,7 @@ repos:
         types: [text]  # overwrite types: [rst]
         types_or: [python, rst]
 -   repo: https://github.com/sphinx-contrib/sphinx-lint
-    rev: v0.6.1
+    rev: v0.6.7
     hooks:
     - id: sphinx-lint
 -   repo: https://github.com/asottile/yesqa
@@ -286,5 +287,4 @@ repos:
         language: python
         additional_dependencies:
         - autotyping==22.9.0
-        - black==22.6.0
         - libcst==0.4.7
diff --git a/README.md b/README.md
@@ -166,6 +166,6 @@ You can also triage issues which may include reproducing bug reports, or asking
 
 Or maybe through using pandas you have an idea of your own or are looking for something in the documentation and thinking ‘this can be improved’...you can do something about it!
 
-Feel free to ask questions on the [mailing list](https://groups.google.com/forum/?fromgroups#!forum/pydata) or on [Gitter](https://gitter.im/pydata/pandas).
+Feel free to ask questions on the [mailing list](https://groups.google.com/forum/?fromgroups#!forum/pydata) or on [Slack](https://pandas.pydata.org/docs/dev/development/community.html?highlight=slack#community-slack).
 
 As contributors and maintainers to this project, you are expected to abide by pandas' code of conduct. More information can be found at: [Contributor Code of Conduct](https://github.com/pandas-dev/.github/blob/master/CODE_OF_CONDUCT.md)
diff --git a/asv_bench/benchmarks/attrs_caching.py b/asv_bench/benchmarks/attrs_caching.py
@@ -15,7 +15,7 @@ def setup(self):
         self.cur_index = self.df.index
 
     def time_get_index(self):
-        self.foo = self.df.index
+        self.df.index
 
     def time_set_index(self):
         self.df.index = self.cur_index

diff --git a/asv_bench/benchmarks/frame_methods.py b/asv_bench/benchmarks/frame_methods.py
@@ -28,26 +28,6 @@ def time_frame_get_numeric_data(self):
         self.df._get_numeric_data()
 
 
-class Lookup:
-    def setup(self):
-        self.df = DataFrame(np.random.randn(10000, 8), columns=list("abcdefgh"))
-        self.df["foo"] = "bar"
-        self.row_labels = list(self.df.index[::10])[:900]
-        self.col_labels = list(self.df.columns) * 100
-        self.row_labels_all = np.array(
-            list(self.df.index) * len(self.df.columns), dtype="object"
-        )
-        self.col_labels_all = np.array(
-            list(self.df.columns) * len(self.df.index), dtype="object"
-        )
-
-    def time_frame_fancy_lookup(self):
-        self.df.lookup(self.row_labels, self.col_labels)
-
-    def time_frame_fancy_lookup_all(self):
-        self.df.lookup(self.row_labels_all, self.col_labels_all)
-
-
 class Reindex:
     def setup(self):
         N = 10**3
@@ -474,10 +454,10 @@ def setup(self, axis):
         )
 
     def time_count_level_multi(self, axis):
-        self.df.count(axis=axis, level=1)
+        self.df.count(axis=axis)
 
     def time_count_level_mixed_dtypes_multi(self, axis):
-        self.df_mixed.count(axis=axis, level=1)
+        self.df_mixed.count(axis=axis)
 
 
 class Apply: