Merge branch 'main' of https://github.com/pandas-dev/pandas into perf-p…

…andas-devgh-46470-slow-performance-of-to-dict

.circleci/config.yml

@@ -13,7 +13,7 @@ jobs:
       PANDAS_CI: "1"
     steps:
       - checkout
-      - run: ci/setup_env.sh
+      - run: .circleci/setup_env.sh
       - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh
 
 workflows:

ci/setup_env.sh → .circleci/setup_env.sh

@@ -51,8 +51,7 @@ echo
 echo "update conda"
 conda config --set ssl_verify false
 conda config --set quiet true --set always_yes true --set changeps1 false
-# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
-conda install -y -c conda-forge -n base 'mamba>=0.21.2' pip
+conda install -y -c conda-forge -n base 'mamba>=0.21.2' pip setuptools
 
 echo "conda info -a"
 conda info -a
@@ -67,21 +66,10 @@ echo "mamba env update --file=${ENV_FILE}"
 # See https://github.com/mamba-org/mamba/issues/633
 mamba create -q -n pandas-dev
 time mamba env update -n pandas-dev --file="${ENV_FILE}"
-# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
-mamba install -n pandas-dev 'setuptools<60'
 
 echo "conda list -n pandas-dev"
 conda list -n pandas-dev
 
-# From pyarrow on MacOS
-# ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib
-# Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib
-# Reason: image not found
-if [[ "$(uname)" == 'Darwin' ]]; then
-    echo "Update pyarrow for pyarrow on MacOS"
-    conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=6
-fi
-
 if [[ "$BITS32" == "yes" ]]; then
     # activate 32-bit compiler
     export CONDA_BUILD=1
@@ -113,6 +101,6 @@ echo "Build extensions"
 python setup.py build_ext -q -j3
 
 echo "Install pandas"
-python -m pip install --no-build-isolation -e .
+python -m pip install --no-build-isolation --no-use-pep517 -e .
 
 echo "done"

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,5 @@
 - [ ] closes #xxxx (Replace xxxx with the Github issue number)
 - [ ] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
 - [ ] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
+- [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
 - [ ] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.

.github/actions/build_pandas/action.yml

@@ -12,6 +12,11 @@ runs:
 
     - name: Build Pandas
       run: |
-        python setup.py build_ext -j 2
+        python setup.py build_ext -j $N_JOBS
         python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
       shell: bash -el {0}
+      env:
+        # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873
+        # GH 47305: Parallel build causes flaky ImportError: /home/runner/work/pandas/pandas/pandas/_libs/tslibs/timestamps.cpython-38-x86_64-linux-gnu.so: undefined symbol: pandas_datetime_to_datetimestruct
+        N_JOBS: 1
+        #N_JOBS: ${{ runner.os == 'Windows' && 1 || 2 }}

.github/actions/run-tests/action.yml

@@ -0,0 +1,27 @@
+name: Run tests and report results
+runs:
+  using: composite
+  steps:
+    - name: Test
+      run: ci/run_tests.sh
+      shell: bash -el {0}
+
+    - name: Publish test results
+      uses: actions/upload-artifact@v2
+      with:
+        name: Test results
+        path: test-data.xml
+      if: failure()
+
+    - name: Report Coverage
+      run: coverage report -m
+      shell: bash -el {0}
+      if: failure()
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v2
+      with:
+        flags: unittests
+        name: codecov-pandas
+        fail_ci_if_error: false
+      if: failure()

.github/actions/setup-conda/action.yml

@@ -0,0 +1,37 @@
+name: Set up Conda environment
+inputs:
+  environment-file:
+    description: Conda environment file to use.
+    default: environment.yml
+  environment-name:
+    description: Name to use for the Conda environment
+    default: test
+  python-version:
+    description: Python version to install
+    required: false
+  pyarrow-version:
+    description: If set, overrides the PyArrow version in the Conda environment to the given string.
+    required: false
+runs:
+  using: composite
+  steps:
+    - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
+      run: |
+        grep -q '  - pyarrow' ${{ inputs.environment-file }}
+        sed -i"" -e "s/  - pyarrow/  - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
+        cat ${{ inputs.environment-file }}
+      shell: bash
+      if: ${{ inputs.pyarrow-version }}
+
+    - name: Install ${{ inputs.environment-file }}
+      uses: conda-incubator/setup-miniconda@v2.1.1
+      with:
+        environment-file: ${{ inputs.environment-file }}
+        activate-environment: ${{ inputs.environment-name }}
+        python-version: ${{ inputs.python-version }}
+        channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }}
+        channels: conda-forge
+        mamba-version: "0.24"
+        use-mamba: true
+        use-only-tar-bz2: true
+        condarc-file: ci/condarc.yml

.github/workflows/32-bit-linux.yml

@@ -0,0 +1,49 @@
+name: 32 Bit Linux
+
+on:
+  push:
+    branches:
+      - main
+      - 1.4.x
+  pull_request:
+    branches:
+      - main
+      - 1.4.x
+    paths-ignore:
+      - "doc/**"
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Run 32-bit manylinux2014 Docker Build / Tests
+        run: |
+          # Without this (line 34), versioneer will not be able to determine the pandas version.
+          # This is because of a security update to git that blocks it from reading the config folder if
+          # it is not owned by the current user. We hit this since the "mounted" folder is not hit by the
+          # Docker container.
+          # xref https://github.com/pypa/manylinux/issues/1309
+          docker pull quay.io/pypa/manylinux2014_i686
+          docker run --platform linux/386 -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \
+          /bin/bash -xc "cd pandas && \
+          git config --global --add safe.directory /pandas && \
+          /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
+          . ~/virtualenvs/pandas-dev/bin/activate && \
+          python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
+          pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
+          python setup.py build_ext -q -j2 && \
+          python -m pip install --no-build-isolation --no-use-pep517 -e . && \
+          export PANDAS_CI=1 && \
+          pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml"
+
+      - name: Publish test results for Python 3.8-32 bit full Linux
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test results
+          path: test-data.xml
+        if: failure()

.github/workflows/asv-bot.yml

@@ -41,12 +41,8 @@ jobs:
 
         # Although asv sets up its own env, deps are still needed
         # during discovery process
-      - uses: conda-incubator/setup-miniconda@v2.1.1
-        with:
-          activate-environment: pandas-dev
-          channel-priority: strict
-          environment-file: ${{ env.ENV_FILE }}
-          use-only-tar-bz2: true
+      - name: Set up Conda
+        uses: ./.github/actions/setup-conda
 
       - name: Run benchmarks
         id: bench