Merge remote-tracking branch 'upstream/main' into stubsabot/protobuf

.flake8

@@ -6,6 +6,7 @@
 #     E501   line too long
 
 # Some rules are considered irrelevant to stub files:
+#     B      All flake8-bugbear rules are .py-specific
 #     E701   multiple statements on one line (colon) -- disallows "..." on the same line
 #     F401   imported but unused -- does not recognize re-exports
 #            https://github.com/PyCQA/pyflakes/issues/474
@@ -25,18 +26,20 @@
 #            into the typeshed codebase to unblock flake8-pyi PRs, meaning these comments
 #            have "no matching violations" since the relevant flake8-pyi checks haven't
 #            yet been released.
+#     Y041   Use "complex" instead of "float | complex" (see "The numeric tower" in PEP 484)
+#            TODO this check is somewhat broken, see https://github.com/PyCQA/flake8-pyi/issues/299
 
 [flake8]
 per-file-ignores =
   *.py: E203, E301, E302, E305, E501
-  *.pyi: E301, E302, E305, E501, E701, E741, NQA102, F401, F403, F405, F822, Y037
+  *.pyi: B, E301, E302, E305, E501, E701, E741, NQA102, F401, F403, F405, F822, Y037, Y041
   # Since typing.pyi defines "overload" this is not recognized by flake8 as typing.overload.
   # Unfortunately, flake8 does not allow to "noqa" just a specific error inside the file itself.
   # https://github.com/PyCQA/flake8/issues/1079
   #     F811 redefinition of unused '...'
-  stdlib/typing.pyi: E301, E302, E305, E501, E701, E741, NQA102, F401, F403, F405, F811, F822, Y037
+  stdlib/typing.pyi: B, E301, E302, E305, E501, E701, E741, NQA102, F401, F403, F405, F811, F822, Y037
   # Generated protobuf files include docstrings
-  *_pb2.pyi: E301, E302, E305, E501, E701, NQA102, Y021, Y026
+  *_pb2.pyi: B, E301, E302, E305, E501, E701, NQA102, Y021, Y026
 
 exclude = .venv*,.git
 noqa_require_code = true

.github/workflows/daily.yml

@@ -4,11 +4,19 @@ on:
   workflow_dispatch:
   schedule:
     - cron: "0 0 * * *"
+  pull_request:
+    paths:
+      - 'requirements-tests.txt'
+      - '.github/workflows/daily.yml'
 
 # Please keep the permissions minimal, as stubtest runs arbitrary code from pypi.
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 env:
   PIP_DISABLE_PIP_VERSION_CHECK: 1
 
@@ -30,8 +38,8 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Update pip
-        run: python -m pip install -U pip
+          cache: pip
+          cache-dependency-path: requirements-tests.txt
       - name: Install dependencies
         run: pip install $(grep tomli== requirements-tests.txt) $(grep mypy== requirements-tests.txt)
       - name: Run stubtest
@@ -40,24 +48,35 @@ jobs:
   stubtest-third-party:
     name: Check third party stubs with stubtest
     if: ${{ github.repository == 'python/typeshed' || github.event_name == 'workflow_dispatch' }}
-    runs-on: ubuntu-20.04
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
         shard-index: [0, 1, 2, 3]
       fail-fast: false
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+          cache: pip
+          cache-dependency-path: |
+            requirements-tests.txt
+            stubs/**/@tests/requirements-stubtest.txt
+            stubs/**/METADATA.toml
       - name: Install dependencies
         run: pip install -r requirements-tests.txt
-      - name: Install apt packages
-        run: |
-          sudo apt update
-          sudo apt install -y $(python tests/get_apt_packages.py)
       - name: Run stubtest
-        run: xvfb-run python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }}
+        shell: bash
+        run: |
+          if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then
+            sudo apt update
+            sudo apt install -y $(python tests/get_packages.py)
+
+            xvfb-run python tests/stubtest_third_party.py --specified-stubs-only --num-shards 4 --shard-index ${{ matrix.shard-index }}
+          else
+            python tests/stubtest_third_party.py --specified-stubs-only --num-shards 4 --shard-index ${{ matrix.shard-index }}
+          fi
 
   stub-uploader:
     name: Run the stub_uploader tests
@@ -76,6 +95,8 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+          cache: pip
+          cache-dependency-path: stub_uploader/requirements.txt
       - name: Run tests
         run: |
           cd stub_uploader
@@ -87,7 +108,7 @@ jobs:
     name: Create an issue if daily tests failed
     runs-on: ubuntu-latest
     needs: [stubtest-stdlib, stubtest-third-party, stub-uploader]
-    if: ${{ github.repository == 'python/typeshed' && always() && (needs.stubtest-stdlib.result == 'failure' || needs.stubtest-third-party.result == 'failure' || needs.stub-uploader.result == 'failure') }}
+    if: ${{ github.repository == 'python/typeshed' && always() && github.event_name != 'pull_request' && (needs.stubtest-stdlib.result == 'failure' || needs.stubtest-third-party.result == 'failure' || needs.stub-uploader.result == 'failure') }}
     permissions:
       issues: write
     steps:

.github/workflows/mypy_primer.yml

@@ -1,7 +1,7 @@
 name: Run mypy_primer
 
 on:
-  # Only run on PR, since we diff against master
+  # Only run on PR, since we diff against main
   pull_request:
     paths:
     - 'stdlib/**'
@@ -41,17 +41,17 @@ jobs:
           MYPY_VERSION=$(grep mypy== requirements-tests.txt | cut -d = -f 3)
           echo "new commit"
           git rev-list --format=%s --max-count=1 $GITHUB_SHA
-          git checkout -b upstream_master origin/master
+          git checkout -b upstream_main origin/main
           echo "base commit"
-          git rev-list --format=%s --max-count=1 upstream_master
+          git rev-list --format=%s --max-count=1 upstream_main
           echo ''
           cd ..
           # fail action if exit code isn't zero or one
           (
             mypy_primer \
             --new v${MYPY_VERSION} --old v${MYPY_VERSION} \
             --custom-typeshed-repo typeshed_to_test \
-            --new-typeshed $GITHUB_SHA --old-typeshed upstream_master \
+            --new-typeshed $GITHUB_SHA --old-typeshed upstream_main \
             --num-shards 4 --shard-index ${{ matrix.shard-index }} \
             --debug \
             --output concise \

.github/workflows/stubsabot.yml

@@ -24,6 +24,8 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+          cache: pip
+          cache-dependency-path: requirements-tests.txt
       - name: git config
         run: |
           git config --global user.name stubsabot

.github/workflows/stubtest_stdlib.yml

@@ -1,17 +1,16 @@
-name: Run stubtest
+name: Stdlib stubtest
 
 on:
   workflow_dispatch:
   push:
     branches:
       - main
-      - master
   pull_request:
     paths:
       - 'stdlib/**'
       - '.github/workflows/stubtest_stdlib.yml'
       - 'tests/**'
-      - 'requirements-tests.txt'
+      # When requirements.txt changes, we run `daily.yml`, which includes stdlib stubtest
 
 permissions:
   contents: read
@@ -40,8 +39,8 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Update pip
-        run: python -m pip install -U pip
+          cache: pip
+          cache-dependency-path: requirements-tests.txt
       - name: Install dependencies
         run: pip install $(grep mypy== requirements-tests.txt)
       - name: Run stubtest

.github/workflows/stubtest_third_party.yml

@@ -0,0 +1,80 @@
+name: Third-party stubtest
+
+on:
+  pull_request:
+    paths:
+      - 'stubs/**'
+      - '.github/workflows/stubtest_third_party.yml'
+      - 'tests/**'
+      # When requirements.txt changes, we run `daily.yml`, which includes third-party stubtest
+
+permissions:
+  contents: read
+
+env:
+  PIP_DISABLE_PIP_VERSION_CHECK: 1
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  stubtest-third-party:
+    name: Check third party stubs with stubtest
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+          cache: pip
+          cache-dependency-path: |
+            requirements-tests.txt
+            stubs/**/@tests/requirements-stubtest.txt
+            stubs/**/METADATA.toml
+      - name: Install dependencies
+        run: pip install -r requirements-tests.txt
+      - name: Run stubtest
+        shell: bash
+        run: |
+          # This only runs stubtest on changed stubs, because it is much faster.
+          # Use the daily.yml workflow to run stubtest on all third party stubs.
+          function find_stubs {
+            git diff --name-only origin/${{ github.base_ref }} HEAD | \
+            egrep ^stubs/ | cut -d "/" -f 2 | sort -u | \
+            (while read stub; do [ -d "stubs/$stub" ] && echo "$stub" || true; done)
+          }
+          STUBS=$(find_stubs || echo '')
+
+          if [ -n "$STUBS" ]; then
+            echo "Testing $STUBS..."
+            PACKAGES=$(python tests/get_packages.py $STUBS)
+
+            if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then
+              if [ -n "$PACKAGES" ]; then
+                echo "Installing apt packages: $PACKAGES"
+                sudo apt update && sudo apt install -y $PACKAGES
+              fi
+              xvfb-run python tests/stubtest_third_party.py --specified-stubs-only $STUBS
+            fi
+
+            if [ "${{ matrix.os }}" = "macos-latest" ]; then
+              # Could install brew packages here if we run into stubs that need it
+              python tests/stubtest_third_party.py --specified-stubs-only $STUBS
+            fi
+
+            if [ "${{ matrix.os }}" = "windows-latest" ]; then
+              # Could install choco packages here if we run into stubs that need it
+              python tests/stubtest_third_party.py --specified-stubs-only $STUBS
+            fi
+          else
+            echo "Nothing to test"
+          fi

.github/workflows/tests.yml

@@ -5,7 +5,6 @@ on:
   push:
     branches:
       - main
-      - master
   pull_request:
     paths-ignore:
       - '**/*.md'
@@ -29,7 +28,9 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.x
+          python-version: "3.10"
+          cache: pip
+          cache-dependency-path: requirements-tests.txt
       - run: pip install -r requirements-tests.txt
       - run: ./tests/check_consistent.py
 
@@ -40,7 +41,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.x
+          python-version: "3.10"
       - run: ./tests/check_new_syntax.py
 
   flake8:
@@ -50,7 +51,9 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.x
+          python-version: "3.10"
+          cache: pip
+          cache-dependency-path: requirements-tests.txt
       - run: pip install -r requirements-tests.txt
       - run: flake8
 
@@ -62,6 +65,8 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+          cache: pip
+          cache-dependency-path: requirements-tests.txt
       - run: pip install -r requirements-tests.txt
       - run: ./tests/pytype_test.py --print-stderr
 
@@ -77,7 +82,9 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.x
+          python-version: "3.10"
+          cache: pip
+          cache-dependency-path: requirements-tests.txt
       - run: pip install -r requirements-tests.txt
       - run: python ./tests/mypy_test.py --platform=${{ matrix.platform }} --python-version=${{ matrix.python-version }}
 
@@ -89,6 +96,8 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+          cache: pip
+          cache-dependency-path: requirements-tests.txt
       - run: pip install -r requirements-tests.txt
       - run: python ./tests/regr_test.py --all
 
@@ -101,7 +110,7 @@ jobs:
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
       fail-fast: false
     env:
-      PYRIGHT_VERSION: 1.1.273 # Must match pyright_test.py.
+      PYRIGHT_VERSION: 1.1.278 # Must match pyright_test.py.
     steps:
       - uses: actions/checkout@v3
       - uses: jakebailey/pyright-action@v1
@@ -142,41 +151,10 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+          cache: pip
+          cache-dependency-path: stub_uploader/requirements.txt
       - name: Run tests
         run: |
           cd stub_uploader
           pip install -r requirements.txt
           python -m pytest tests
-
-  stubtest-third-party:
-    name: Check third party stubs with stubtest
-    runs-on: ubuntu-20.04
-    if: github.event_name == 'pull_request'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.9"
-      - name: Install dependencies
-        run: pip install -r requirements-tests.txt
-      - name: Run stubtest
-        run: |
-          STUBS=$(
-            git diff --name-only origin/${{ github.base_ref }} HEAD |
-            # Uncomment the following to (very slowly) run on all third party stubs:
-            # git ls-files |
-            egrep ^stubs/ | cut -d "/" -f 2 | sort -u | (while read stub; do [ -d stubs/$stub ] && echo $stub || true; done)
-          )
-          if test -n "$STUBS"; then
-            echo "Testing $STUBS..."
-            APT_PACKAGES=$(python tests/get_apt_packages.py $STUBS)
-            if test -n "$APT_PACKAGES"; then
-              echo "Installing apt packages: $APT_PACKAGES"
-              sudo apt update && sudo apt install -y $APT_PACKAGES
-            fi
-            xvfb-run python tests/stubtest_third_party.py $STUBS
-          else
-            echo "Nothing to test"
-          fi