Fix .pyi type stubs to show up in python_distribution (cherrypick…

… of #14033) (#14035)

@asherf discovered that even though we were including `.pyi` files in the chroot we give to `setuptools`, we need to set the value in `package_data` for setuptools to actually include the file. See https://blog.ian.stapletoncordas.co/2019/02/distributing-python-libraries-with-type-annotations.html.

Because of this issue, `pantsbuild.pants` was not including `native_engine.pyi` in the wheel.

[ci skip-rust]
[ci skip-build-wheels]

src/python/pants/backend/python/goals/setup_py.py

@@ -847,24 +847,32 @@ def find_packages(
     """
     # Find all packages implied by all the sources.
     packages: set[str] = set()
-    package_data: DefaultDict[str, list[str]] = defaultdict(list)
     for file_path in itertools.chain(python_files, resource_files):
         # Python 2: An __init__.py file denotes a package.
         # Python 3: Any directory containing python source files is a package.
         if (file_path.endswith(".py") and not py2) or os.path.basename(file_path) == "__init__.py":
             packages.add(os.path.dirname(file_path).replace(os.path.sep, "."))
 
     # Now find all package_data.
-    for resource_file in resource_files:
-        # Find the closest enclosing package, if any.  Resources will be loaded relative to that.
-        maybe_package: str = os.path.dirname(resource_file).replace(os.path.sep, ".")
+    package_data: DefaultDict[str, list[str]] = defaultdict(list)
+
+    def maybe_add_resource(fp: str) -> None:
+        # Find the closest enclosing package, if any. Resources will be loaded relative to that.
+        maybe_package: str = os.path.dirname(fp).replace(os.path.sep, ".")
         while maybe_package and maybe_package not in packages:
             maybe_package = maybe_package.rpartition(".")[0]
         # If resource is not in a package, ignore it. There's no principled way to load it anyway.
-        if maybe_package:
-            package_data[maybe_package].append(
-                os.path.relpath(resource_file, maybe_package.replace(".", os.path.sep))
-            )
+        if not maybe_package:
+            return
+        package_data[maybe_package].append(
+            os.path.relpath(fp, maybe_package.replace(".", os.path.sep))
+        )
+
+    for resource_file in resource_files:
+        maybe_add_resource(resource_file)
+    for py_file in python_files:
+        if py_file.endswith(".pyi"):
+            maybe_add_resource(py_file)
 
     # See which packages are pkg_resources-style namespace packages.
     # Note that implicit PEP 420 namespace packages and pkgutil-style namespace packages

src/python/pants/backend/python/goals/setup_py_test.py

@@ -407,7 +407,7 @@ def test_generate_chroot(chroot_rule_runner: RuleRunner) -> None:
             "plugin_demo": "hello world",
             "packages": ("foo", "foo.qux"),
             "namespace_packages": ("foo",),
-            "package_data": {"foo": ("resources/js/code.js",)},
+            "package_data": {"foo": ("resources/js/code.js",), "foo.qux": ("qux.pyi",)},
             "install_requires": ("baz==1.1.1",),
             "entry_points": {"console_scripts": ["foo_main = foo.qux.bin:main"]},
         },