Automatically add files referenced by configuration to sdist

changelog.d/3779.change.rst

@@ -0,0 +1,4 @@
+Files referenced by ``file:`` in ``setup.cfg`` and by ``project.readme.file``,
+``project.license.file`` or ``tool.setuptools.dynamic.*.file`` in
+``pyproject.toml`` are now automatically included in the generated sdists.
+

docs/userguide/declarative_config.rst

@@ -169,11 +169,16 @@ Special directives:
       The ``file:`` directive is sandboxed and won't reach anything outside the
       project directory (i.e. the directory containing ``setup.cfg``/``pyproject.toml``).
 
-  .. attention::
-      When using the ``file:`` directive, please make sure that all necessary
-      files are included in the ``sdist``. You can do that via ``MANIFEST.in``
-      or using plugins such as ``setuptools-scm``.
-      Please have a look on :doc:`/userguide/miscellaneous` for more information.
+  .. note::
+      If you are using an old version of ``setuptools``, you might need to ensure
+      that all files referenced by the ``file:`` directive are included in the ``sdist``
+      (you can do that via ``MANIFEST.in`` or using plugins such as ``setuptools-scm``,
+      please have a look on :doc:`/userguide/miscellaneous` for more information).
+
+      .. TODO add versionchanged with specific version when the behavior changed
+
+      Newer versions of ``setuptools`` will automatically add these files to
+      the ``sdist``.
 
 
 Metadata

docs/userguide/pyproject_config.rst

@@ -220,11 +220,16 @@ however please keep in mind that all non-comment lines must conform with :pep:`5
 (``pip``-specify syntaxes, e.g. ``-c/-r/-e`` flags, are not supported).
 
 
-.. attention::
-   When using the ``file`` directive, please make sure that all necessary
-   files are included in the ``sdist``. You can do that via ``MANIFEST.in``
-   or using plugins such as ``setuptools-scm``.
-   Please have a look on :doc:`/userguide/miscellaneous` for more information.
+.. note::
+   If you are using an old version of ``setuptools``, you might need to ensure
+   that all files referenced by the ``file`` directive are included in the ``sdist``
+   (you can do that via ``MANIFEST.in`` or using plugins such as ``setuptools-scm``,
+   please have a look on :doc:`/userguide/miscellaneous` for more information).
+
+   .. TODO add versionchanged with specific version when the behavior changed
+
+   Newer versions of ``setuptools`` will automatically add these files to
+   the ``sdist``.
 
 ----
 

setuptools/command/egg_info.py

@@ -565,6 +565,7 @@ def run(self):
         if os.path.exists(self.template):
             self.read_template()
         self.add_license_files()
+        self._add_referenced_files()
         self.prune_file_list()
         self.filelist.sort()
         self.filelist.remove_duplicates()
@@ -619,9 +620,16 @@ def add_license_files(self):
         license_files = self.distribution.metadata.license_files or []
         for lf in license_files:
             log.info("adding license file '%s'", lf)
-            pass
         self.filelist.extend(license_files)
 
+    def _add_referenced_files(self):
+        """Add files referenced by the config (e.g. `file:` directive) to filelist"""
+        referenced = getattr(self.distribution, '_referenced_files', [])
+        # ^-- fallback if dist comes from distutils or is a custom class
+        for rf in referenced:
+            log.debug("adding file referenced by config '%s'", rf)
+        self.filelist.extend(referenced)
+
     def prune_file_list(self):
         build = self.get_finalized_command('build')
         base_dir = self.distribution.get_fullname()

setuptools/config/_apply_pyprojecttoml.py

@@ -16,7 +16,7 @@
 from itertools import chain
 from types import MappingProxyType
 from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple,
-                    Type, Union)
+                    Type, Union, cast)
 
 from setuptools._deprecation_warning import SetuptoolsDeprecationWarning
 
@@ -142,22 +142,29 @@ def _long_description(dist: "Distribution", val: _DictOrStr, root_dir: _Path):
     from setuptools.config import expand
 
     if isinstance(val, str):
-        text = expand.read_files(val, root_dir)
+        file: Union[str, list] = val
+        text = expand.read_files(file, root_dir)
         ctype = _guess_content_type(val)
     else:
-        text = val.get("text") or expand.read_files(val.get("file", []), root_dir)
+        file = val.get("file") or []
+        text = val.get("text") or expand.read_files(file, root_dir)
         ctype = val["content-type"]
 
     _set_config(dist, "long_description", text)
+
     if ctype:
         _set_config(dist, "long_description_content_type", ctype)
 
+    if file:
+        dist._referenced_files.add(cast(str, file))
+
 
 def _license(dist: "Distribution", val: dict, root_dir: _Path):
     from setuptools.config import expand
 
     if "file" in val:
         _set_config(dist, "license", expand.read_files([val["file"]], root_dir))
+        dist._referenced_files.add(val["file"])
     else:
         _set_config(dist, "license", val["text"])
 

setuptools/config/pyprojecttoml.py

@@ -8,7 +8,7 @@
 import warnings
 from contextlib import contextmanager
 from functools import partial
-from typing import TYPE_CHECKING, Callable, Dict, Optional, Mapping, Union
+from typing import TYPE_CHECKING, Callable, Dict, Optional, Mapping, Set, Union
 
 from setuptools.errors import FileError, OptionError
 
@@ -84,8 +84,8 @@ def read_configuration(
 
     :param Distribution|None: Distribution object to which the configuration refers.
         If not given a dummy object will be created and discarded after the
-        configuration is read. This is used for auto-discovery of packages in the case
-        a dynamic configuration (e.g. ``attr`` or ``cmdclass``) is expanded.
+        configuration is read. This is used for auto-discovery of packages and in the
+        case a dynamic configuration (e.g. ``attr`` or ``cmdclass``) is expanded.
         When ``expand=False`` this object is simply ignored.
 
     :rtype: dict
@@ -211,6 +211,7 @@ def __init__(
         self.dynamic_cfg = self.setuptools_cfg.get("dynamic", {})
         self.ignore_option_errors = ignore_option_errors
         self._dist = dist
+        self._referenced_files: Set[str] = set()
 
     def _ensure_dist(self) -> "Distribution":
         from setuptools.dist import Distribution
@@ -241,6 +242,7 @@ def expand(self):
             self._expand_cmdclass(package_dir)
             self._expand_all_dynamic(dist, package_dir)
 
+        dist._referenced_files.update(self._referenced_files)
         return self.config
 
     def _expand_packages(self):
@@ -310,6 +312,7 @@ def _expand_directive(
         with _ignore_errors(self.ignore_option_errors):
             root_dir = self.root_dir
             if "file" in directive:
+                self._referenced_files.update(directive["file"])
                 return _expand.read_files(directive["file"], root_dir)
             if "attr" in directive:
                 return _expand.read_attr(directive["attr"], package_dir, root_dir)

setuptools/config/setupcfg.py

@@ -12,7 +12,7 @@
 from functools import partial
 from functools import wraps
 from typing import (TYPE_CHECKING, Callable, Any, Dict, Generic, Iterable, List,
-                    Optional, Tuple, TypeVar, Union)
+                    Optional, Set, Tuple, TypeVar, Union)
 
 from distutils.errors import DistutilsOptionError, DistutilsFileError
 from setuptools.extern.packaging.requirements import Requirement, InvalidRequirement
@@ -172,6 +172,9 @@ def parse_configuration(
             distribution.src_root,
         )
         meta.parse()
+        distribution._referenced_files.update(
+            options._referenced_files, meta._referenced_files
+        )
 
     return meta, options
 
@@ -247,6 +250,10 @@ def __init__(
         self.sections = sections
         self.set_options: List[str] = []
         self.ensure_discovered = ensure_discovered
+        self._referenced_files: Set[str] = set()
+        """After parsing configurations, this property will enumerate
+        all files referenced by the "file:" directive. Private API for setuptools only.
+        """
 
     @property
     def parsers(self):
@@ -365,8 +372,7 @@ def parser(value):
 
         return parser
 
-    @classmethod
-    def _parse_file(cls, value, root_dir: _Path):
+    def _parse_file(self, value, root_dir: _Path):
         """Represents value as a string, allowing including text
         from nearest files using `file:` directive.
 
@@ -388,7 +394,8 @@ def _parse_file(cls, value, root_dir: _Path):
             return value
 
         spec = value[len(include_directive) :]
-        filepaths = (path.strip() for path in spec.split(','))
+        filepaths = [path.strip() for path in spec.split(',')]
+        self._referenced_files.update(filepaths)
         return expand.read_files(filepaths, root_dir)
 
     def _parse_attr(self, value, package_dir, root_dir: _Path):

setuptools/dist.py

@@ -17,7 +17,7 @@
 from glob import iglob
 import itertools
 import textwrap
-from typing import List, Optional, TYPE_CHECKING
+from typing import List, Optional, Set, TYPE_CHECKING
 from pathlib import Path
 
 from collections import defaultdict
@@ -481,6 +481,11 @@ def __init__(self, attrs=None):
             },
         )
 
+        # Private API (setuptools-use only, not restricted to Distribution)
+        # Stores files that are referenced by the configuration and need to be in the
+        # sdist (e.g. `version = file: VERSION.txt`)
+        self._referenced_files: Set[str] = set()
+
         # Save the original dependencies before they are processed into the egg format
         self._orig_extras_require = {}
         self._orig_install_requires = []

setuptools/tests/test_sdist.py

@@ -498,6 +498,48 @@ def test_sdist_with_latin1_encoded_filename(self):
             filename = filename.decode('latin-1')
             filename not in cmd.filelist.files
 
+    _EXAMPLE_DIRECTIVES = {
+        "setup.cfg - long_description and version": """
+            [metadata]
+            name = testing
+            version = file: VERSION.txt
+            license_files = DOWHATYOUWANT
+            long_description = file: README.rst, USAGE.rst
+            """,
+        "pyproject.toml - static readme/license files and dynamic version": """
+            [project]
+            name = "testing"
+            readme = "USAGE.rst"
+            license = {file = "DOWHATYOUWANT"}
+            dynamic = ["version"]
+            [tool.setuptools.dynamic]
+            version = {file = ["VERSION.txt"]}
+            """
+    }
+
+    @pytest.mark.parametrize("config", _EXAMPLE_DIRECTIVES.keys())
+    def test_add_files_referenced_by_config_directives(self, tmp_path, config):
+        config_file, _, _ = config.partition(" - ")
+        config_text = self._EXAMPLE_DIRECTIVES[config]
+        (tmp_path / 'VERSION.txt').write_text("0.42", encoding="utf-8")
+        (tmp_path / 'README.rst').write_text("hello world!", encoding="utf-8")
+        (tmp_path / 'USAGE.rst').write_text("hello world!", encoding="utf-8")
+        (tmp_path / 'DOWHATYOUWANT').write_text("hello world!", encoding="utf-8")
+        (tmp_path / config_file).write_text(config_text, encoding="utf-8")
+
+        dist = Distribution({"packages": []})
+        dist.script_name = 'setup.py'
+        dist.parse_config_files()
+
+        cmd = sdist(dist)
+        cmd.ensure_finalized()
+        with quiet():
+            cmd.run()
+
+        assert 'VERSION.txt' in cmd.filelist.files
+        assert 'USAGE.rst' in cmd.filelist.files
+        assert 'DOWHATYOUWANT' in cmd.filelist.files
+
     def test_pyproject_toml_in_sdist(self, tmpdir):
         """
         Check if pyproject.toml is included in source distribution if present