Fix find, data-files and tool.distutils

Purpose: make the output of `ini2toml` valid according to
`validate-pyproject`.

Side effects:
- enforce inline_tables when possible
- allow nesting of CommentedList or CommentedKV

src/ini2toml/drivers/full_toml.py

@@ -45,6 +45,7 @@
 MAX_INLINE_TABLE_LEN = 60
 INLINE_TABLE_LONG_ELEM = 10
 MAX_INLINE_TABLE_LONG_ELEM = 5
+LONG = 120
 
 
 def convert(irepr: IntermediateRepr) -> str:
@@ -73,7 +74,7 @@ def _collapse_commented_list(obj: CommentedList, root=False) -> Array:
         if multiline:
             cast(list, out).append(Whitespace("\n" + 4 * " "))
         for value in values:
-            cast(list, out).append(value)
+            cast(list, out).append(collapse(value))
         if entry.has_comment():
             if multiline:
                 cast(list, out).append(_no_trail_comment(entry.comment))
@@ -95,7 +96,7 @@ def _collapse_commented_kv(obj: CommentedKV, root=False) -> Union[Table, InlineT
         k: Optional[str] = None  # if the for loop is empty, k = None
         for value in values:
             k, v = value
-            out[cast(str, k)] = v
+            out[cast(str, k)] = collapse(v)
         if not entry.has_comment():
             continue
         if not multiline:
@@ -115,12 +116,27 @@ def _collapse_irepr(obj: IntermediateRepr, root=False):
     if root:
         return _convert_irepr_to_toml(obj, document())
     rough_repr = repr(obj).replace(obj.__class__.__name__, "").strip()
-    out = table() if len(rough_repr) > 120 or "\n" in rough_repr else inline_table()
+    out = table() if len(rough_repr) > LONG or "\n" in rough_repr else inline_table()
     return _convert_irepr_to_toml(obj, cast(Union[Table, InlineTable], out))
 
 
+@collapse.register(dict)
+def _collapse_dict(obj: dict, root=False) -> Union[Table, InlineTable]:
+    if not obj:
+        return inline_table()
+    out = (
+        table()
+        if any(v and isinstance(v, (list, dict)) for v in obj.values())
+        or len(repr(obj)) > LONG  # simple heuristic
+        else inline_table()
+    )
+    for key, value in obj.items():
+        out.append(key, collapse(value))
+    return out
+
+
 @collapse.register(list)
-def _collapse_list_repr(obj: list, root=False) -> Union[AoT, Array]:
+def _collapse_list(obj: list, root=False) -> Union[AoT, Array]:
     is_aot, max_len, has_nl, num_elem = classify_list(obj)
     # Just some heuristics which kind of array we are going to use
     if is_aot:
@@ -150,14 +166,29 @@ def _convert_irepr_to_toml(irepr: IntermediateRepr, out: T) -> T:
             parent_key, *rest = key
             if not isinstance(parent_key, str):
                 raise InvalidTOMLKey(key)
+            if len(rest) == 1:
+                nested_key = rest[0]
+                collapsed_value = collapse(value)
+                collapsed_str = f"{nested_key} = {dumps(collapsed_value)}"
+                # Force inline table for the simplest cases
+                if (
+                    not isinstance(collapsed_value, (Table, AoT))
+                    and len(collapsed_str) < LONG
+                    and "\n" not in collapsed_str.strip()
+                ):
+                    child = inline_table()
+                    child[nested_key] = collapsed_value
+                    out[parent_key] = child
+                    continue
+            else:
+                nested_key = tuple(rest)
             p = out.setdefault(parent_key, {})
-            nested_key = rest[0] if len(rest) == 1 else tuple(rest)
             _convert_irepr_to_toml(IntermediateRepr({nested_key: value}), p)
-        elif isinstance(key, str):
+        elif isinstance(key, (int, str)):
             if isinstance(value, IntermediateRepr):
-                _convert_irepr_to_toml(value, out.setdefault(key, {}))
+                _convert_irepr_to_toml(value, out.setdefault(str(key), {}))
             else:
-                out[key] = collapse(value)
+                out[str(key)] = collapse(value)
     return out
 
 

src/ini2toml/drivers/plain_builtins.py

@@ -4,6 +4,7 @@
 
 This is **not a loss-less** process, since comments are not preserved.
 """
+from collections.abc import Mapping
 from functools import singledispatch
 from typing import Any
 
@@ -33,25 +34,25 @@ def _collapse_commented(obj: Commented) -> Any:
 
 @collapse.register(CommentedList)
 def _collapse_commented_list(obj: CommentedList) -> list:
-    return obj.as_list()
+    return [collapse(v) for v in obj.as_list()]
 
 
 @collapse.register(CommentedKV)
 def _collapse_commented_kv(obj: CommentedKV) -> dict:
-    return obj.as_dict()
+    return {k: collapse(v) for k, v in obj.as_dict().items()}
 
 
-@collapse.register(IntermediateRepr)
-def _collapse_irepr(obj: IntermediateRepr) -> dict:
+@collapse.register(Mapping)
+def _collapse_mapping(obj: Mapping) -> dict:
     return _convert_irepr_to_dict(obj, {})
 
 
 @collapse.register(list)
-def _collapse_list_repr(obj: list) -> list:
+def _collapse_list(obj: list) -> list:
     return [collapse(e) for e in obj]
 
 
-def _convert_irepr_to_dict(irepr: IntermediateRepr, out: dict) -> dict:
+def _convert_irepr_to_dict(irepr: Mapping, out: dict) -> dict:
     for key, value in irepr.items():
         if isinstance(key, HiddenKey):
             continue
@@ -61,10 +62,7 @@ def _convert_irepr_to_dict(irepr: IntermediateRepr, out: dict) -> dict:
                 raise InvalidTOMLKey(key)
             p = out.setdefault(parent_key, {})
             nested_key = rest[0] if len(rest) == 1 else tuple(rest)
-            _convert_irepr_to_dict(IntermediateRepr({nested_key: value}), p)
-        elif isinstance(key, str):
-            if isinstance(value, IntermediateRepr):
-                _convert_irepr_to_dict(value, out.setdefault(key, {}))
-            else:
-                out[key] = collapse(value)
+            _convert_irepr_to_dict({nested_key: value}, p)
+        elif isinstance(key, (int, str)):
+            out[str(key)] = collapse(value)
     return out

src/ini2toml/plugins/profile_independent_tasks.py

@@ -7,7 +7,7 @@
 
 NEWLINES = re.compile(r"\n+", re.M)
 TABLE_START = re.compile(r"^\[(.*)\]", re.M)
-EMPTY_TABLES = re.compile(r"^\[.*\]\n+\[(.*)\]", re.M)
+EMPTY_TABLES = re.compile(r"^\[(.*)\]\n+\[(\1\.(?:.*))\]", re.M)
 TRAILING_SPACES = re.compile(r"[ \t]+$", re.M)
 
 
@@ -40,4 +40,4 @@ def normalise_newlines(text: str) -> str:
 
 def remove_empty_table_headers(text: str) -> str:
     """Remove empty TOML table headers"""
-    return EMPTY_TABLES.sub(r"[\1]", text).strip()
+    return EMPTY_TABLES.sub(r"[\2]", text).strip()

src/ini2toml/plugins/setuptools_pep621.py

@@ -2,11 +2,12 @@
 import re
 from functools import partial, reduce
 from itertools import chain
-from typing import Dict, List, Mapping, Sequence, Tuple, Type, TypeVar, Union, cast
+from typing import Dict, List, Mapping, Sequence, Set, Tuple, Type, TypeVar, Union, cast
 
 from ..transformations import (
     apply,
     coerce_bool,
+    deprecated,
     kebab_case,
     remove_prefixes,
     split_comment,
@@ -15,9 +16,14 @@
 )
 from ..types import CommentKey, HiddenKey
 from ..types import IntermediateRepr as IR
-from ..types import Transformation, Translator
+from ..types import Transformation, Translator, WhitespaceKey
 from .best_effort import BestEffort
 
+try:
+    from setuptools._distutils import command as distutils_commands
+except ImportError:  # pragma: no cover
+    from distutils import command as distutils_commands
+
 R = TypeVar("R", bound=IR)
 
 RenameRules = Dict[Tuple[str, ...], Union[Tuple[Union[str, int], ...], None]]
@@ -29,26 +35,27 @@
 chain_iter = chain.from_iterable
 
 # Functions that split values from comments and parse those values
+split_directive = partial(split_kv_pairs, key_sep=":")
 split_list_comma = partial(split_list, sep=",", subsplit_dangling=False)
 split_list_semi = partial(split_list, sep=";", subsplit_dangling=False)
 split_hash_comment = partial(split_comment, comment_prefixes="#")  # avoid splitting `;`
 split_bool = partial(split_comment, coerce_fn=coerce_bool)
 split_kv_nocomments = partial(split_kv_pairs, comment_prefixes="")
-split_directive = partial(split_kv_pairs, key_sep=":")
+split_kv_of_lists = partial(split_kv_pairs, coerce_fn=split_list_comma)
 
 SECTION_SPLITTER = re.compile(r"\.|:")
-SETUPTOOLS_COMMAND_SECTIONS = (
+SETUPTOOLS_SECTIONS = ("metadata", "options")
+SKIP_CHILD_NORMALISATION = ("options.entry_points",)
+COMMAND_SECTIONS = (
+    "global",
     "alias",
-    "bdist",
-    "sdist",
-    "build",
     "install",
     "develop",
-    "dist_info",
-    "egg_info",
+    "sdist",
+    "bdist",
+    "bdist_wheel",
+    *getattr(distutils_commands, "__all__", []),
 )
-SETUPTOOLS_SECTIONS = ("metadata", "options", *SETUPTOOLS_COMMAND_SECTIONS)
-SKIP_CHILD_NORMALISATION = ("options.entry_points",)
 
 
 def activate(translator: Translator):
@@ -118,11 +125,15 @@ def processing_rules(self) -> ProcessingRules:
             # => ("metadata", "license_files",): in PEP621 should be a single file
             ("metadata", "keywords"): split_list_comma,
             # => ("metadata", "project-urls") => merge_and_rename_urls
-            ("metadata", "provides"): split_list_comma,
-            ("metadata", "requires"): split_list_comma,
-            ("metadata", "obsoletes"): split_list_comma,
             # ("metadata", "long-description-content-type") =>
             #     merge_and_rename_long_description_and_content_type
+            # ---- the following options are originally part of `metadata`,
+            #      but we move it to "options" because they are not part of PEP 621
+            ("options", "provides"): split_list_comma,
+            ("options", "requires"): split_list_comma,
+            ("options", "obsoletes"): split_list_comma,
+            ("options", "platforms"): split_list_comma,
+            # ----
             ("options", "zip-safe"): split_bool,
             ("options", "setup-requires"): split_list_semi,
             ("options", "install-requires"): split_list_semi,
@@ -135,7 +146,7 @@ def processing_rules(self) -> ProcessingRules:
             ("options", "package-dir"): split_kv_pairs,
             ("options", "namespace-packages"): split_list_comma,
             ("options", "py-modules"): split_list_comma,
-            ("options", "data-files"): split_kv_pairs,
+            ("options", "data-files"): deprecated("data-files", split_kv_of_lists),
             ("options.packages.find", "include"): split_list_comma,
             ("options.packages.find", "exclude"): split_list_comma,
             ("options.packages.find-namespace", "include"): split_list_comma,
@@ -263,7 +274,7 @@ def move_and_split_entrypoints(self, doc: R) -> R:
             scripts = split_kv_pairs(entrypoints.pop(key)).to_ir()
             new_key = key.replace("_", "-").replace("console-", "")
             doc.append(f"project:{new_key}", scripts)
-        if not entrypoints:
+        if not entrypoints or all(isinstance(k, WhitespaceKey) for k in entrypoints):
             doc.pop("project:entry-points")
         return doc
 
@@ -294,13 +305,15 @@ def remove_metadata_not_in_pep621(self, doc: R) -> R:
     def parse_setup_py_command_options(self, doc: R) -> R:
         # ---- distutils/setuptools command specifics outside of "options" ----
         sections = list(doc.keys())
-        extras = {
-            k: self._be.apply_best_effort_to_section(doc.pop(k))
-            for k in sections
-            for p in SETUPTOOLS_COMMAND_SECTIONS
-            if isinstance(k, str) and k.startswith(p) and k != "build-system"
-        }
-        doc["options"].update(extras)
+        commands = _distutils_commands()
+        for k in sections:
+            if isinstance(k, str) and k in commands:
+                section = self._be.apply_best_effort_to_section(doc[k])
+                for option in section:
+                    if isinstance(option, str):
+                        section.rename(option, self.normalise_key(option))
+                doc[k] = section
+                doc.rename(k, ("distutils", k))
         return doc
 
     def convert_directives(self, out: R) -> R:
@@ -384,12 +397,12 @@ def fix_packages(self, doc: R) -> R:
         prefix = next((p for p in prefixes if packages.startswith(f"{p}:")), None)
         if not prefix:
             return doc
+        kebab_prefix = prefix.replace("_", "-")
         if "options.packages.find" not in doc:
-            options["packages"] = split_kv_pairs(packages, key_sep=":")
+            options["packages"] = {kebab_prefix: {}}
             return doc
         options.pop("packages")
-        prefix = prefix.replace("_", "-")
-        doc.rename("options.packages.find", f"options.packages.{prefix}")
+        doc.rename("options.packages.find", f"options.packages.{kebab_prefix}")
         return doc
 
     def fix_setup_requires(self, doc: R) -> R:
@@ -492,7 +505,7 @@ def normalise_keys(self, cfg: R) -> R:
                 option_name = section.order[j]
                 if not isinstance(option_name, str):
                     continue
-                section.rename(option_name, kebab_case(option_name))
+                section.rename(option_name, self.normalise_key(option_name))
         # Normalise aliases
         metadata = cfg.get("metadata")
         if not metadata:
@@ -502,9 +515,23 @@ def normalise_keys(self, cfg: R) -> R:
                 metadata.rename(alias, cannonic)
         return cfg
 
+    def normalise_key(self, key: str) -> str:
+        """Normalise a single key for option"""
+        return kebab_case(key)
+
 
 # ---- Helpers ----
 
 
 def isdirective(value, valid=("file", "attr")) -> bool:
     return isinstance(value, str) and any(value.startswith(f"{p}:") for p in valid)
+
+
+def _distutils_commands() -> Set[str]:
+    try:
+        from . import iterate_entry_points
+
+        commands = [ep.name for ep in iterate_entry_points("distutils.commands")]
+    except Exception:
+        commands = []
+    return {*commands, *COMMAND_SECTIONS}