Make all crates use workspace dependencies when shared

[cronokirby]

Whenever a crate is used more than once, this switches to a workspace dependency.

This makes upgrading versions much easier.

[cronokirby]

The python script I used to do this:

import os
import tomlkit


def cargo_config_files():
    for dir in ["crates/", "tools/"]:
        for (root, dirs, files) in os.walk(dir):
            if "Cargo.toml" in files:
                yield (root, os.path.join(root, "Cargo.toml"))


def read_config_file(path):
    with open(path, "r") as fp:
        return tomlkit.parse(fp.read())


NIL_VERSION = None


def version(obj):
    if isinstance(obj, str):
        return ("version", obj, True)
    obj = dict(obj.items())
    default_features = min(obj.get("default-features", True),
                           obj.get("default_features", True))
    if "version" in obj:
        return ("version", obj["version"], default_features)
    if "path" in obj:
        return ("path", default_features)
    if "git" in obj:
        return ("git", obj["git"], obj.get("rev"), default_features)


def merge(a, b):
    if a is None:
        return b
    if a[0] == "version" and b[0] == "version":
        return ("version", max(a[1], b[1]), min(a[2], b[2]))
    if a[0] == "path" and b[0] == "path":
        return ("path", min(a[1], b[1]))
    if a[0] == "git" and b[0] == "git":
        if a[1:3] == b[1:3]:
            return ("git", a[1], a[2], min(a[3], b[3]))
    raise RuntimeError(f"mismatched versions: {a[0]} and {b[0]}")


def format_version(path, version):
    out = tomlkit.inline_table()
    if not version[-1]:
        out.update({"default-features": False})
    if version[0] == "version":
        out.update({"version": version[1]})
        return out
    if version[0] == "path":
        out.update({"path": path})
        return out
    if version[0] == "git":
        if version[2] is None:
            out.update({"git": version[1]})
        else:
            out.update({"git": version[1], "rev": version[2]})
        return out
    raise RuntimeError(f"unexpected version {version}")


def apply(spike, obj):
    for key in spike[0][:-1]:
        if key not in obj:
            return
        obj = obj[key]
    obj[spike[0][-1]] = spike[1](obj[spike[0][-1]])


def workspace_true(x):
    out = tomlkit.inline_table()
    out.update({"workspace": True})
    return out


def replace_version_with_workspace(no_default_features, x):
    if isinstance(x, str):
        return workspace_true(x)
    out = tomlkit.inline_table()
    out.update({"workspace": True})
    for k, v in x.items():
        if k in ["version", "git", "path", "rev"]:
            continue
        if k == "default_features":
            k = "default-features"
        out.update({k: v})
    if no_default_features and ("default-features" not in out and "default_features" not in out):
        out.update({"default-features": True})
    return out


def append_object(dependencies, paths, count, dev_count):
    package = tomlkit.table()
    package.append("authors", ["Penumbra Labs <team@penumbra.zone>"])
    package.append("edition", "2021")
    package.append("version", "0.65.0-alpha.1")
    package.append("repository", "https://github.com/penumbra-zone/penumbra")
    package.append("homepage", "https://penumbra.zone")
    package.append("license", "MIT OR Apache-2.0")

    dep_table = tomlkit.table()
    for dep in sorted(dependencies.keys()):
        version = dependencies[dep]
        if count.get(dep, 0) + dev_count.get(dep, 0) <= 1:
            continue
        formatted = format_version(paths.get(dep), version)
        dep_table.append(dep, formatted)

    obj = tomlkit.document()
    obj.append("workspace.package", package)
    obj.append("workspace.dependencies", dep_table)
    return obj


def main():
    dependencies = dict()
    count = dict()
    dev_count = dict()
    paths = dict()
    for (crate, x) in cargo_config_files():
        config = read_config_file(x)
        paths[config["package"]["name"]] = crate
        for key in ["dependencies", "dev-dependencies"]:
            if key not in config:
                continue
            for k, v in config[key].items():
                dependencies[k] = merge(
                    dependencies.get(k, NIL_VERSION), version(v))
                if key == "dependencies":
                    count[k] = count.get(k, 0) + 1
                elif key == "dev-dependencies":
                    dev_count[k] = dev_count.get(k, 0) + 1
    no_default_features = {x for x, v in dependencies.items() if not v[-1]}
    # Create an append object
    obj = append_object(dependencies, paths, count, dev_count)
    # Append it to the workspace file
    with open("Cargo.toml", "a") as fp:
        fp.write(tomlkit.dumps(obj))
    spikes = {
        "package": {
            "authors": workspace_true(None),
            "edition": workspace_true(None),
            "version": workspace_true(None),
            "repository": workspace_true(None),
            "homepage": workspace_true(None),
            "license": workspace_true(None)
        },
        "dependencies": dict(),
        "dev-dependencies": dict()
    }
    for k in dependencies.keys():
        if count.get(k, 0) + dev_count.get(k, 0) <= 1:
            continue
        spikes["dependencies"][k] = replace_version_with_workspace
        spikes["dev-dependencies"][k] = replace_version_with_workspace
    # Apply the spikes to each config file
    for (_, x) in cargo_config_files():
        config = read_config_file(x)
        package = config.pop("package")
        dependencies = config.pop("dependencies", None)
        dev_dependencies = config.pop("dev-dependencies", None)

        new_package = tomlkit.table()
        for k, v in package.items():
            value = spikes["package"].get(k, v)
            new_package.append(k, value)

        new_dependencies = None
        if dependencies is not None:
            new_dependencies = tomlkit.table()
            for k, v in dependencies.items():
                if k in spikes["dependencies"]:
                    value = tomlkit.inline_table()
                    value.update(replace_version_with_workspace(
                        k in no_default_features, v))
                    new_dependencies.append(k, value)
                else:
                    new_dependencies.append(k, v)

        new_dev_dependencies = None
        if dev_dependencies is not None:
            new_dev_dependencies = tomlkit.table()
            for k, v in dev_dependencies.items():
                if k in spikes["dev-dependencies"]:
                    value = tomlkit.inline_table()
                    value.update(replace_version_with_workspace(
                        k in no_default_features, v))
                    new_dev_dependencies.append(k, value)
                else:
                    new_dev_dependencies.append(k, v)

        new_config = tomlkit.document()
        new_config.append("package", new_package)
        for k, v in config.items():
            new_config.append(k, v)
        if new_dependencies is not None:
            new_config.append("dependencies", new_dependencies)
        if new_dev_dependencies is not None:
            new_config.append("dev-dependencies", new_dev_dependencies)
        with open(x, "w") as fp:
            fp.write(tomlkit.dumps(new_config))


if __name__ == "__main__":
    main()

[hdevalence]

Going to merge this optimistically since it seems like it will be a source of conflicts otherwise.