setup_development.py

#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

"""
Setup mozbase packages for development.

Packages may be specified as command line arguments.
If no arguments are given, install all packages.

See https://wiki.mozilla.org/Auto-tools/Projects/Mozbase
"""

import os
import subprocess
import sys
from optparse import OptionParser
from subprocess import PIPE

try:
    from subprocess import check_call as call
except ImportError:
    from subprocess import call


# directory containing this file
here = os.path.dirname(os.path.abspath(__file__))

# all python packages
mozbase_packages = [
    i for i in os.listdir(here) if os.path.exists(os.path.join(here, i, "setup.py"))
]

# testing: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Tests
test_packages = ["mock"]

# documentation: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Documentation
extra_packages = ["sphinx"]


def cycle_check(order, dependencies):
    """ensure no cyclic dependencies"""
    order_dict = dict([(j, i) for i, j in enumerate(order)])
    for package, deps in dependencies.items():
        index = order_dict[package]
        for d in deps:
            assert index > order_dict[d], "Cyclic dependencies detected"


def info(directory):
    "get the package setup.py information"

    assert os.path.exists(os.path.join(directory, "setup.py"))

    # setup the egg info
    try:
        call([sys.executable, "setup.py", "egg_info"], cwd=directory, stdout=PIPE)
    except subprocess.CalledProcessError:
        print("Error running setup.py in %s" % directory)
        raise

    # get the .egg-info directory
    egg_info = [entry for entry in os.listdir(directory) if entry.endswith(".egg-info")]
    assert len(egg_info) == 1, "Expected one .egg-info directory in %s, got: %s" % (
        directory,
        egg_info,
    )
    egg_info = os.path.join(directory, egg_info[0])
    assert os.path.isdir(egg_info), "%s is not a directory" % egg_info

    # read the package information
    pkg_info = os.path.join(egg_info, "PKG-INFO")
    info_dict = {}
    for line in open(pkg_info).readlines():
        if not line or line[0].isspace():
            continue  # XXX neglects description
        assert ":" in line
        key, value = [i.strip() for i in line.split(":", 1)]
        info_dict[key] = value

    return info_dict


def get_dependencies(directory):
    "returns the package name and dependencies given a package directory"

    # get the package metadata
    info_dict = info(directory)

    # get the .egg-info directory
    egg_info = [
        entry for entry in os.listdir(directory) if entry.endswith(".egg-info")
    ][0]

    # read the dependencies
    requires = os.path.join(directory, egg_info, "requires.txt")
    dependencies = []
    if os.path.exists(requires):
        for line in open(requires):
            line = line.strip()
            # in requires.txt file, a dependency is a non empty line
            # Also lines like [device] are sections to mark optional
            # dependencies, we don't want those sections.
            if line and not (line.startswith("[") and line.endswith("]")):
                dependencies.append(line)

    # return the information
    return info_dict["Name"], dependencies


def dependency_info(dep):
    "return dictionary of dependency information from a dependency string"
    retval = dict(Name=None, Type=None, Version=None)
    for joiner in ("==", "<=", ">="):
        if joiner in dep:
            retval["Type"] = joiner
            name, version = [i.strip() for i in dep.split(joiner, 1)]
            retval["Name"] = name
            retval["Version"] = version
            break
    else:
        retval["Name"] = dep.strip()
    return retval


def unroll_dependencies(dependencies):
    """
    unroll a set of dependencies to a flat list

    dependencies = {'packageA': set(['packageB', 'packageC', 'packageF']),
                    'packageB': set(['packageC', 'packageD', 'packageE', 'packageG']),
                    'packageC': set(['packageE']),
                    'packageE': set(['packageF', 'packageG']),
                    'packageF': set(['packageG']),
                    'packageX': set(['packageA', 'packageG'])}
    """

    order = []

    # flatten all
    packages = set(dependencies.keys())
    for deps in dependencies.values():
        packages.update(deps)

    while len(order) != len(packages):
        for package in packages.difference(order):
            if set(dependencies.get(package, set())).issubset(order):
                order.append(package)
                break
        else:
            raise AssertionError("Cyclic dependencies detected")

    cycle_check(order, dependencies)  # sanity check

    return order


def main(args=sys.argv[1:]):
    # parse command line options
    usage = "%prog [options] [package] [package] [...]"
    parser = OptionParser(usage=usage, description=__doc__)
    parser.add_option(
        "-d",
        "--dependencies",
        dest="list_dependencies",
        action="store_true",
        default=False,
        help="list dependencies for the packages",
    )
    parser.add_option(
        "--list", action="store_true", default=False, help="list what will be installed"
    )
    parser.add_option(
        "--extra",
        "--install-extra-packages",
        action="store_true",
        default=False,
        help="installs extra supporting packages as well as core mozbase ones",
    )
    options, packages = parser.parse_args(args)

    if not packages:
        # install all packages
        packages = sorted(mozbase_packages)

    # ensure specified packages are in the list
    assert set(packages).issubset(
        mozbase_packages
    ), "Packages should be in %s (You gave: %s)" % (mozbase_packages, packages)

    if options.list_dependencies:
        # list the package dependencies
        for package in packages:
            print("%s: %s" % get_dependencies(os.path.join(here, package)))
        parser.exit()

    # gather dependencies
    # TODO: version conflict checking
    deps = {}
    alldeps = {}
    mapping = {}  # mapping from subdir name to package name
    # core dependencies
    for package in packages:
        key, value = get_dependencies(os.path.join(here, package))
        deps[key] = [dependency_info(dep)["Name"] for dep in value]
        mapping[package] = key

        # keep track of all dependencies for non-mozbase packages
        for dep in value:
            alldeps[dependency_info(dep)["Name"]] = "".join(dep.split())

    # indirect dependencies
    flag = True
    while flag:
        flag = False
        for value in deps.values():
            for dep in value:
                if dep in mozbase_packages and dep not in deps:
                    key, value = get_dependencies(os.path.join(here, dep))
                    deps[key] = [dep for dep in value]

                    for dep in value:
                        alldeps[dep] = "".join(dep.split())
                    mapping[package] = key
                    flag = True
                    break
            if flag:
                break

    # get the remaining names for the mapping
    for package in mozbase_packages:
        if package in mapping:
            continue
        key, value = get_dependencies(os.path.join(here, package))
        mapping[package] = key

    # unroll dependencies
    unrolled = unroll_dependencies(deps)

    # make a reverse mapping: package name -> subdirectory
    reverse_mapping = dict([(j, i) for i, j in mapping.items()])

    # we only care about dependencies in mozbase
    unrolled = [package for package in unrolled if package in reverse_mapping]

    if options.list:
        # list what will be installed
        for package in unrolled:
            print(package)
        parser.exit()

    # set up the packages for development
    for package in unrolled:
        call(
            [sys.executable, "setup.py", "develop", "--no-deps"],
            cwd=os.path.join(here, reverse_mapping[package]),
        )

    # add the directory of sys.executable to path to aid the correct
    # `easy_install` getting called
    # https://bugzilla.mozilla.org/show_bug.cgi?id=893878
    os.environ["PATH"] = "%s%s%s" % (
        os.path.dirname(os.path.abspath(sys.executable)),
        os.path.pathsep,
        os.environ.get("PATH", "").strip(os.path.pathsep),
    )

    # install non-mozbase dependencies
    # these need to be installed separately and the --no-deps flag
    # subsequently used due to a bug in setuptools; see
    # https://bugzilla.mozilla.org/show_bug.cgi?id=759836
    pypi_deps = dict([(i, j) for i, j in alldeps.items() if i not in unrolled])
    for package, version in pypi_deps.items():
        # easy_install should be available since we rely on setuptools
        call(["easy_install", version])

    # install packages required for unit testing
    for package in test_packages:
        call(["easy_install", package])

    # install extra non-mozbase packages if desired
    if options.extra:
        for package in extra_packages:
            call(["easy_install", package])


if __name__ == "__main__":
    main()