Skip to content

Commit

Permalink
Fix setup.py rendering. (#6439)
Browse files Browse the repository at this point in the history 
We now render setup.py from both python 2 & 3 such that the result
is ingestible by both python 2 & 3.

Fixes #6401
Fixes #6392
  • Loading branch information
@jsirois
jsirois committed Sep 6, 2018
1 parent 60f0d17 commit 775be32
Show file tree
Hide file tree
Showing 2 changed files with 83 additions and 69 deletions.
143 changes: 76 additions & 67 deletions src/python/pants/backend/python/tasks/setup_py.py
View file
Expand Up @@ -5,16 +5,17 @@
from __future__ import absolute_import, division, print_function, unicode_literals

import inspect
import io
import itertools
import os
import pprint
import shutil
import textwrap
from abc import abstractmethod
from builtins import map, object, str, zip
from builtins import bytes, map, object, str, zip
from collections import OrderedDict, defaultdict

from future.utils import PY2, PY3
from future.moves import collections
from future.utils import PY2
from pex.installer import InstallerBase, Packager
from pex.interpreter import PythonInterpreter
from pex.pex import PEX
Expand All @@ -40,7 +41,7 @@
from pants.util.memo import memoized_property
from pants.util.meta import AbstractClass
from pants.util.process_handler import subprocess
from pants.util.strutil import safe_shlex_split
from pants.util.strutil import ensure_binary, ensure_text, safe_shlex_split


SETUP_BOILERPLATE = """
Expand All @@ -49,12 +50,65 @@
from setuptools import setup
setup(**
{setup_dict}
)
setup(**{setup_dict})
"""


# Distutils does not support unicode strings in setup.py, so we must explicitly convert to binary
# strings as pants uses unicode_literals. A natural and prior technique was to use `pprint.pformat`,
# but that embeds u's in the string itself during conversion. For that reason we roll out own
# literal pretty-printer here.
#
# For more information, see http://bugs.python.org/issue13943
def distutils_repr(obj):
output = io.StringIO()
linesep = os.linesep

def _write(data):
output.write(ensure_text(data))

def _write_repr(o, indent=False, level=0):
pad = ' ' * 4 * level
if indent:
_write(pad)
level += 1

if isinstance(o, (bytes, str)):
# The py2 repr of str (unicode) is `u'...'` and we don't want the `u` prefix; likewise,
# the py3 repr of bytes is `b'...'` and we don't want the `b` prefix so we hand-roll a
# repr here.
if linesep in o:
_write('"""{}"""'.format(ensure_text(o.replace('"""', r'\"\"\"'))))
else:
_write("'{}'".format(ensure_text(o.replace("'", r"\'"))))
elif isinstance(o, collections.Mapping):
_write('{' + linesep)
for k, v in o.items():
_write_repr(k, indent=True, level=level)
_write(': ')
_write_repr(v, indent=False, level=level)
_write(',' + linesep)
_write(pad + '}')
elif isinstance(o, collections.Iterable):
if isinstance(o, collections.MutableSequence):
open_collection, close_collection = '[]'
elif isinstance(o, collections.Set):
open_collection, close_collection = '{}'
else:
open_collection, close_collection = '()'

_write(open_collection + linesep)
for i in o:
_write_repr(i, indent=True, level=level)
_write(',' + linesep)
_write(pad + close_collection)
else:
_write(repr(o)) # Numbers and bools.

_write_repr(obj)
return output.getvalue()


class SetupPyRunner(InstallerBase):
_EXTRAS = ('setuptools', 'wheel')

Expand Down Expand Up @@ -327,7 +381,7 @@ def declares_namespace_package(filename):
class SetupPy(Task):
"""Generate setup.py-based Python projects."""

SOURCE_ROOT = 'src' if PY3 else b'src'
SOURCE_ROOT = 'src'

PYTHON_DISTS_PRODUCT = 'python_dists'

Expand Down Expand Up @@ -411,9 +465,9 @@ def nsutil_pex(self):
pex_info = PexInfo.default(interpreter=interpreter)
with safe_concurrent_creation(chroot) as scratch:
builder = PEXBuilder(path=scratch, interpreter=interpreter, pex_info=pex_info, copy=True)
with temporary_file() as fp:
with temporary_file(binary_mode=False) as fp:
declares_namespace_package_code = inspect.getsource(declares_namespace_package)
fp.write(textwrap.dedent('''
fp.write(textwrap.dedent("""
import sys
Expand All @@ -424,7 +478,7 @@ def nsutil_pex(self):
for path in sys.argv[1:]:
if declares_namespace_package(path):
print(path)
''').strip().format(declares_namespace_package_code=declares_namespace_package_code))
""").strip().format(declares_namespace_package_code=declares_namespace_package_code))
fp.close()
builder.set_executable(filename=fp.name, env_filename='main.py')
builder.freeze()
Expand All @@ -447,7 +501,7 @@ def filter_namespace_packages(self, root_target, inits):
# and then writing it out after the process has finished like we do here.
def write(stream_name, data):
stream = workunit.output(stream_name)
stream.write(data)
stream.write(ensure_binary(data) if PY2 else ensure_text(data))
stream.flush()

write('stdout', stdout)
Expand All @@ -460,7 +514,7 @@ def write(stream_name, data):
exit_code=exit_code,
failed_targets=[root_target])

return stdout.splitlines()
return ensure_text(stdout).splitlines()

def find_packages(self, root_target, chroot):
"""Detect packages, namespace packages and resources from an existing chroot.
Expand Down Expand Up @@ -577,33 +631,20 @@ def write_setup(self, root_target, reduced_dependencies, chroot):
Must be run after writing the contents to the chroot.
"""
# NB: several explicit str conversions below force non-unicode strings in order to comply
# with setuptools expectations.
#
# Because we rely on pprint to generate the content, we must treat Py2 and Py3 differently.
# In Py2, unicode strings have the prefix 'u', and in Py3 byte strings have the prefix 'b'.
# The goal is to have no prefix displayed, so to do this we have to use bytes in Py2 and
# unicode in Py3.

setup_keywords = root_target.provides.setup_py_keywords.copy()

package_dir = {'' if PY3 else b'': self.SOURCE_ROOT}
package_dir = {'': self.SOURCE_ROOT}
packages, namespace_packages, resources = self.find_packages(root_target, chroot)

if namespace_packages:
setup_keywords['namespace_packages'] = list(sorted(namespace_packages))

if packages:
normalized_package_data = (
resources.items()
if PY3 else
((package.encode('utf-8'), [v.encode('utf-8') for v in rs])
for (package, rs) in resources.items())
)
setup_keywords.update(
package_dir=package_dir,
packages=list(sorted(packages)),
package_data=dict(normalized_package_data))
package_data=dict((str(package), list(map(str, rs)))
for (package, rs) in resources.items()))

setup_keywords['install_requires'] = list(self.install_requires(reduced_dependencies))

Expand All @@ -615,44 +656,12 @@ def write_setup(self, root_target, reduced_dependencies, chroot):
setup_keywords['entry_points']['console_scripts'].append(
'{} = {}'.format(binary_name, entry_point))

# From http://stackoverflow.com/a/13105359
def convert(input):
if isinstance(input, dict):
out = dict()
for key, value in input.items():
out[convert(key)] = convert(value)
return out
elif isinstance(input, list):
return [convert(element) for element in input]
elif PY2 and isinstance(input, str):
return input.encode('utf-8')
else:
return input

# Distutils does not support unicode strings in setup.py, so we must
# explicitly convert to binary strings as pants uses unicode_literals.
# Ideally we would write the output stream with an encoding, however,
# pprint.pformat embeds u's in the string itself during conversion.
# For that reason we convert each unicode string independently.
#
# jsirois@gill ~ $ python2
# Python 2.7.13 (default, Jul 21 2017, 03:24:34)
# [GCC 7.1.1 20170630] on linux2
# Type "help", "copyright", "credits" or "license" for more information.
# >>> import pprint
# >>> data = {u'entry_points': {u'console_scripts': [u'pants = pants.bin.pants_exe:main']}}
# >>> pprint.pformat(data, indent=4)
# "{ u'entry_points': { u'console_scripts': [ u'pants = pants.bin.pants_exe:main']}}"
# >>>
#
# For more information, see http://bugs.python.org/issue13943
chroot.write(self._setup_boilerplate().format(
setup_dict=pprint.pformat(convert(setup_keywords), indent=4),
setup_target=repr(root_target)
).encode('utf-8'), 'setup.py')

# make sure that setup.py is included
chroot.write('include *.py'.encode('utf8'), 'MANIFEST.in')
setup_py = self._setup_boilerplate().format(setup_dict=distutils_repr(setup_keywords),
setup_target=root_target.address.reference())
chroot.write(ensure_binary(setup_py), 'setup.py')

# Make sure that `setup.py` is included.
chroot.write('include *.py', 'MANIFEST.in', mode='w')

def create_setup_py(self, target, dist_dir):
chroot = Chroot(dist_dir, name=target.provides.name)
Expand Down
9 changes: 7 additions & 2 deletions tests/python/pants_test/backend/python/tasks/test_setup_py.py
View file
Expand Up @@ -125,9 +125,14 @@ def run(self):
with environment_as(PYTHONPATH=sdist_srcdir):
with self.run_execute(foo):
with open(os.path.join(sdist_srcdir, 'foo', 'commands', 'sys_path.txt'), 'r') as fp:
load_package = lambda: Package.from_href(fp.readline().strip())
# We don't care about the ordering of `wheel` and `setuptools` on the `sys.path`, just
# that they are 1st as a group.
extras = {p.name: p for p in (load_package(), load_package())}

def assert_extra(name, expected_version):
package = Package.from_href(fp.readline().strip())
self.assertEqual(name, package.name)
package = extras.get(name)
self.assertIsNotNone(package)
self.assertEqual(expected_version, package.raw_version)

# The 1st two elements of the sys.path should be our custom SetupPyRunner Installer's
Expand Down

0 comments on commit 775be32

Please sign in to comment.