Remove -s option and allow for PEXing directories directly.

This is a backwards incompatible command line change with pre-1.x as it
removes the -s flag.

CHANGES.rst

@@ -2,6 +2,15 @@
 CHANGES
 =======
 
+----------
+1.0.0.dev2
+----------
+
+* Now supports extras for static URLs and installable directories.
+
+* BREAKING CHANGE: Removes the ``-s`` option in favor of specifying directories directly as
+  arguments to the pex command line.
+
 ----------
 1.0.0.dev1
 ----------

pex/bin/pex.py

@@ -20,15 +20,15 @@
 from pex.crawler import Crawler
 from pex.fetcher import Fetcher, PyPIFetcher
 from pex.http import Context
-from pex.installer import EggInstaller, InstallerBase, Packager
+from pex.installer import EggInstaller
 from pex.interpreter import PythonInterpreter
 from pex.iterator import Iterator
 from pex.package import EggPackage, SourcePackage
 from pex.pex import PEX
 from pex.pex_builder import PEXBuilder
 from pex.platforms import Platform
 from pex.requirements import requirements_from_file
-from pex.resolvable import Resolvable, ResolvablePackage
+from pex.resolvable import Resolvable
 from pex.resolver import CachingResolver, Resolver
 from pex.resolver_options import ResolverOptionsBuilder
 from pex.tracer import TRACER, TraceLogger
@@ -300,15 +300,6 @@ def configure_clp():
       help='Add requirements from the given requirements file.  This option can be used multiple '
            'times.')
 
-  parser.add_option(
-      '-s', '--source-dir',
-      dest='source_dirs',
-      metavar='DIR',
-      default=[],
-      action='append',
-      help='Source to be packaged; This <DIR> should be a pip-installable project '
-           'with a setup.py.')
-
   parser.add_option(
       '-v',
       dest='verbosity',
@@ -432,15 +423,6 @@ def build_pex(args, options, resolver_option_builder):
   for requirements_txt in options.requirement_files:
     resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder))
 
-  if options.source_dirs:
-    for source_dir in options.source_dirs:
-      try:
-        sdist = Packager(source_dir, interpreter=interpreter).sdist()
-      except InstallerBase.Error:
-        die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL)
-
-      resolvables.append(ResolvablePackage.from_string(sdist, resolver_option_builder))
-
   resolver_kwargs = dict(interpreter=interpreter, platform=options.platform)
 
   if options.cache_dir:

pex/iterator.py

@@ -22,7 +22,7 @@ class Iterator(IteratorInterface):
 
   def __init__(self, fetchers=None, crawler=None, follow_links=False):
     self._crawler = crawler or Crawler()
-    self._fetchers = fetchers or [PyPIFetcher()]
+    self._fetchers = fetchers if fetchers is not None else [PyPIFetcher()]
     self.__follow_links = follow_links
 
   def _iter_requirement_urls(self, req):

pex/resolvable.py

@@ -1,17 +1,35 @@
 # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+import os
+import re
 from abc import abstractmethod, abstractproperty
 
-from pkg_resources import Requirement
+from pkg_resources import Requirement, safe_extra
 
 from .base import maybe_requirement, requirement_is_exact
 from .compatibility import string as compatibility_string
 from .compatibility import AbstractClass
+from .installer import InstallerBase, Packager
 from .package import Package
 from .resolver_options import ResolverOptionsBuilder, ResolverOptionsInterface
 
 
+# Extract extras as specified per "declaring extras":
+# https://pythonhosted.org/setuptools/setuptools.html
+_EXTRAS_PATTERN = re.compile(r'(?P<main>.*)\[(?P<extras>.*)\]$')
+
+
+def strip_extras(resolvable_string):
+  match = _EXTRAS_PATTERN.match(resolvable_string)
+  if match:
+    resolvable_string, extras = match.groupdict()['main'], match.groupdict()['extras']
+    extras = [safe_extra(extra.strip()) for extra in extras.split(',')]
+  else:
+    extras = []
+  return resolvable_string, extras
+
+
 class Resolvable(AbstractClass):
   """An entity that can be resolved into a package."""
 
@@ -136,13 +154,15 @@ class ResolvablePackage(Resolvable):
   # TODO(wickman) Implement extras parsing for ResolvablePackage
   @classmethod
   def from_string(cls, requirement_string, options_builder):
+    requirement_string, extras = strip_extras(requirement_string)
     package = Package.from_href(requirement_string)
     if package is None:
       raise cls.InvalidRequirement('Requirement string does not appear to be a package.')
-    return cls(package, options_builder.build(package.name))
+    return cls(package, options_builder.build(package.name), extras=extras)
 
-  def __init__(self, package, options):
+  def __init__(self, package, options, extras=None):
     self.package = package
+    self._extras = extras
     super(ResolvablePackage, self).__init__(options)
 
   def compatible(self, iterator):
@@ -159,9 +179,8 @@ def name(self):
   def exact(self):
     return True
 
-  # TODO(wickman) Implement extras parsing for ResolvablePackages
   def extras(self, interpreter=None):
-    return []
+    return self._extras
 
   def __eq__(self, other):
     return isinstance(other, ResolvablePackage) and self.package == other.package
@@ -219,6 +238,35 @@ def __str__(self):
     return str(self.requirement)
 
 
+class ResolvableDirectory(ResolvablePackage):
+  """A source directory (with setup.py) resolvable."""
+
+  @classmethod
+  def is_installable(cls, requirement_string):
+    if not os.path.isdir(requirement_string):
+      return False
+    return os.path.isfile(os.path.join(requirement_string, 'setup.py'))
+
+  @classmethod
+  def from_string(cls, requirement_string, options_builder):
+    requirement_string, extras = strip_extras(requirement_string)
+    if cls.is_installable(requirement_string):
+      try:
+        # TODO(wickman) This is one case where interpreter is necessary to be fully correct.  This
+        # may indicate that packages() should take interpreter like extras does.  Once we have
+        # metadata in setup.cfg or whatever, then we can get the interpreter out of the equation.
+        sdist = Packager(requirement_string).sdist()
+      except InstallerBase.Error:
+        raise cls.InvalidRequirement('Could not create source distribution for %s' %
+            requirement_string)
+      package = Package.from_href(sdist)
+      return ResolvablePackage(package, options_builder.build(package.name), extras=extras)
+    else:
+      raise cls.InvalidRequirement('%s does not appear to be an installable directory.'
+          % requirement_string)
+
+
+Resolvable.register(ResolvableDirectory)
 Resolvable.register(ResolvableRepository)
 Resolvable.register(ResolvablePackage)
 Resolvable.register(ResolvableRequirement)

pex/testing.py

@@ -92,6 +92,13 @@ def make_installer(name='my_project', installer_impl=EggInstaller, zip_safe=True
     yield installer_impl(td)
 
 
+@contextlib.contextmanager
+def make_source_dir(name='my_project'):
+  interp = {'project_name': name, 'zip_safe': True}
+  with temporary_content(PROJECT_CONTENT, interp=interp) as td:
+    yield td
+
+
 def make_sdist(name='my_project', zip_safe=True):
   with make_installer(name=name, installer_impl=Packager, zip_safe=zip_safe) as packager:
     return packager.sdist()

pex/version.py

@@ -1,7 +1,7 @@
 # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-__version__ = '1.0.0.dev1'
+__version__ = '1.0.0.dev2'
 
 SETUPTOOLS_REQUIREMENT = 'setuptools>=2.2,<16'
 WHEEL_REQUIREMENT = 'wheel>=0.24.0,<0.25.0'

scripts/coverage.sh

@@ -4,4 +4,4 @@ coverage run -p -m py.test tests
 coverage run -p -m pex.bin.pex -v --help >&/dev/null
 coverage run -p -m pex.bin.pex -v -- scripts/do_nothing.py
 coverage run -p -m pex.bin.pex -v requests -- scripts/do_nothing.py
-coverage run -p -m pex.bin.pex -v -s . setuptools -- scripts/do_nothing.py
+coverage run -p -m pex.bin.pex -v . setuptools -- scripts/do_nothing.py

tests/test_resolvable.py

@@ -8,12 +8,14 @@
 from pex.package import Package, SourcePackage
 from pex.resolvable import (
     Resolvable,
+    ResolvableDirectory,
     ResolvablePackage,
     ResolvableRepository,
     ResolvableRequirement,
     resolvables_from_iterable
 )
 from pex.resolver_options import ResolverOptionsBuilder
+from pex.testing import make_source_dir
 
 try:
   from unittest import mock
@@ -22,9 +24,10 @@
 
 
 def test_resolvable_package():
+  builder = ResolverOptionsBuilder()
   source_name = 'foo-2.3.4.tar.gz'
   pkg = SourcePackage.from_href(source_name)
-  resolvable = ResolvablePackage.from_string(source_name, ResolverOptionsBuilder())
+  resolvable = ResolvablePackage.from_string(source_name, builder)
   assert resolvable.packages() == [pkg]
 
   mock_iterator = mock.create_autospec(Iterator, spec_set=True)
@@ -34,14 +37,16 @@ def test_resolvable_package():
   assert mock_iterator.iter.mock_calls == []
   assert resolvable.name == 'foo'
   assert resolvable.exact is True
-  # TODO(wickman) Implement extras parsing for resolvable packages.
   assert resolvable.extras() == []
 
+  resolvable = ResolvablePackage.from_string(source_name + '[extra1,extra2]', builder)
+  assert resolvable.extras() == ['extra1', 'extra2']
+
   assert Resolvable.get('foo-2.3.4.tar.gz') == ResolvablePackage.from_string(
-      'foo-2.3.4.tar.gz', ResolverOptionsBuilder())
+      'foo-2.3.4.tar.gz', builder)
 
   with pytest.raises(ResolvablePackage.InvalidRequirement):
-    ResolvablePackage.from_string('foo', ResolverOptionsBuilder())
+    ResolvablePackage.from_string('foo', builder)
 
 
 def test_resolvable_repository():
@@ -53,11 +58,12 @@ def test_resolvable_repository():
 
 def test_resolvable_requirement():
   req = 'foo[bar]==2.3.4'
-  resolvable = ResolvableRequirement.from_string(req, ResolverOptionsBuilder())
+  resolvable = ResolvableRequirement.from_string(req, ResolverOptionsBuilder(fetchers=[]))
   assert resolvable.requirement == pkg_resources.Requirement.parse('foo[bar]==2.3.4')
   assert resolvable.name == 'foo'
   assert resolvable.exact is True
   assert resolvable.extras() == ['bar']
+  assert resolvable.options._fetchers == []
   assert resolvable.packages() == []
 
   source_pkg = SourcePackage.from_href('foo-2.3.4.tar.gz')
@@ -76,6 +82,19 @@ def test_resolvable_requirement():
       'foo', ResolverOptionsBuilder())
 
 
+def test_resolvable_directory():
+  builder = ResolverOptionsBuilder()
+
+  with make_source_dir(name='my_project') as td:
+    rdir = ResolvableDirectory.from_string(td, builder)
+    assert rdir.name == pkg_resources.safe_name('my_project')
+    assert rdir.extras() == []
+
+    rdir = ResolvableDirectory.from_string(td + '[extra1,extra2]', builder)
+    assert rdir.name == pkg_resources.safe_name('my_project')
+    assert rdir.extras() == ['extra1', 'extra2']
+
+
 def test_resolvables_from_iterable():
   builder = ResolverOptionsBuilder()
 

tox.ini

@@ -111,10 +111,10 @@ commands = pex {posargs:}
 commands = pex {posargs:}
 
 [testenv:py27-package]
-commands = pex --cache-dir {envtmpdir}/buildcache setuptools wheel requests -s . -o dist/pex -e pex.bin.pex:main -v
+commands = pex --cache-dir {envtmpdir}/buildcache setuptools wheel requests . -o dist/pex -e pex.bin.pex:main -v
 
 [testenv:py34-package]
-commands = pex --cache-dir {envtmpdir}/buildcache setuptools wheel requests -s . -o dist/pex -e pex.bin.pex:main -v
+commands = pex --cache-dir {envtmpdir}/buildcache setuptools wheel requests . -o dist/pex -e pex.bin.pex:main -v
 
 # Would love if you didn't have to enumerate environments here :-\
 [testenv:py26]