diff --git a/README.md b/README.md
index a6a87109cb..aea05d57ab 100644
--- a/README.md
+++ b/README.md
@@ -106,6 +106,10 @@ for dependencies, however, it is recommended that folks stick with the
`requirement` pattern in case the need arises for us to make changes to this
format in the future.
+["Extras"](
+https://packaging.python.org/tutorials/installing-packages/#installing-setuptools-extras)
+will have a target of the extra name (in place of `pkg` above).
+
## Updating `docs/`
All of the content (except `BUILD`) under `docs/` is generated. To update the
diff --git a/WORKSPACE b/WORKSPACE
index 0f205befec..430dfe7f07 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -26,8 +26,8 @@ sass_repositories()
git_repository(
name = "io_bazel_skydoc",
- remote = "https://github.com/bazelbuild/skydoc.git",
commit = "e9be81cf5be41e4200749f5d8aa2db7955f8aacc",
+ remote = "https://github.com/bazelbuild/skydoc.git",
)
load("@io_bazel_skydoc//skylark:skylark.bzl", "skydoc_repositories")
@@ -92,6 +92,15 @@ http_file(
"mock-2.0.0-py2.py3-none-any.whl"),
)
+http_file(
+ name = "google_cloud_language_whl",
+ sha256 = "a2dd34f0a0ebf5705dcbe34bd41199b1d0a55c4597d38ed045bd183361a561e9",
+ # From https://pypi.python.org/pypi/google-cloud-language
+ url = ("https://pypi.python.org/packages/6e/86/" +
+ "cae57e4802e72d9e626ee5828ed5a646cf4016b473a4a022f1038dba3460/" +
+ "google_cloud_language-0.29.0-py2.py3-none-any.whl"),
+)
+
# Imports for examples
pip_import(
name = "examples_helloworld",
@@ -128,3 +137,15 @@ load(
)
_boto_install()
+
+pip_import(
+ name = "examples_extras",
+ requirements = "//examples/extras:requirements.txt",
+)
+
+load(
+ "@examples_extras//:requirements.bzl",
+ _extras_install = "pip_install",
+)
+
+_extras_install()
diff --git a/examples/extras/BUILD b/examples/extras/BUILD
new file mode 100644
index 0000000000..94880ce47c
--- /dev/null
+++ b/examples/extras/BUILD
@@ -0,0 +1,29 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"]) # Apache 2.0
+
+load("@examples_extras//:requirements.bzl", "requirement")
+load("//python:python.bzl", "py_test")
+
+py_test(
+ name = "extras_test",
+ srcs = ["extras_test.py"],
+ deps = [
+ requirement("google-cloud-language"),
+ # Make sure that we can resolve the "extra" dependency
+ requirement("googleapis-common-protos[grpc]"),
+ ],
+)
diff --git a/examples/extras/extras_test.py b/examples/extras/extras_test.py
new file mode 100644
index 0000000000..3ad249d5a0
--- /dev/null
+++ b/examples/extras/extras_test.py
@@ -0,0 +1,25 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+
+# The test is the build itself, which should not work if extras are missing.
+class ExtrasTest(unittest.TestCase):
+ def test_nothing(self):
+ pass
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/examples/extras/requirements.txt b/examples/extras/requirements.txt
new file mode 100644
index 0000000000..743bbe7921
--- /dev/null
+++ b/examples/extras/requirements.txt
@@ -0,0 +1 @@
+google-cloud-language==0.27.0
diff --git a/examples/extras/version_test.py b/examples/extras/version_test.py
new file mode 100644
index 0000000000..9b469b76d3
--- /dev/null
+++ b/examples/extras/version_test.py
@@ -0,0 +1,26 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pip
+import unittest
+
+
+class VersionTest(unittest.TestCase):
+
+ def test_version(self):
+ self.assertEqual(pip.__version__, '9.0.1')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/python/whl.bzl b/python/whl.bzl
index f7827ed51f..7a19947c5f 100644
--- a/python/whl.bzl
+++ b/python/whl.bzl
@@ -16,12 +16,17 @@
def _whl_impl(repository_ctx):
"""Core implementation of whl_library."""
- result = repository_ctx.execute([
+ args = [
"python",
repository_ctx.path(repository_ctx.attr._script),
"--whl", repository_ctx.path(repository_ctx.attr.whl),
"--requirements", repository_ctx.attr.requirements,
- ])
+ ]
+
+ if repository_ctx.attr.extras:
+ args += ["--extras", ",".join(repository_ctx.attr.extras)]
+
+ result = repository_ctx.execute(args)
if result.return_code:
fail("whl_library failed: %s (%s)" % (result.stdout, result.stderr))
@@ -33,6 +38,7 @@ whl_library = repository_rule(
single_file = True,
),
"requirements": attr.string(),
+ "extras": attr.string_list(),
"_script": attr.label(
executable = True,
default = Label("//rules_python:whl.py"),
@@ -64,4 +70,7 @@ Args:
requirements: The name of the pip_import repository rule from which to
load this .whl's dependencies.
+
+ extras: A subset of the "extras" available from this .whl
for which
+ requirements
has the dependencies.
"""
diff --git a/rules_python/BUILD b/rules_python/BUILD
index 08aca7e530..e0bf79667e 100644
--- a/rules_python/BUILD
+++ b/rules_python/BUILD
@@ -28,6 +28,7 @@ py_test(
data = [
"@futures_3_1_1_whl//file",
"@futures_2_2_0_whl//file",
+ "@google_cloud_language_whl//file",
"@grpc_whl//file",
"@mock_whl//file",
],
diff --git a/rules_python/piptool.py b/rules_python/piptool.py
index 98dfce9c48..f5d504aa87 100644
--- a/rules_python/piptool.py
+++ b/rules_python/piptool.py
@@ -18,6 +18,7 @@
import json
import os
import pkgutil
+import pkg_resources
import re
import shutil
import sys
@@ -78,36 +79,7 @@ def pip_main(argv):
argv = ["--disable-pip-version-check", "--cert", cert_path] + argv
return pip.main(argv)
-
-# TODO(mattmoor): We can't easily depend on other libraries when
-# being invoked as a raw .py file. Once bundled, we should be able
-# to remove this fallback on a stub implementation of Wheel.
-try:
- from rules_python.whl import Wheel
-except:
- class Wheel(object):
-
- def __init__(self, path):
- self._path = path
-
- def basename(self):
- return os.path.basename(self._path)
-
- def distribution(self):
- # See https://www.python.org/dev/peps/pep-0427/#file-name-convention
- parts = self.basename().split('-')
- return parts[0]
-
- def version(self):
- # See https://www.python.org/dev/peps/pep-0427/#file-name-convention
- parts = self.basename().split('-')
- return parts[1]
-
- def repository_name(self):
- # Returns the canonical name of the Bazel repository for this package.
- canonical = 'pypi__{}_{}'.format(self.distribution(), self.version())
- # Escape any illegal characters with underscore.
- return re.sub('[-.]', '_', canonical)
+from rules_python.whl import Wheel
parser = argparse.ArgumentParser(
description='Import Python dependencies into Bazel.')
@@ -124,6 +96,59 @@ def repository_name(self):
parser.add_argument('--directory', action='store',
help=('The directory into which to put .whl files.'))
+def determine_possible_extras(whls):
+ """Determines the list of possible "extras" for each .whl
+
+ The possibility of an extra is determined by looking at its
+ additional requirements, and determinine whether they are
+ satisfied by the complete list of available wheels.
+
+ Args:
+ whls: a list of Wheel objects
+
+ Returns:
+ a dict that is keyed by the Wheel objects in whls, and whose
+ values are lists of possible extras.
+ """
+ whl_map = {
+ whl.distribution(): whl
+ for whl in whls
+ }
+
+ # TODO(mattmoor): Consider memoizing if this recursion ever becomes
+ # expensive enough to warrant it.
+ def is_possible(distro, extra):
+ distro = distro.replace("-", "_")
+ # If we don't have the .whl at all, then this isn't possible.
+ if distro not in whl_map:
+ return False
+ whl = whl_map[distro]
+ # If we have the .whl, and we don't need anything extra then
+ # we can satisfy this dependency.
+ if not extra:
+ return True
+ # If we do need something extra, then check the extra's
+ # dependencies to make sure they are fully satisfied.
+ for extra_dep in whl.dependencies(extra=extra):
+ req = pkg_resources.Requirement.parse(extra_dep)
+ # Check that the dep and any extras are all possible.
+ if not is_possible(req.project_name, None):
+ return False
+ for e in req.extras:
+ if not is_possible(req.project_name, e):
+ return False
+ # If all of the dependencies of the extra are satisfiable then
+ # it is possible to construct this dependency.
+ return True
+
+ return {
+ whl: [
+ extra
+ for extra in whl.extras()
+ if is_possible(whl.distribution(), extra)
+ ]
+ for whl in whls
+ }
def main():
args = parser.parse_args()
@@ -140,6 +165,9 @@ def list_whls():
if fname.endswith('.whl'):
yield os.path.join(root, fname)
+ whls = [Wheel(path) for path in list_whls()]
+ possible_extras = determine_possible_extras(whls)
+
def whl_library(wheel):
# Indentation here matters. whl_library must be within the scope
# of the function below. We also avoid reimporting an existing WHL.
@@ -149,10 +177,25 @@ def whl_library(wheel):
name = "{repo_name}",
whl = "@{name}//:{path}",
requirements = "@{name}//:requirements.bzl",
+ extras = [{extras}]
)""".format(name=args.name, repo_name=wheel.repository_name(),
- path=wheel.basename())
-
- whls = [Wheel(path) for path in list_whls()]
+ path=wheel.basename(),
+ extras=','.join([
+ '"%s"' % extra
+ for extra in possible_extras.get(wheel, [])
+ ]))
+
+ whl_targets = ','.join([
+ ','.join([
+ '"%s": "@%s//:pkg"' % (whl.distribution().lower(), whl.repository_name())
+ ] + [
+ # For every extra that is possible from this requirements.txt
+ '"%s[%s]": "@%s//:%s"' % (whl.distribution().lower(), extra.lower(),
+ whl.repository_name(), extra)
+ for extra in possible_extras.get(whl, [])
+ ])
+ for whl in whls
+ ])
with open(args.output, 'w') as f:
f.write("""\
@@ -178,10 +221,7 @@ def requirement(name):
return _requirements[name_key]
""".format(input=args.input,
whl_libraries='\n'.join(map(whl_library, whls)) if whls else "pass",
- mappings=','.join([
- '"%s": "@%s//:pkg"' % (wheel.distribution().lower(), wheel.repository_name())
- for wheel in whls
- ])))
+ mappings=whl_targets))
if __name__ == '__main__':
main()
diff --git a/rules_python/whl.py b/rules_python/whl.py
index 14d775a778..e3829bd9d6 100644
--- a/rules_python/whl.py
+++ b/rules_python/whl.py
@@ -70,13 +70,21 @@ def metadata(self):
def name(self):
return self.metadata().get('name')
- def dependencies(self):
+ def dependencies(self, extra=None):
+ """Access the dependencies of this Wheel.
+
+ Args:
+ extra: if specified, include the additional dependencies
+ of the named "extra".
+
+ Yields:
+ the names of requirements from the metadata.json
+ """
# TODO(mattmoor): Is there a schema to follow for this?
run_requires = self.metadata().get('run_requires', [])
for requirement in run_requires:
- if 'extra' in requirement:
- # TODO(mattmoor): What's the best way to support "extras"?
- # https://packaging.python.org/tutorials/installing-packages/#installing-setuptools-extras
+ if requirement.get('extra') != extra:
+ # Match the requirements for the extra we're looking for.
continue
if 'environment' in requirement:
# TODO(mattmoor): What's the best way to support "environment"?
@@ -89,6 +97,9 @@ def dependencies(self):
parts = re.split('[ ><=()]', entry)
yield parts[0]
+ def extras(self):
+ return self.metadata().get('extras', [])
+
def expand(self, directory):
with zipfile.ZipFile(self.path(), 'r') as whl:
whl.extractall(directory)
@@ -112,6 +123,9 @@ def _parse_metadata(self, content):
parser.add_argument('--directory', action='store', default='.',
help='The directory into which to expand things.')
+parser.add_argument('--extras', action='append',
+ help='The set of extras for which to generate library targets.')
+
def main():
args = parser.parse_args()
whl = Wheel(args.whl)
@@ -126,19 +140,33 @@ def main():
load("{requirements}", "requirement")
py_library(
- name = "pkg",
- srcs = glob(["**/*.py"]),
- data = glob(["**/*"], exclude=["**/*.py", "**/* *", "BUILD", "WORKSPACE"]),
- # This makes this directory a top-level in the python import
- # search path for anything that depends on this.
- imports = ["."],
- deps = [{dependencies}],
- )""".format(
- requirements=args.requirements,
- dependencies=','.join([
- 'requirement("%s")' % d
- for d in whl.dependencies()
- ])))
+ name = "pkg",
+ srcs = glob(["**/*.py"]),
+ data = glob(["**/*"], exclude=["**/*.py", "**/* *", "BUILD", "WORKSPACE"]),
+ # This makes this directory a top-level in the python import
+ # search path for anything that depends on this.
+ imports = ["."],
+ deps = [{dependencies}],
+)
+{extras}""".format(
+ requirements=args.requirements,
+ dependencies=','.join([
+ 'requirement("%s")' % d
+ for d in whl.dependencies()
+ ]),
+ extras='\n\n'.join([
+ """py_library(
+ name = "{extra}",
+ deps = [
+ ":pkg",{deps}
+ ],
+)""".format(extra=extra,
+ deps=','.join([
+ 'requirement("%s")' % dep
+ for dep in whl.dependencies(extra)
+ ]))
+ for extra in args.extras or []
+ ])))
if __name__ == '__main__':
main()
diff --git a/rules_python/whl_test.py b/rules_python/whl_test.py
index e148328275..c56a4e997d 100644
--- a/rules_python/whl_test.py
+++ b/rules_python/whl_test.py
@@ -33,6 +33,7 @@ def test_grpc_whl(self):
self.assertEqual(set(wheel.dependencies()),
set(['enum34', 'futures', 'protobuf', 'six']))
self.assertEqual('pypi__grpcio_1_6_0', wheel.repository_name())
+ self.assertEqual([], wheel.extras())
def test_futures_whl(self):
td = TestData('futures_3_1_1_whl/file/futures-3.1.1-py2-none-any.whl')
@@ -42,6 +43,7 @@ def test_futures_whl(self):
self.assertEqual(wheel.version(), '3.1.1')
self.assertEqual(set(wheel.dependencies()), set())
self.assertEqual('pypi__futures_3_1_1', wheel.repository_name())
+ self.assertEqual([], wheel.extras())
def test_whl_with_METADATA_file(self):
td = TestData('futures_2_2_0_whl/file/futures-2.2.0-py2.py3-none-any.whl')
@@ -61,6 +63,23 @@ def test_mock_whl(self):
self.assertEqual(set(wheel.dependencies()),
set(['pbr', 'six']))
self.assertEqual('pypi__mock_2_0_0', wheel.repository_name())
+ self.assertEqual(['docs', 'test'], wheel.extras())
+ self.assertEqual(set(wheel.dependencies(extra='docs')), set())
+ self.assertEqual(set(wheel.dependencies(extra='test')), set(['unittest2']))
+
+ def test_google_cloud_language_whl(self):
+ td = TestData('google_cloud_language_whl/file/' +
+ 'google_cloud_language-0.29.0-py2.py3-none-any.whl')
+ wheel = whl.Wheel(td)
+ self.assertEqual(wheel.name(), 'google-cloud-language')
+ self.assertEqual(wheel.distribution(), 'google_cloud_language')
+ self.assertEqual(wheel.version(), '0.29.0')
+ self.assertEqual(set(wheel.dependencies()),
+ set(['google-gax', 'google-cloud-core',
+ 'googleapis-common-protos[grpc]']))
+ self.assertEqual('pypi__google_cloud_language_0_29_0',
+ wheel.repository_name())
+ self.assertEqual([], wheel.extras())
if __name__ == '__main__':
unittest.main()
diff --git a/tools/piptool.par b/tools/piptool.par
index 29b1289770..41019234d3 100755
Binary files a/tools/piptool.par and b/tools/piptool.par differ