Permalink
Browse files

Remove hack for building Python support with Bazel.

This change makes use of new imports attribute for Bazel's Python rules, which
enable adding directories to the PYTHONPATH. This allows us to remove
the hack for building protobuf's Python support with Bazel and now
allows projects to include protobuf using a Bazel external repository
rather than requiring it to be imported directly into the source tree as
//google/protobuf.

This change also updates the protobuf BUILD file to use a named
repository, @python//, for including Python headers rather than
//util/python. This allows projects to specify their own package for
Python headers when including protobuf with an external repository.

Fixes #1230
  • Loading branch information...
davidzchen committed Feb 12, 2016
1 parent fb714b3 commit 985c968443e5124327fb600a91856192df4476ac
Showing with 51 additions and 96 deletions.
  1. +15 −37 BUILD
  2. +19 −14 WORKSPACE
  3. +6 −44 protobuf.bzl
  4. +11 −1 util/python/BUILD
View
52 BUILD
@@ -22,7 +22,6 @@ load(
"protobuf",
"cc_proto_library",
"py_proto_library",
- "internal_copied_filegroup",
"internal_protobuf_py_tests",
)
@@ -484,25 +483,7 @@ java_library(
# Python support
################################################################################
-# Hack:
-# protoc generated files contain imports like:
-# "from google.protobuf.xxx import yyy"
-# However, the sources files of the python runtime are not directly under
-# "google/protobuf" (they are under python/google/protobuf). We workaround
-# this by copying runtime source files into the desired location to workaround
-# the import issue. Ideally py_library should support something similiar to the
-# "include" attribute in cc_library to inject the PYTHON_PATH for all libraries
-# that depend on the target.
-#
-# If you use python protobuf as a third_party library in your bazel managed
-# project:
-# 1) Please import the whole package to //google/protobuf in your
-# project. Otherwise, bazel disallows generated files out of the current
-# package, thus we won't be able to copy protobuf runtime files into
-# //google/protobuf/.
-# 2) The runtime also requires "six" for Python2/3 compatibility, please see the
-# WORKSPACE file and bind "six" to your workspace as well.
-internal_copied_filegroup(
+py_library(
name = "python_srcs",
srcs = glob(
[
@@ -514,7 +495,7 @@ internal_copied_filegroup(
"python/google/protobuf/internal/test_util.py",
],
),
- include = "python",
+ imports = ["python"],
)
cc_binary(
@@ -527,7 +508,7 @@ cc_binary(
linkstatic = 1,
deps = select({
"//conditions:default": [],
- ":use_fast_cpp_protos": ["//util/python:python_headers"],
+ ":use_fast_cpp_protos": ["//external:python_headers"],
}),
)
@@ -553,7 +534,7 @@ cc_binary(
":protobuf",
] + select({
"//conditions:default": [],
- ":use_fast_cpp_protos": ["//util/python:python_headers"],
+ ":use_fast_cpp_protos": ["//external:python_headers"],
}),
)
@@ -584,23 +565,14 @@ py_proto_library(
}),
default_runtime = "",
protoc = ":protoc",
- py_extra_srcs = [":python_srcs"],
- py_libs = ["//external:six"],
+ py_libs = [
+ ":python_srcs",
+ "//external:six"
+ ],
srcs_version = "PY2AND3",
visibility = ["//visibility:public"],
)
-internal_copied_filegroup(
- name = "python_test_srcs",
- srcs = glob(
- [
- "python/google/protobuf/internal/*_test.py",
- "python/google/protobuf/internal/test_util.py",
- ],
- ),
- include = "python",
-)
-
py_proto_library(
name = "python_common_test_protos",
srcs = LITE_TEST_PROTOS + TEST_PROTOS,
@@ -624,7 +596,13 @@ py_proto_library(
py_library(
name = "python_tests",
- srcs = [":python_test_srcs"],
+ srcs = glob(
+ [
+ "python/google/protobuf/internal/*_test.py",
+ "python/google/protobuf/internal/test_util.py",
+ ],
+ ),
+ imports = ["python"],
srcs_version = "PY2AND3",
deps = [
":protobuf_python",
View
@@ -1,28 +1,33 @@
new_http_archive(
- name = "gmock_archive",
- url = "https://googlemock.googlecode.com/files/gmock-1.7.0.zip",
- sha256 = "26fcbb5925b74ad5fc8c26b0495dfc96353f4d553492eb97e85a8a6d2f43095b",
- build_file = "gmock.BUILD",
+ name = "gmock_archive",
+ url = "https://googlemock.googlecode.com/files/gmock-1.7.0.zip",
+ sha256 = "26fcbb5925b74ad5fc8c26b0495dfc96353f4d553492eb97e85a8a6d2f43095b",
+ build_file = "gmock.BUILD",
)
new_http_archive(
- name = "six_archive",
- url = "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz#md5=34eed507548117b2ab523ab14b2f8b55",
- sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
- build_file = "six.BUILD",
+ name = "six_archive",
+ url = "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz#md5=34eed507548117b2ab523ab14b2f8b55",
+ sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
+ build_file = "six.BUILD",
)
bind(
- name = "gtest",
- actual = "@gmock_archive//:gtest",
+ name = "python_headers",
+ actual = "//util/python:python_headers",
)
bind(
- name = "gtest_main",
- actual = "@gmock_archive//:gtest_main",
+ name = "gtest",
+ actual = "@gmock_archive//:gtest",
)
bind(
- name = "six",
- actual = "@six_archive//:six",
+ name = "gtest_main",
+ actual = "@gmock_archive//:gtest_main",
+)
+
+bind(
+ name = "six",
+ actual = "@six_archive//:six",
)
View
@@ -117,10 +117,10 @@ def cc_proto_library(
deps=[],
cc_libs=[],
include=None,
- protoc="//google/protobuf:protoc",
+ protoc="//:protoc",
internal_bootstrap_hack=False,
use_grpc_plugin=False,
- default_runtime="//google/protobuf:protobuf",
+ default_runtime="//:protobuf",
**kargs):
"""Bazel rule to create a C++ protobuf library from proto source files
@@ -199,44 +199,15 @@ def cc_proto_library(
includes=includes,
**kargs)
-def internal_copied_filegroup(
- name,
- srcs,
- include,
- **kargs):
- """Bazel rule to fix sources file to workaround with python path issues.
-
- Args:
- name: the name of the internal_copied_filegroup rule, which will be the
- name of the generated filegroup.
- srcs: the source files to be copied.
- include: the expected import root of the source.
- **kargs: extra arguments that will be passed into the filegroup.
- """
- outs = [_RelativeOutputPath(s, include) for s in srcs]
-
- native.genrule(
- name=name+"_genrule",
- srcs=srcs,
- outs=outs,
- cmd=" && ".join(["cp $(location %s) $(location %s)" %
- (s, _RelativeOutputPath(s, include))
- for s in srcs]))
-
- native.filegroup(
- name=name,
- srcs=outs,
- **kargs)
-
def py_proto_library(
name,
srcs=[],
deps=[],
py_libs=[],
py_extra_srcs=[],
include=None,
- default_runtime="//google/protobuf:protobuf_python",
- protoc="//google/protobuf:protoc",
+ default_runtime="//:protobuf_python",
+ protoc="//:protoc",
**kargs):
"""Bazel rule to create a Python protobuf library from proto source files
@@ -276,22 +247,14 @@ def py_proto_library(
visibility=["//visibility:public"],
)
- if include != None:
- # Copy the output files to the desired location to make the import work.
- internal_copied_filegroup_name=name + "_internal_copied_filegroup"
- internal_copied_filegroup(
- name=internal_copied_filegroup_name,
- srcs=outs,
- include=include)
- outs=[internal_copied_filegroup_name]
-
if default_runtime and not default_runtime in py_libs + deps:
py_libs += [default_runtime]
native.py_library(
name=name,
srcs=outs+py_extra_srcs,
deps=py_libs+deps,
+ imports=includes,
**kargs)
def internal_protobuf_py_tests(
@@ -308,8 +271,7 @@ def internal_protobuf_py_tests(
"""
for m in modules:
- s = _RelativeOutputPath(
- "python/google/protobuf/internal/%s.py" % m, "python")
+ s = "python/google/protobuf/internal/%s.py" % m
native.py_test(
name="py_%s" % m,
srcs=[s],
View
@@ -1,7 +1,17 @@
# This is a placeholder for python headers. Projects needing to use
# fast cpp protos in protobuf's python interface should build with
# --define=use_fast_cpp_protos=true, and in addition, provide
-# //util/python:python_headers dependency that in turn provides Python.h.
+# //external:python_headers dependency that in turn provides Python.h.
+#
+# Projects that include protobuf using a Bazel external repository will need to
+# add a workspace rule to their WORKSPACE files to add an external workspace
+# that includes the Python headers. For example, the protobuf WORKSPACE file
+# includes the following local_repository rule that points to this directory:
+#
+# new_local_repository(
+# name = "python",
+# path = __workspace_dir__ + "/util/python",
+# )
cc_library(
name = "python_headers",
visibility = ["//visibility:public"],

5 comments on commit 985c968

@damienmg

This comment has been minimized.

Show comment
Hide comment
@damienmg

damienmg Feb 26, 2016

Member

@davidzchen : I think this has broken the python test suite, can you have a look? http://ci.bazel.io/job/protobuf/92/

/cc @philwo

Member

damienmg replied Feb 26, 2016

@davidzchen : I think this has broken the python test suite, can you have a look? http://ci.bazel.io/job/protobuf/92/

/cc @philwo

@davidzchen

This comment has been minimized.

Show comment
Hide comment
@davidzchen

davidzchen Feb 26, 2016

Member

Looking.

Member

davidzchen replied Feb 26, 2016

Looking.

@davidzchen

This comment has been minimized.

Show comment
Hide comment
@davidzchen

davidzchen Feb 26, 2016

Member

I have root-caused the test failures. Interestingly, they are not caused by this patch but by a bug in the Python launcher where it does not check whether the file is a directory when looking for the python binary.

The fix is simple. I have opened bazelbuild/bazel#981 for the bug.

Member

davidzchen replied Feb 26, 2016

I have root-caused the test failures. Interestingly, they are not caused by this patch but by a bug in the Python launcher where it does not check whether the file is a directory when looking for the python binary.

The fix is simple. I have opened bazelbuild/bazel#981 for the bug.

@damienmg

This comment has been minimized.

Show comment
Hide comment
@damienmg

damienmg Feb 26, 2016

Member
Member

damienmg replied Feb 26, 2016

@andrewharp

This comment has been minimized.

Show comment
Hide comment
@andrewharp

andrewharp Apr 5, 2016

Member

I'm trying to update Tensorflow to use a newer version of protobuf, but am having some trouble with this commit.

With no other change than bumping our protobuf version up to this commit from the preceding, I get this error:

$ bazel build -c opt //tensorflow/core:core
ERROR: /tmp/tf-update-pb/tensorflow/tensorflow/core/BUILD:74:1: no such package '': BUILD file not found on package path and referenced by '//tensorflow/core:protos_all_cc'.

I then added:

new_local_repository(
  name = "python",
  path = __workspace_dir__ + "/google/protobuf/util/python",
)

to our WORKSPACE file, but this made no difference. Any ideas?

Member

andrewharp replied Apr 5, 2016

I'm trying to update Tensorflow to use a newer version of protobuf, but am having some trouble with this commit.

With no other change than bumping our protobuf version up to this commit from the preceding, I get this error:

$ bazel build -c opt //tensorflow/core:core
ERROR: /tmp/tf-update-pb/tensorflow/tensorflow/core/BUILD:74:1: no such package '': BUILD file not found on package path and referenced by '//tensorflow/core:protos_all_cc'.

I then added:

new_local_repository(
  name = "python",
  path = __workspace_dir__ + "/google/protobuf/util/python",
)

to our WORKSPACE file, but this made no difference. Any ideas?

Please sign in to comment.