Skip to content

Commit

Permalink
Remove hack for building Python support with Bazel.
Browse files Browse the repository at this point in the history 
This change makes use of new imports attribute for Bazel's Python rules, which
enable adding directories to the PYTHONPATH. This allows us to remove
the hack for building protobuf's Python support with Bazel and now
allows projects to include protobuf using a Bazel external repository
rather than requiring it to be imported directly into the source tree as
//google/protobuf.

This change also updates the protobuf BUILD file to use a named
repository, @python//, for including Python headers rather than
//util/python. This allows projects to specify their own package for
Python headers when including protobuf with an external repository.

Fixes #1230
  • Loading branch information
@davidzchen
davidzchen committed Feb 25, 2016
1 parent fb714b3 commit 985c968
Show file tree
Hide file tree
Showing 4 changed files with 51 additions and 96 deletions.
52 changes: 15 additions & 37 deletions BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ load(
"protobuf",
"cc_proto_library",
"py_proto_library",
"internal_copied_filegroup",
"internal_protobuf_py_tests",
)

Expand Down Expand Up @@ -484,25 +483,7 @@ java_library(
# Python support
################################################################################

# Hack:
# protoc generated files contain imports like:
# "from google.protobuf.xxx import yyy"
# However, the sources files of the python runtime are not directly under
# "google/protobuf" (they are under python/google/protobuf). We workaround
# this by copying runtime source files into the desired location to workaround
# the import issue. Ideally py_library should support something similiar to the
# "include" attribute in cc_library to inject the PYTHON_PATH for all libraries
# that depend on the target.
#
# If you use python protobuf as a third_party library in your bazel managed
# project:
# 1) Please import the whole package to //google/protobuf in your
# project. Otherwise, bazel disallows generated files out of the current
# package, thus we won't be able to copy protobuf runtime files into
# //google/protobuf/.
# 2) The runtime also requires "six" for Python2/3 compatibility, please see the
# WORKSPACE file and bind "six" to your workspace as well.
internal_copied_filegroup(
py_library(
name = "python_srcs",
srcs = glob(
[
Expand All @@ -514,7 +495,7 @@ internal_copied_filegroup(
"python/google/protobuf/internal/test_util.py",
],
),
include = "python",
imports = ["python"],
)

cc_binary(
Expand All @@ -527,7 +508,7 @@ cc_binary(
linkstatic = 1,
deps = select({
"//conditions:default": [],
":use_fast_cpp_protos": ["//util/python:python_headers"],
":use_fast_cpp_protos": ["//external:python_headers"],
}),
)

Expand All @@ -553,7 +534,7 @@ cc_binary(
":protobuf",
] + select({
"//conditions:default": [],
":use_fast_cpp_protos": ["//util/python:python_headers"],
":use_fast_cpp_protos": ["//external:python_headers"],
}),
)

Expand Down Expand Up @@ -584,23 +565,14 @@ py_proto_library(
}),
default_runtime = "",
protoc = ":protoc",
py_extra_srcs = [":python_srcs"],
py_libs = ["//external:six"],
py_libs = [
":python_srcs",
"//external:six"
],
srcs_version = "PY2AND3",
visibility = ["//visibility:public"],
)

internal_copied_filegroup(
name = "python_test_srcs",
srcs = glob(
[
"python/google/protobuf/internal/*_test.py",
"python/google/protobuf/internal/test_util.py",
],
),
include = "python",
)

py_proto_library(
name = "python_common_test_protos",
srcs = LITE_TEST_PROTOS + TEST_PROTOS,
Expand All @@ -624,7 +596,13 @@ py_proto_library(

py_library(
name = "python_tests",
srcs = [":python_test_srcs"],
srcs = glob(
[
"python/google/protobuf/internal/*_test.py",
"python/google/protobuf/internal/test_util.py",
],
),
imports = ["python"],
srcs_version = "PY2AND3",
deps = [
":protobuf_python",
Expand Down
33 changes: 19 additions & 14 deletions WORKSPACE
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,33 @@
new_http_archive(
name = "gmock_archive",
url = "https://googlemock.googlecode.com/files/gmock-1.7.0.zip",
sha256 = "26fcbb5925b74ad5fc8c26b0495dfc96353f4d553492eb97e85a8a6d2f43095b",
build_file = "gmock.BUILD",
name = "gmock_archive",
url = "https://googlemock.googlecode.com/files/gmock-1.7.0.zip",
sha256 = "26fcbb5925b74ad5fc8c26b0495dfc96353f4d553492eb97e85a8a6d2f43095b",
build_file = "gmock.BUILD",
)

new_http_archive(
name = "six_archive",
url = "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz#md5=34eed507548117b2ab523ab14b2f8b55",
sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
build_file = "six.BUILD",
name = "six_archive",
url = "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz#md5=34eed507548117b2ab523ab14b2f8b55",
sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
build_file = "six.BUILD",
)

bind(
name = "gtest",
actual = "@gmock_archive//:gtest",
name = "python_headers",
actual = "//util/python:python_headers",
)

bind(
name = "gtest_main",
actual = "@gmock_archive//:gtest_main",
name = "gtest",
actual = "@gmock_archive//:gtest",
)

bind(
name = "six",
actual = "@six_archive//:six",
name = "gtest_main",
actual = "@gmock_archive//:gtest_main",
)

bind(
name = "six",
actual = "@six_archive//:six",
)
50 changes: 6 additions & 44 deletions protobuf.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,10 +117,10 @@ def cc_proto_library(
deps=[],
cc_libs=[],
include=None,
protoc="//google/protobuf:protoc",
protoc="//:protoc",
internal_bootstrap_hack=False,
use_grpc_plugin=False,
default_runtime="//google/protobuf:protobuf",
default_runtime="//:protobuf",
**kargs):
"""Bazel rule to create a C++ protobuf library from proto source files
Expand Down Expand Up @@ -199,44 +199,15 @@ def cc_proto_library(
includes=includes,
**kargs)

def internal_copied_filegroup(
name,
srcs,
include,
**kargs):
"""Bazel rule to fix sources file to workaround with python path issues.
Args:
name: the name of the internal_copied_filegroup rule, which will be the
name of the generated filegroup.
srcs: the source files to be copied.
include: the expected import root of the source.
**kargs: extra arguments that will be passed into the filegroup.
"""
outs = [_RelativeOutputPath(s, include) for s in srcs]

native.genrule(
name=name+"_genrule",
srcs=srcs,
outs=outs,
cmd=" && ".join(["cp $(location %s) $(location %s)" %
(s, _RelativeOutputPath(s, include))
for s in srcs]))

native.filegroup(
name=name,
srcs=outs,
**kargs)

def py_proto_library(
name,
srcs=[],
deps=[],
py_libs=[],
py_extra_srcs=[],
include=None,
default_runtime="//google/protobuf:protobuf_python",
protoc="//google/protobuf:protoc",
default_runtime="//:protobuf_python",
protoc="//:protoc",
**kargs):
"""Bazel rule to create a Python protobuf library from proto source files
Expand Down Expand Up @@ -276,22 +247,14 @@ def py_proto_library(
visibility=["//visibility:public"],
)

if include != None:
# Copy the output files to the desired location to make the import work.
internal_copied_filegroup_name=name + "_internal_copied_filegroup"
internal_copied_filegroup(
name=internal_copied_filegroup_name,
srcs=outs,
include=include)
outs=[internal_copied_filegroup_name]

if default_runtime and not default_runtime in py_libs + deps:
py_libs += [default_runtime]

native.py_library(
name=name,
srcs=outs+py_extra_srcs,
deps=py_libs+deps,
imports=includes,
**kargs)

def internal_protobuf_py_tests(
Expand All @@ -308,8 +271,7 @@ def internal_protobuf_py_tests(
"""
for m in modules:
s = _RelativeOutputPath(
"python/google/protobuf/internal/%s.py" % m, "python")
s = "python/google/protobuf/internal/%s.py" % m
native.py_test(
name="py_%s" % m,
srcs=[s],
Expand Down
12 changes: 11 additions & 1 deletion util/python/BUILD
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,17 @@
# This is a placeholder for python headers. Projects needing to use
# fast cpp protos in protobuf's python interface should build with
# --define=use_fast_cpp_protos=true, and in addition, provide
# //util/python:python_headers dependency that in turn provides Python.h.
# //external:python_headers dependency that in turn provides Python.h.
#
# Projects that include protobuf using a Bazel external repository will need to
# add a workspace rule to their WORKSPACE files to add an external workspace
# that includes the Python headers. For example, the protobuf WORKSPACE file
# includes the following local_repository rule that points to this directory:
#
# new_local_repository(
# name = "python",
# path = __workspace_dir__ + "/util/python",
# )
cc_library(
name = "python_headers",
visibility = ["//visibility:public"],
Expand Down

5 comments on commit 985c968

@damienmg
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@davidzchen : I think this has broken the python test suite, can you have a look? http://ci.bazel.io/job/protobuf/92/

/cc @philwo

@davidzchen
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looking.

@davidzchen
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have root-caused the test failures. Interestingly, they are not caused by this patch but by a bug in the Python launcher where it does not check whether the file is a directory when looking for the python binary.

The fix is simple. I have opened bazelbuild/bazel#981 for the bug.

@damienmg
Copy link
Contributor

@damienmg damienmg commented on 985c968 Feb 26, 2016 via email

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@andrewharp
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm trying to update Tensorflow to use a newer version of protobuf, but am having some trouble with this commit.

With no other change than bumping our protobuf version up to this commit from the preceding, I get this error:

$ bazel build -c opt //tensorflow/core:core
ERROR: /tmp/tf-update-pb/tensorflow/tensorflow/core/BUILD:74:1: no such package '': BUILD file not found on package path and referenced by '//tensorflow/core:protos_all_cc'.

I then added:

new_local_repository(
  name = "python",
  path = __workspace_dir__ + "/google/protobuf/util/python",
)

to our WORKSPACE file, but this made no difference. Any ideas?

Please sign in to comment.