-
Notifications
You must be signed in to change notification settings - Fork 10.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Basic setup to build gRPC Python with Bazel
Building gRPC Python with Bazel has been one of the long requested additions to gRPC (#8079). Doing so had been made complex by the fact that Bazel itself is still in active development. There has been extensive work on building Cython code at tensorflow, which can be reused for gRPC's purposes as well. Major included changes required for building grpcio with Bazel are: - Include Cython as a third party Bazel package, to compile the Cython parts of gRPC Python. - Include rules for Python autoconfiguration so Python headers can be detected by cygrpc.
- Loading branch information
1 parent
0ed80ea
commit 3af1aaa
Showing
6 changed files
with
454 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,74 @@ | ||
"""Custom rules for gRPC Python""" | ||
|
||
|
||
# Adapted with modifications from | ||
# tensorflow/tensorflow/core/platform/default/build_config.bzl | ||
# Native Bazel rules don't exist yet to compile Cython code, but rules have | ||
# been written at cython/cython and tensorflow/tensorflow. We branch from | ||
# Tensorflow's version as it is more actively maintained and works for gRPC | ||
# Python's needs. | ||
def pyx_library(name, deps=[], py_deps=[], srcs=[], **kwargs): | ||
"""Compiles a group of .pyx / .pxd / .py files. | ||
First runs Cython to create .cpp files for each input .pyx or .py + .pxd | ||
pair. Then builds a shared object for each, passing "deps" to each cc_binary | ||
rule (includes Python headers by default). Finally, creates a py_library rule | ||
with the shared objects and any pure Python "srcs", with py_deps as its | ||
dependencies; the shared objects can be imported like normal Python files. | ||
Args: | ||
name: Name for the rule. | ||
deps: C/C++ dependencies of the Cython (e.g. Numpy headers). | ||
py_deps: Pure Python dependencies of the final library. | ||
srcs: .py, .pyx, or .pxd files to either compile or pass through. | ||
**kwargs: Extra keyword arguments passed to the py_library. | ||
""" | ||
# First filter out files that should be run compiled vs. passed through. | ||
py_srcs = [] | ||
pyx_srcs = [] | ||
pxd_srcs = [] | ||
for src in srcs: | ||
if src.endswith(".pyx") or (src.endswith(".py") and | ||
src[:-3] + ".pxd" in srcs): | ||
pyx_srcs.append(src) | ||
elif src.endswith(".py"): | ||
py_srcs.append(src) | ||
else: | ||
pxd_srcs.append(src) | ||
if src.endswith("__init__.py"): | ||
pxd_srcs.append(src) | ||
|
||
# Invoke cython to produce the shared object libraries. | ||
for filename in pyx_srcs: | ||
native.genrule( | ||
name=filename + "_cython_translation", | ||
srcs=[filename], | ||
outs=[filename.split(".")[0] + ".cpp"], | ||
# Optionally use PYTHON_BIN_PATH on Linux platforms so that python 3 | ||
# works. Windows has issues with cython_binary so skip PYTHON_BIN_PATH. | ||
cmd= | ||
"PYTHONHASHSEED=0 $(location @cython//:cython_binary) --cplus $(SRCS) --output-file $(OUTS)", | ||
tools=["@cython//:cython_binary"] + pxd_srcs, | ||
) | ||
|
||
shared_objects = [] | ||
for src in pyx_srcs: | ||
stem = src.split(".")[0] | ||
shared_object_name = stem + ".so" | ||
native.cc_binary( | ||
name=shared_object_name, | ||
srcs=[stem + ".cpp"], | ||
deps=deps + ["@local_config_python//:python_headers"], | ||
linkshared=1, | ||
) | ||
shared_objects.append(shared_object_name) | ||
|
||
# Now create a py_library with these shared objects as data. | ||
native.py_library( | ||
name=name, | ||
srcs=py_srcs, | ||
deps=py_deps, | ||
srcs_version="PY2AND3", | ||
data=shared_objects, | ||
**kwargs) | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
# Adapted with modifications from tensorflow/third_party/cython.BUILD | ||
|
||
py_library( | ||
name="cython_lib", | ||
srcs=glob( | ||
["Cython/**/*.py"], | ||
exclude=[ | ||
"**/Tests/*.py", | ||
], | ||
) + ["cython.py"], | ||
data=glob([ | ||
"Cython/**/*.pyx", | ||
"Cython/Utility/*.*", | ||
"Cython/Includes/**/*.pxd", | ||
]), | ||
srcs_version="PY2AND3", | ||
visibility=["//visibility:public"], | ||
) | ||
|
||
# May not be named "cython", since that conflicts with Cython/ on OSX | ||
py_binary( | ||
name="cython_binary", | ||
srcs=["cython.py"], | ||
main="cython.py", | ||
srcs_version="PY2AND3", | ||
visibility=["//visibility:public"], | ||
deps=["cython_lib"], | ||
) | ||
|
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
# Adapted with modifications from tensorflow/third_party/py/ | ||
|
||
package(default_visibility=["//visibility:public"]) | ||
|
||
# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib | ||
# See https://docs.python.org/3/extending/windows.html | ||
cc_import( | ||
name="python_lib", | ||
interface_library=select({ | ||
":windows": ":python_import_lib", | ||
# A placeholder for Unix platforms which makes --no_build happy. | ||
"//conditions:default": "not-existing.lib", | ||
}), | ||
system_provided=1, | ||
) | ||
|
||
cc_library( | ||
name="python_headers", | ||
hdrs=[":python_include"], | ||
deps=select({ | ||
":windows": [":python_lib"], | ||
"//conditions:default": [], | ||
}), | ||
includes=["python_include"], | ||
) | ||
|
||
config_setting( | ||
name="windows", | ||
values={"cpu": "x64_windows"}, | ||
visibility=["//visibility:public"], | ||
) | ||
|
||
%{PYTHON_INCLUDE_GENRULE} | ||
%{PYTHON_IMPORT_LIB_GENRULE} | ||
|
||
|
Oops, something went wrong.