Skip to content

Commit

Permalink
Basic setup to build gRPC Python with Bazel
Browse files Browse the repository at this point in the history
Building gRPC Python with Bazel has been one of the long requested
additions to gRPC (#8079). Doing so had been made complex by the fact
that Bazel itself is still in active development.

There has been extensive work on building Cython code at tensorflow,
which can be reused for gRPC's purposes as well.

Major included changes required for building grpcio with Bazel are:
- Include Cython as a third party Bazel package, to compile the Cython
parts of gRPC Python.
- Include rules for Python autoconfiguration so Python headers can be
detected by cygrpc.
  • Loading branch information
ghostwriternr committed Jul 27, 2018
1 parent 0ed80ea commit 3af1aaa
Show file tree
Hide file tree
Showing 6 changed files with 454 additions and 0 deletions.
74 changes: 74 additions & 0 deletions bazel/cython_library.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
"""Custom rules for gRPC Python"""


# Adapted with modifications from
# tensorflow/tensorflow/core/platform/default/build_config.bzl
# Native Bazel rules don't exist yet to compile Cython code, but rules have
# been written at cython/cython and tensorflow/tensorflow. We branch from
# Tensorflow's version as it is more actively maintained and works for gRPC
# Python's needs.
def pyx_library(name, deps=[], py_deps=[], srcs=[], **kwargs):
"""Compiles a group of .pyx / .pxd / .py files.
First runs Cython to create .cpp files for each input .pyx or .py + .pxd
pair. Then builds a shared object for each, passing "deps" to each cc_binary
rule (includes Python headers by default). Finally, creates a py_library rule
with the shared objects and any pure Python "srcs", with py_deps as its
dependencies; the shared objects can be imported like normal Python files.
Args:
name: Name for the rule.
deps: C/C++ dependencies of the Cython (e.g. Numpy headers).
py_deps: Pure Python dependencies of the final library.
srcs: .py, .pyx, or .pxd files to either compile or pass through.
**kwargs: Extra keyword arguments passed to the py_library.
"""
# First filter out files that should be run compiled vs. passed through.
py_srcs = []
pyx_srcs = []
pxd_srcs = []
for src in srcs:
if src.endswith(".pyx") or (src.endswith(".py") and
src[:-3] + ".pxd" in srcs):
pyx_srcs.append(src)
elif src.endswith(".py"):
py_srcs.append(src)
else:
pxd_srcs.append(src)
if src.endswith("__init__.py"):
pxd_srcs.append(src)

# Invoke cython to produce the shared object libraries.
for filename in pyx_srcs:
native.genrule(
name=filename + "_cython_translation",
srcs=[filename],
outs=[filename.split(".")[0] + ".cpp"],
# Optionally use PYTHON_BIN_PATH on Linux platforms so that python 3
# works. Windows has issues with cython_binary so skip PYTHON_BIN_PATH.
cmd=
"PYTHONHASHSEED=0 $(location @cython//:cython_binary) --cplus $(SRCS) --output-file $(OUTS)",
tools=["@cython//:cython_binary"] + pxd_srcs,
)

shared_objects = []
for src in pyx_srcs:
stem = src.split(".")[0]
shared_object_name = stem + ".so"
native.cc_binary(
name=shared_object_name,
srcs=[stem + ".cpp"],
deps=deps + ["@local_config_python//:python_headers"],
linkshared=1,
)
shared_objects.append(shared_object_name)

# Now create a py_library with these shared objects as data.
native.py_library(
name=name,
srcs=py_srcs,
deps=py_deps,
srcs_version="PY2AND3",
data=shared_objects,
**kwargs)

29 changes: 29 additions & 0 deletions third_party/cython.BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# Adapted with modifications from tensorflow/third_party/cython.BUILD

py_library(
name="cython_lib",
srcs=glob(
["Cython/**/*.py"],
exclude=[
"**/Tests/*.py",
],
) + ["cython.py"],
data=glob([
"Cython/**/*.pyx",
"Cython/Utility/*.*",
"Cython/Includes/**/*.pxd",
]),
srcs_version="PY2AND3",
visibility=["//visibility:public"],
)

# May not be named "cython", since that conflicts with Cython/ on OSX
py_binary(
name="cython_binary",
srcs=["cython.py"],
main="cython.py",
srcs_version="PY2AND3",
visibility=["//visibility:public"],
deps=["cython_lib"],
)

Empty file added third_party/py/BUILD
Empty file.
36 changes: 36 additions & 0 deletions third_party/py/BUILD.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Adapted with modifications from tensorflow/third_party/py/

package(default_visibility=["//visibility:public"])

# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
# See https://docs.python.org/3/extending/windows.html
cc_import(
name="python_lib",
interface_library=select({
":windows": ":python_import_lib",
# A placeholder for Unix platforms which makes --no_build happy.
"//conditions:default": "not-existing.lib",
}),
system_provided=1,
)

cc_library(
name="python_headers",
hdrs=[":python_include"],
deps=select({
":windows": [":python_lib"],
"//conditions:default": [],
}),
includes=["python_include"],
)

config_setting(
name="windows",
values={"cpu": "x64_windows"},
visibility=["//visibility:public"],
)

%{PYTHON_INCLUDE_GENRULE}
%{PYTHON_IMPORT_LIB_GENRULE}


Loading

0 comments on commit 3af1aaa

Please sign in to comment.