From a49e001eeefc9c30b633d6d587a40dc3bd46fa80 Mon Sep 17 00:00:00 2001 From: simuons Date: Tue, 15 Jan 2019 08:48:21 +0200 Subject: [PATCH 01/15] Init --- scala_proto/scala_proto.bzl | 45 +++++++++++++++++++++++++++ src/scala/scripts/BUILD | 11 +++++++ src/scala/scripts/ScalaPBPlugin.scala | 10 ++++++ test/proto/a.proto | 4 +++ 4 files changed, 70 insertions(+) create mode 100644 src/scala/scripts/ScalaPBPlugin.scala create mode 100644 test/proto/a.proto diff --git a/scala_proto/scala_proto.bzl b/scala_proto/scala_proto.bzl index 81fe08eb8..dde97c682 100644 --- a/scala_proto/scala_proto.bzl +++ b/scala_proto/scala_proto.bzl @@ -18,6 +18,8 @@ load( "create_java_provider", ) +load("@com_google_protobuf//:protobuf.bzl", "proto_gen") + def scala_proto_repositories( scala_version = _default_scala_version(), maven_servers = ["http://central.maven.org/maven2"]): @@ -594,3 +596,46 @@ def scalapb_proto_library( scalac_jvm_flags = scalac_jvm_flags, visibility = visibility, ) + + +#def _scala_proto_library_impl(ctx): +#@com_google_protobuf//:protoc + +def scala_proto_library( + name, + deps, + flags = [], + plugin = "@io_bazel_rules_scala//src/scala/scripts:scalapb_plugin", + protoc = "@com_google_protobuf//:protoc", + visibility = None): + proto_gen( + name = name + "_genproto", + srcs = [], + deps = deps, + includes = [], + protoc = protoc, + plugin = plugin, + plugin_language = "scala", + plugin_options = flags, + outs = [], + visibility = ["//visibility:public"], + ) + + +#scala_proto_library = rule( +# implementation = _scala_proto_library_impl, +# attrs = { +# "deps": attr.label_list( +# mandatory = True, +# providers = [["proto"],[JavaInfo]], +# ), +# "flags": attr.string_list(default = []), +# "plugin": attr.label( +# default = "@io_bazel_rules_scala//src/scala/scripts:scalapb_plugin", +# executable = True, +# cfg = "host", +# ), +# "runtime": attr.label_list(), +# "_compiler": attr # or protoc +# }, +#) diff --git a/src/scala/scripts/BUILD b/src/scala/scripts/BUILD index e4ba98d4a..a610112c2 100644 --- a/src/scala/scripts/BUILD +++ b/src/scala/scripts/BUILD @@ -55,3 +55,14 @@ scala_binary( ":scalapb_generator_lib", ], ) + +scala_binary( + name = "scalapb_plugin", + srcs = ["ScalaPBPlugin.scala"], + main_class = "scripts.ScalaPBPlugin", + deps = [ + "//external:io_bazel_rules_scala/dependency/proto/scalapb_plugin", + "//external:io_bazel_rules_scala/dependency/com_google_protobuf/protobuf_java", + ], + visibility = ["//visibility:public"], +) diff --git a/src/scala/scripts/ScalaPBPlugin.scala b/src/scala/scripts/ScalaPBPlugin.scala new file mode 100644 index 000000000..15ab6019b --- /dev/null +++ b/src/scala/scripts/ScalaPBPlugin.scala @@ -0,0 +1,10 @@ +package scripts + +import com.google.protobuf.compiler.PluginProtos.CodeGeneratorRequest.parseFrom +import scalapb.compiler.ProtobufGenerator.handleCodeGeneratorRequest + +object ScalaPBPlugin extends App { + + handleCodeGeneratorRequest(parseFrom(System.in)).writeTo(System.out) + +} diff --git a/test/proto/a.proto b/test/proto/a.proto new file mode 100644 index 000000000..c6a55bd26 --- /dev/null +++ b/test/proto/a.proto @@ -0,0 +1,4 @@ +syntax = "proto3"; + +message A { +} From 66d10dd20543320acdd64a08bcde874cd0168091 Mon Sep 17 00:00:00 2001 From: simuons Date: Tue, 12 Feb 2019 14:09:57 +0200 Subject: [PATCH 02/15] introduce scala_proto_gen --- scala_proto/scala_proto.bzl | 78 +++++++++++++++++-------------------- test/proto/BUILD | 8 +++- test/proto/a.proto | 4 -- 3 files changed, 43 insertions(+), 47 deletions(-) delete mode 100644 test/proto/a.proto diff --git a/scala_proto/scala_proto.bzl b/scala_proto/scala_proto.bzl index dde97c682..45e24acdc 100644 --- a/scala_proto/scala_proto.bzl +++ b/scala_proto/scala_proto.bzl @@ -18,8 +18,6 @@ load( "create_java_provider", ) -load("@com_google_protobuf//:protobuf.bzl", "proto_gen") - def scala_proto_repositories( scala_version = _default_scala_version(), maven_servers = ["http://central.maven.org/maven2"]): @@ -597,45 +595,41 @@ def scalapb_proto_library( visibility = visibility, ) +def _scala_proto_gen_impl(ctx): + srcs = [f for dep in ctx.attr.deps for f in dep.proto.direct_sources] + includes = [f for dep in ctx.attr.deps for f in dep.proto.transitive_imports] -#def _scala_proto_library_impl(ctx): -#@com_google_protobuf//:protoc - -def scala_proto_library( - name, - deps, - flags = [], - plugin = "@io_bazel_rules_scala//src/scala/scripts:scalapb_plugin", - protoc = "@com_google_protobuf//:protoc", - visibility = None): - proto_gen( - name = name + "_genproto", - srcs = [], - deps = deps, - includes = [], - protoc = protoc, - plugin = plugin, - plugin_language = "scala", - plugin_options = flags, - outs = [], - visibility = ["//visibility:public"], - ) - + srcdotjar = ctx.actions.declare_file("_" + ctx.label.name + "_src.jar") -#scala_proto_library = rule( -# implementation = _scala_proto_library_impl, -# attrs = { -# "deps": attr.label_list( -# mandatory = True, -# providers = [["proto"],[JavaInfo]], -# ), -# "flags": attr.string_list(default = []), -# "plugin": attr.label( -# default = "@io_bazel_rules_scala//src/scala/scripts:scalapb_plugin", -# executable = True, -# cfg = "host", -# ), -# "runtime": attr.label_list(), -# "_compiler": attr # or protoc -# }, -#) + ctx.actions.run( + inputs = [ctx.executable._protoc, ctx.executable.plugin] + srcs + includes, + outputs = [srcdotjar], + arguments = [ + "--plugin=protoc-gen-scala=" + ctx.executable.plugin.path, + "--scala_out=%s:%s" % (",".join(ctx.attr.flags), srcdotjar.path)] + + ["-I{0}={1}".format(include.short_path, include.path) for include in includes] + + [src.short_path for src in srcs], + executable = ctx.executable._protoc, + mnemonic = "ScalaProtoGen", + use_default_shell_env = True, + ) + + ctx.actions.run_shell( + command = "cp $1 $2", + inputs = [srcdotjar], + outputs = [ctx.outputs.srcjar], + arguments = [srcdotjar.path, ctx.outputs.srcjar.path]) + +scala_proto_gen = rule( + _scala_proto_gen_impl, + attrs = { + "deps": attr.label_list(mandatory = True, providers = [["proto"]]), + "blacklisted_protos" : attr.label_list(providers = [["proto"]]), + "flags": attr.string_list(default = []), + "plugin": attr.label(executable = True, cfg = "host"), + "_protoc": attr.label(executable = True, cfg = "host", default = "@com_google_protobuf//:protoc") + }, + outputs = { + "srcjar": "lib%{name}.srcjar", + }, +) diff --git a/test/proto/BUILD b/test/proto/BUILD index 278440091..1c84933e7 100644 --- a/test/proto/BUILD +++ b/test/proto/BUILD @@ -1,7 +1,8 @@ load( "//scala_proto:scala_proto.bzl", "scalapb_proto_library", - "scala_proto_srcjar" + "scala_proto_srcjar", + "scala_proto_gen", ) load( @@ -116,3 +117,8 @@ scala_library( "@com_google_protobuf//:protobuf_java", "@scala_proto_rules_scalapb_lenses", "@scala_proto_rules_scalapb_runtime"]) + +scala_proto_gen( + name = "a_proto_scala", + deps = ["//test/proto2:test"], + plugin = "@io_bazel_rules_scala//src/scala/scripts:scalapb_plugin") diff --git a/test/proto/a.proto b/test/proto/a.proto deleted file mode 100644 index c6a55bd26..000000000 --- a/test/proto/a.proto +++ /dev/null @@ -1,4 +0,0 @@ -syntax = "proto3"; - -message A { -} From 262bd8d74d178086e8860f92afcb5be31f3d4647 Mon Sep 17 00:00:00 2001 From: simuons Date: Wed, 13 Feb 2019 08:49:02 +0200 Subject: [PATCH 03/15] scala_proto_gen based on descriptor sets --- scala_proto/scala_proto.bzl | 21 ++++++++++----------- test/proto/BUILD | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/scala_proto/scala_proto.bzl b/scala_proto/scala_proto.bzl index 45e24acdc..8a21e0cc7 100644 --- a/scala_proto/scala_proto.bzl +++ b/scala_proto/scala_proto.bzl @@ -596,29 +596,28 @@ def scalapb_proto_library( ) def _scala_proto_gen_impl(ctx): - srcs = [f for dep in ctx.attr.deps for f in dep.proto.direct_sources] - includes = [f for dep in ctx.attr.deps for f in dep.proto.transitive_imports] - + sources = [f for dep in ctx.attr.deps for f in dep.proto.direct_sources] + descriptors = [f for dep in ctx.attr.deps for f in dep.proto.transitive_descriptor_sets] srcdotjar = ctx.actions.declare_file("_" + ctx.label.name + "_src.jar") ctx.actions.run( - inputs = [ctx.executable._protoc, ctx.executable.plugin] + srcs + includes, + inputs = [ctx.executable._protoc, ctx.executable.plugin] + descriptors + sources, outputs = [srcdotjar], arguments = [ "--plugin=protoc-gen-scala=" + ctx.executable.plugin.path, - "--scala_out=%s:%s" % (",".join(ctx.attr.flags), srcdotjar.path)] - + ["-I{0}={1}".format(include.short_path, include.path) for include in includes] - + [src.short_path for src in srcs], + "--scala_out=%s:%s" % (",".join(ctx.attr.flags), srcdotjar.path), + "--descriptor_set_in=" + ":".join([descriptor.path for descriptor in descriptors])] + + [source.short_path for source in sources], executable = ctx.executable._protoc, mnemonic = "ScalaProtoGen", use_default_shell_env = True, ) ctx.actions.run_shell( - command = "cp $1 $2", - inputs = [srcdotjar], - outputs = [ctx.outputs.srcjar], - arguments = [srcdotjar.path, ctx.outputs.srcjar.path]) + command = "cp $1 $2", + inputs = [srcdotjar], + outputs = [ctx.outputs.srcjar], + arguments = [srcdotjar.path, ctx.outputs.srcjar.path]) scala_proto_gen = rule( _scala_proto_gen_impl, diff --git a/test/proto/BUILD b/test/proto/BUILD index 1c84933e7..e3266225d 100644 --- a/test/proto/BUILD +++ b/test/proto/BUILD @@ -122,3 +122,17 @@ scala_proto_gen( name = "a_proto_scala", deps = ["//test/proto2:test"], plugin = "@io_bazel_rules_scala//src/scala/scripts:scalapb_plugin") + +scala_proto_gen( + name = "test2_proto_scala", + deps = [":test2"], + plugin = "@io_bazel_rules_scala//src/scala/scripts:scalapb_plugin") + +scala_proto_gen( + name = "multi_src_proto_scala", + deps = [":test2", "//test/proto2:test"], + plugin = "@io_bazel_rules_scala//src/scala/scripts:scalapb_plugin") + +java_proto_library( + name = "test2_proto_java", + deps = [":test2", "//test/proto2:test"]) From 280f111bcc4730c9da4fff1eda97383942731f66 Mon Sep 17 00:00:00 2001 From: simuons Date: Wed, 27 Feb 2019 13:41:18 +0200 Subject: [PATCH 04/15] base scalapb_proto_library on scala_proto_gen --- scala_proto/scala_proto.bzl | 30 +++++++++++++++++++++++------- test/proto/BUILD | 3 ++- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/scala_proto/scala_proto.bzl b/scala_proto/scala_proto.bzl index 8a21e0cc7..cc43c7da1 100644 --- a/scala_proto/scala_proto.bzl +++ b/scala_proto/scala_proto.bzl @@ -562,6 +562,7 @@ def scalapb_proto_library( with_flat_package = False, with_single_line_to_string = False, scalac_jvm_flags = [], + java_conversions_deps = [], visibility = None): srcjar = name + "_srcjar" flags = [] @@ -573,17 +574,18 @@ def scalapb_proto_library( flags.append("flat_package") if with_single_line_to_string: flags.append("single_line_to_string") - scala_proto_srcjar( + + scala_proto_gen( name = srcjar, - flags = flags, - generator = "@io_bazel_rules_scala//src/scala/scripts:scalapb_generator", deps = deps, + flags = flags, + plugin = "@io_bazel_rules_scala//src/scala/scripts:scalapb_plugin", visibility = visibility, ) external_deps = list(SCALAPB_DEPS + GRPC_DEPS if ( with_grpc - ) else SCALAPB_DEPS) + ) else SCALAPB_DEPS) + java_conversions_deps scala_library( name = name, @@ -595,19 +597,33 @@ def scalapb_proto_library( visibility = visibility, ) +def _strip_roots(roots, f): + if f.is_source: + for prefix in roots: + if f.short_path.startswith(prefix + "/"): + return f.short_path.replace(prefix + "/", "") + return f.short_path + else: + for prefix in roots: + if f.path.startswith(f.root.path + "/" + prefix + "/"): + return f.path.replace(f.root.path + "/" + prefix + "/", "") + return f.short_path + def _scala_proto_gen_impl(ctx): - sources = [f for dep in ctx.attr.deps for f in dep.proto.direct_sources] descriptors = [f for dep in ctx.attr.deps for f in dep.proto.transitive_descriptor_sets] + roots = [f for dep in ctx.attr.deps for f in dep.proto.transitive_proto_path] + sources = depset([_strip_roots(roots, f) for dep in ctx.attr.deps for f in dep.proto.transitive_sources]).to_list() + srcdotjar = ctx.actions.declare_file("_" + ctx.label.name + "_src.jar") ctx.actions.run( - inputs = [ctx.executable._protoc, ctx.executable.plugin] + descriptors + sources, + inputs = [ctx.executable._protoc, ctx.executable.plugin] + descriptors, outputs = [srcdotjar], arguments = [ "--plugin=protoc-gen-scala=" + ctx.executable.plugin.path, "--scala_out=%s:%s" % (",".join(ctx.attr.flags), srcdotjar.path), "--descriptor_set_in=" + ":".join([descriptor.path for descriptor in descriptors])] - + [source.short_path for source in sources], + + sources, executable = ctx.executable._protoc, mnemonic = "ScalaProtoGen", use_default_shell_env = True, diff --git a/test/proto/BUILD b/test/proto/BUILD index e3266225d..cbd56b63a 100644 --- a/test/proto/BUILD +++ b/test/proto/BUILD @@ -85,9 +85,10 @@ scalapb_proto_library( with_java = True, deps = [ ":test2", - ":test_proto_java_lib", + "//test/proto2:test", ], + java_conversions_deps = [":test_proto_java_lib"] ) scalapb_proto_library( From fe4ceab902f663be49b9bce747296c93ffa85849 Mon Sep 17 00:00:00 2001 From: simuons Date: Wed, 27 Feb 2019 13:46:04 +0200 Subject: [PATCH 05/15] add blacklisted_protos support --- scala_proto/scala_proto.bzl | 2 +- test/proto/BUILD | 28 ++++------------------------ 2 files changed, 5 insertions(+), 25 deletions(-) diff --git a/scala_proto/scala_proto.bzl b/scala_proto/scala_proto.bzl index cc43c7da1..ffb2f241d 100644 --- a/scala_proto/scala_proto.bzl +++ b/scala_proto/scala_proto.bzl @@ -612,7 +612,7 @@ def _strip_roots(roots, f): def _scala_proto_gen_impl(ctx): descriptors = [f for dep in ctx.attr.deps for f in dep.proto.transitive_descriptor_sets] roots = [f for dep in ctx.attr.deps for f in dep.proto.transitive_proto_path] - sources = depset([_strip_roots(roots, f) for dep in ctx.attr.deps for f in dep.proto.transitive_sources]).to_list() + sources = depset([_strip_roots(roots, f) for dep in ctx.attr.deps for f in _retained_protos(dep.proto.transitive_sources, ctx.attr.blacklisted_protos)]).to_list() srcdotjar = ctx.actions.declare_file("_" + ctx.label.name + "_src.jar") diff --git a/test/proto/BUILD b/test/proto/BUILD index cbd56b63a..e8462a505 100644 --- a/test/proto/BUILD +++ b/test/proto/BUILD @@ -85,7 +85,6 @@ scalapb_proto_library( with_java = True, deps = [ ":test2", - "//test/proto2:test", ], java_conversions_deps = [":test_proto_java_lib"] @@ -98,16 +97,16 @@ scalapb_proto_library( deps = [":test_service"], ) -scala_proto_srcjar( +scala_proto_gen( name = "test1_proto_scala", deps = ["//test/proto2:test"], - generator = "@io_bazel_rules_scala//src/scala/scripts:scalapb_generator") + plugin = "@io_bazel_rules_scala//src/scala/scripts:scalapb_plugin") -scala_proto_srcjar( +scala_proto_gen( name = "test2_proto_scala_with_blacklisted_test1_proto_scala", deps = [":test2"], blacklisted_protos = ["//test/proto2:test"], - generator = "@io_bazel_rules_scala//src/scala/scripts:scalapb_generator") + plugin = "@io_bazel_rules_scala//src/scala/scripts:scalapb_plugin") scala_library( name = "lib_scala_should_fail_on_duplicated_sources_unless_duplicates_are_blacklisted", @@ -118,22 +117,3 @@ scala_library( "@com_google_protobuf//:protobuf_java", "@scala_proto_rules_scalapb_lenses", "@scala_proto_rules_scalapb_runtime"]) - -scala_proto_gen( - name = "a_proto_scala", - deps = ["//test/proto2:test"], - plugin = "@io_bazel_rules_scala//src/scala/scripts:scalapb_plugin") - -scala_proto_gen( - name = "test2_proto_scala", - deps = [":test2"], - plugin = "@io_bazel_rules_scala//src/scala/scripts:scalapb_plugin") - -scala_proto_gen( - name = "multi_src_proto_scala", - deps = [":test2", "//test/proto2:test"], - plugin = "@io_bazel_rules_scala//src/scala/scripts:scalapb_plugin") - -java_proto_library( - name = "test2_proto_java", - deps = [":test2", "//test/proto2:test"]) From aaa48899321e1d6618ffb63686bb9aef5fbe128e Mon Sep 17 00:00:00 2001 From: simuons Date: Wed, 27 Feb 2019 13:53:36 +0200 Subject: [PATCH 06/15] remove srcjar from scala_library deps in scalapb_proto_library because scala_proto_gen produces only sources --- scala_proto/scala_proto.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scala_proto/scala_proto.bzl b/scala_proto/scala_proto.bzl index ffb2f241d..863625390 100644 --- a/scala_proto/scala_proto.bzl +++ b/scala_proto/scala_proto.bzl @@ -590,8 +590,8 @@ def scalapb_proto_library( scala_library( name = name, srcs = [srcjar], - deps = [srcjar] + external_deps, - unused_dependency_checker_ignored_targets = [srcjar] + external_deps, + deps = external_deps, + unused_dependency_checker_ignored_targets = external_deps, exports = external_deps, scalac_jvm_flags = scalac_jvm_flags, visibility = visibility, From ac05d0bcbb1e4319ecb437c85c5e9219bb5641b8 Mon Sep 17 00:00:00 2001 From: simuons Date: Wed, 27 Feb 2019 15:28:20 +0200 Subject: [PATCH 07/15] Cleanup --- scala_proto/scala_proto.bzl | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/scala_proto/scala_proto.bzl b/scala_proto/scala_proto.bzl index 863625390..31fcf5e24 100644 --- a/scala_proto/scala_proto.bzl +++ b/scala_proto/scala_proto.bzl @@ -597,22 +597,18 @@ def scalapb_proto_library( visibility = visibility, ) -def _strip_roots(roots, f): - if f.is_source: - for prefix in roots: - if f.short_path.startswith(prefix + "/"): - return f.short_path.replace(prefix + "/", "") - return f.short_path - else: - for prefix in roots: - if f.path.startswith(f.root.path + "/" + prefix + "/"): - return f.path.replace(f.root.path + "/" + prefix + "/", "") - return f.short_path +def _strip_root(file, roots): + for root in roots: + prefix = root + "/" if file.is_source else file.root.path + "/" + root + "/" + if file.path.startswith(prefix): + return file.path.replace(prefix, "") + return file.short_path def _scala_proto_gen_impl(ctx): - descriptors = [f for dep in ctx.attr.deps for f in dep.proto.transitive_descriptor_sets] - roots = [f for dep in ctx.attr.deps for f in dep.proto.transitive_proto_path] - sources = depset([_strip_roots(roots, f) for dep in ctx.attr.deps for f in _retained_protos(dep.proto.transitive_sources, ctx.attr.blacklisted_protos)]).to_list() + descriptors = depset([f for dep in ctx.attr.deps for f in dep.proto.transitive_descriptor_sets]).to_list() + sources = depset([f for dep in ctx.attr.deps for f in dep.proto.transitive_sources]).to_list() + roots = depset([f for dep in ctx.attr.deps for f in dep.proto.transitive_proto_path]).to_list() + inputs = depset([_strip_root(f, roots) for f in _retained_protos(sources, ctx.attr.blacklisted_protos)]).to_list() srcdotjar = ctx.actions.declare_file("_" + ctx.label.name + "_src.jar") @@ -623,7 +619,7 @@ def _scala_proto_gen_impl(ctx): "--plugin=protoc-gen-scala=" + ctx.executable.plugin.path, "--scala_out=%s:%s" % (",".join(ctx.attr.flags), srcdotjar.path), "--descriptor_set_in=" + ":".join([descriptor.path for descriptor in descriptors])] - + sources, + + inputs, executable = ctx.executable._protoc, mnemonic = "ScalaProtoGen", use_default_shell_env = True, From ca97f8875891987a65f68854dcd5d82046cda2fc Mon Sep 17 00:00:00 2001 From: simuons Date: Thu, 28 Feb 2019 08:56:36 +0200 Subject: [PATCH 08/15] Naming --- scala_proto/scala_proto.bzl | 6 +++--- test/proto/BUILD | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scala_proto/scala_proto.bzl b/scala_proto/scala_proto.bzl index 31fcf5e24..85251d3ac 100644 --- a/scala_proto/scala_proto.bzl +++ b/scala_proto/scala_proto.bzl @@ -562,7 +562,7 @@ def scalapb_proto_library( with_flat_package = False, with_single_line_to_string = False, scalac_jvm_flags = [], - java_conversions_deps = [], + runtime = [], visibility = None): srcjar = name + "_srcjar" flags = [] @@ -583,9 +583,9 @@ def scalapb_proto_library( visibility = visibility, ) - external_deps = list(SCALAPB_DEPS + GRPC_DEPS if ( + external_deps = runtime + list(SCALAPB_DEPS + GRPC_DEPS if ( with_grpc - ) else SCALAPB_DEPS) + java_conversions_deps + ) else SCALAPB_DEPS) scala_library( name = name, diff --git a/test/proto/BUILD b/test/proto/BUILD index e8462a505..1cbe5ea8c 100644 --- a/test/proto/BUILD +++ b/test/proto/BUILD @@ -87,7 +87,7 @@ scalapb_proto_library( ":test2", "//test/proto2:test", ], - java_conversions_deps = [":test_proto_java_lib"] + runtime = [":test_proto_java_lib"] ) scalapb_proto_library( From b0c1ef1037110da4a9932d0ed0db319f843476ae Mon Sep 17 00:00:00 2001 From: simuons Date: Thu, 28 Feb 2019 09:33:52 +0200 Subject: [PATCH 09/15] Comment runtime arg --- scala_proto/scala_proto.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/scala_proto/scala_proto.bzl b/scala_proto/scala_proto.bzl index 85251d3ac..c991f0c06 100644 --- a/scala_proto/scala_proto.bzl +++ b/scala_proto/scala_proto.bzl @@ -548,6 +548,7 @@ Args: with_flat_package: When true, ScalaPB will not append the protofile base name to the package name with_single_line_to_string: Enables generation of toString() methods that use the single line format scalac_jvm_flags: List of JVM flags to pass to the underlying scala_library attribute + runtime: List of libraries that the generated code is compiled against (ScalaPB dependencies are added implicitly) Outputs: A scala_library rule that includes the generated scalapb bindings, as From 7beb38f6e132001b2d6bff00969b48c2a2a9f80b Mon Sep 17 00:00:00 2001 From: simuons Date: Thu, 28 Feb 2019 09:55:24 +0200 Subject: [PATCH 10/15] Add comments to scala_proto_gen --- scala_proto/scala_proto.bzl | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scala_proto/scala_proto.bzl b/scala_proto/scala_proto.bzl index c991f0c06..ee213f855 100644 --- a/scala_proto/scala_proto.bzl +++ b/scala_proto/scala_proto.bzl @@ -632,6 +632,25 @@ def _scala_proto_gen_impl(ctx): outputs = [ctx.outputs.srcjar], arguments = [srcdotjar.path, ctx.outputs.srcjar.path]) +"""Generates code with scala plugin passed to implicit @com_google_protobuf//:protoc + +Example: + scala_proto_gen( + name = "a_proto_scala", + deps = [":a_proto"], + plugin = "@io_bazel_rules_scala//src/scala/scripts:scalapb_plugin") + +Args: + deps: List of proto_library rules to generate code for + blacklisted_protos: List of proto_library rules to exclude from protoc inputs + (used for libraries that comes from runtime like any.proto) + flags: list of plugin flags passed to --scala_out + plugin: an executable passed to --plugin=protoc-gen-scala= which implements protoc plugin contract + https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/compiler/plugin.proto + +Outputs: + Single srcjar with generated sources for all deps and all the transitives +""" scala_proto_gen = rule( _scala_proto_gen_impl, attrs = { From 88bcafdd982846c24dc5ce90b05727374d9b51e5 Mon Sep 17 00:00:00 2001 From: simuons Date: Thu, 28 Feb 2019 10:19:45 +0200 Subject: [PATCH 11/15] Update readme --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ee1dfd2ad..85f58a357 100644 --- a/README.md +++ b/README.md @@ -545,7 +545,7 @@ Then you can import `scalapb_proto_library` in any BUILD file like this: ```python load("@io_bazel_rules_scala//scala_proto:scala_proto.bzl", "scalapb_proto_library") -scalapb_proto_library(name, deps, with_grpc, with_java, with_flat_package, with_single_line_to_string, scalac_jvm_flags) +scalapb_proto_library(name, deps, with_grpc, with_java, with_flat_package, with_single_line_to_string, scalac_jvm_flags, runtime) ``` `scalapb_proto_library` generates a scala library of scala proto bindings @@ -572,7 +572,7 @@ generated by the [ScalaPB compiler](https://github.com/scalapb/ScalaPB). deps

List of labels, required

-

List of dependencies for this target. Must either be of type proto_library or java_proto_library (allowed only if with_java is enabled)

+

List of proto_library dependencies for this target

@@ -610,6 +610,14 @@ generated by the [ScalaPB compiler](https://github.com/scalapb/ScalaPB).

List of JVM flags to pass to the underlying scala_library attribute

+ + runtime + +

List of labels; optional

+

List of *_library that the generated code is compiled against (ScalaPB dependencies are added implicitly)

+

If with_java is used then java_proto_library for which conversions are generated must be supplied

+ + From 86ec56987b9b278926180179473d6f45c289acee Mon Sep 17 00:00:00 2001 From: simuons Date: Thu, 28 Feb 2019 11:04:56 +0200 Subject: [PATCH 12/15] Rename scalapb_proto_library runtime arg to with_java_deps as it is retated to java_conversions only --- README.md | 5 ++--- scala_proto/scala_proto.bzl | 7 +++---- test/proto/BUILD | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 85f58a357..586633245 100644 --- a/README.md +++ b/README.md @@ -611,11 +611,10 @@ generated by the [ScalaPB compiler](https://github.com/scalapb/ScalaPB). - runtime + with_java_deps

List of labels; optional

-

List of *_library that the generated code is compiled against (ScalaPB dependencies are added implicitly)

-

If with_java is used then java_proto_library for which conversions are generated must be supplied

+

List of java_proto_library which are required for java_conversions with_java=True

diff --git a/scala_proto/scala_proto.bzl b/scala_proto/scala_proto.bzl index ee213f855..c998b7ee0 100644 --- a/scala_proto/scala_proto.bzl +++ b/scala_proto/scala_proto.bzl @@ -548,8 +548,7 @@ Args: with_flat_package: When true, ScalaPB will not append the protofile base name to the package name with_single_line_to_string: Enables generation of toString() methods that use the single line format scalac_jvm_flags: List of JVM flags to pass to the underlying scala_library attribute - runtime: List of libraries that the generated code is compiled against (ScalaPB dependencies are added implicitly) - + with_java_deps: List of java_proto_library which are required for java_conversions (with_java=True) Outputs: A scala_library rule that includes the generated scalapb bindings, as well as any library dependencies needed to compile and use these. @@ -563,7 +562,7 @@ def scalapb_proto_library( with_flat_package = False, with_single_line_to_string = False, scalac_jvm_flags = [], - runtime = [], + with_java_deps = [], visibility = None): srcjar = name + "_srcjar" flags = [] @@ -584,7 +583,7 @@ def scalapb_proto_library( visibility = visibility, ) - external_deps = runtime + list(SCALAPB_DEPS + GRPC_DEPS if ( + external_deps = with_java_deps + list(SCALAPB_DEPS + GRPC_DEPS if ( with_grpc ) else SCALAPB_DEPS) diff --git a/test/proto/BUILD b/test/proto/BUILD index 1cbe5ea8c..f7fb1da04 100644 --- a/test/proto/BUILD +++ b/test/proto/BUILD @@ -83,11 +83,11 @@ scalapb_proto_library( visibility = ["//visibility:public"], with_flat_package = True, with_java = True, + with_java_deps = [":test_proto_java_lib"], deps = [ ":test2", "//test/proto2:test", ], - runtime = [":test_proto_java_lib"] ) scalapb_proto_library( From c258a1a478a902ac94ed4648094faf0ec3a0a4ba Mon Sep 17 00:00:00 2001 From: simuons Date: Thu, 28 Feb 2019 15:33:45 +0200 Subject: [PATCH 13/15] Preserve scalapb_proto_library behaviour --- README.md | 11 ++------ scala_proto/scala_proto.bzl | 55 +++++++++++++++++++++++++++++++------ test/proto/BUILD | 2 +- 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 586633245..ee1dfd2ad 100644 --- a/README.md +++ b/README.md @@ -545,7 +545,7 @@ Then you can import `scalapb_proto_library` in any BUILD file like this: ```python load("@io_bazel_rules_scala//scala_proto:scala_proto.bzl", "scalapb_proto_library") -scalapb_proto_library(name, deps, with_grpc, with_java, with_flat_package, with_single_line_to_string, scalac_jvm_flags, runtime) +scalapb_proto_library(name, deps, with_grpc, with_java, with_flat_package, with_single_line_to_string, scalac_jvm_flags) ``` `scalapb_proto_library` generates a scala library of scala proto bindings @@ -572,7 +572,7 @@ generated by the [ScalaPB compiler](https://github.com/scalapb/ScalaPB). deps

List of labels, required

-

List of proto_library dependencies for this target

+

List of dependencies for this target. Must either be of type proto_library or java_proto_library (allowed only if with_java is enabled)

@@ -610,13 +610,6 @@ generated by the [ScalaPB compiler](https://github.com/scalapb/ScalaPB).

List of JVM flags to pass to the underlying scala_library attribute

- - with_java_deps - -

List of labels; optional

-

List of java_proto_library which are required for java_conversions with_java=True

- - diff --git a/scala_proto/scala_proto.bzl b/scala_proto/scala_proto.bzl index c998b7ee0..87d9fa451 100644 --- a/scala_proto/scala_proto.bzl +++ b/scala_proto/scala_proto.bzl @@ -548,7 +548,7 @@ Args: with_flat_package: When true, ScalaPB will not append the protofile base name to the package name with_single_line_to_string: Enables generation of toString() methods that use the single line format scalac_jvm_flags: List of JVM flags to pass to the underlying scala_library attribute - with_java_deps: List of java_proto_library which are required for java_conversions (with_java=True) + Outputs: A scala_library rule that includes the generated scalapb bindings, as well as any library dependencies needed to compile and use these. @@ -562,7 +562,6 @@ def scalapb_proto_library( with_flat_package = False, with_single_line_to_string = False, scalac_jvm_flags = [], - with_java_deps = [], visibility = None): srcjar = name + "_srcjar" flags = [] @@ -575,7 +574,7 @@ def scalapb_proto_library( if with_single_line_to_string: flags.append("single_line_to_string") - scala_proto_gen( + _scalapb_proto_gen_with_jvm_deps( name = srcjar, deps = deps, flags = flags, @@ -583,20 +582,57 @@ def scalapb_proto_library( visibility = visibility, ) - external_deps = with_java_deps + list(SCALAPB_DEPS + GRPC_DEPS if ( + external_deps = list(SCALAPB_DEPS + GRPC_DEPS if ( with_grpc ) else SCALAPB_DEPS) scala_library( name = name, srcs = [srcjar], - deps = external_deps, - unused_dependency_checker_ignored_targets = external_deps, + deps = [srcjar] + external_deps, + unused_dependency_checker_ignored_targets = [srcjar] + external_deps, exports = external_deps, scalac_jvm_flags = scalac_jvm_flags, visibility = visibility, ) +def _scalapb_proto_gen_with_jvm_deps_impl(ctx): + _scala_proto_gen_impl(ctx) + + jvm_deps = [p for p in ctx.attr.deps if hasattr(p, "proto") == False] + + if "java_conversions" in ctx.attr.flags and len(jvm_deps) == 0: + fail("must have at least one jvm dependency if with_java is True (java_conversions is turned on)") + + deps_jars = collect_jars(jvm_deps) + + srcjarsattr = struct(srcjar = ctx.outputs.srcjar) + scalaattr = struct( + outputs = None, + compile_jars = deps_jars.compile_jars, + transitive_runtime_jars = deps_jars.transitive_runtime_jars, + ) + java_provider = create_java_provider(scalaattr, depset()) + return struct( + scala = scalaattr, + providers = [java_provider], + srcjars = srcjarsattr, + ) + +_scalapb_proto_gen_with_jvm_deps = rule( + _scalapb_proto_gen_with_jvm_deps_impl, + attrs = { + "deps": attr.label_list(mandatory = True, providers = [["proto"], [JavaInfo]]), + "blacklisted_protos" : attr.label_list(providers = [["proto"]]), + "flags": attr.string_list(default = []), + "plugin": attr.label(executable = True, cfg = "host"), + "_protoc": attr.label(executable = True, cfg = "host", default = "@com_google_protobuf//:protoc") + }, + outputs = { + "srcjar": "lib%{name}.srcjar", + }, +) + def _strip_root(file, roots): for root in roots: prefix = root + "/" if file.is_source else file.root.path + "/" + root + "/" @@ -605,9 +641,10 @@ def _strip_root(file, roots): return file.short_path def _scala_proto_gen_impl(ctx): - descriptors = depset([f for dep in ctx.attr.deps for f in dep.proto.transitive_descriptor_sets]).to_list() - sources = depset([f for dep in ctx.attr.deps for f in dep.proto.transitive_sources]).to_list() - roots = depset([f for dep in ctx.attr.deps for f in dep.proto.transitive_proto_path]).to_list() + protos = [p for p in ctx.attr.deps if hasattr(p, "proto")] # because scalapb can pass JavaInfo as well + descriptors = depset([f for dep in protos for f in dep.proto.transitive_descriptor_sets]).to_list() + sources = depset([f for dep in protos for f in dep.proto.transitive_sources]).to_list() + roots = depset([f for dep in protos for f in dep.proto.transitive_proto_path]).to_list() inputs = depset([_strip_root(f, roots) for f in _retained_protos(sources, ctx.attr.blacklisted_protos)]).to_list() srcdotjar = ctx.actions.declare_file("_" + ctx.label.name + "_src.jar") diff --git a/test/proto/BUILD b/test/proto/BUILD index f7fb1da04..a823b3bc2 100644 --- a/test/proto/BUILD +++ b/test/proto/BUILD @@ -83,9 +83,9 @@ scalapb_proto_library( visibility = ["//visibility:public"], with_flat_package = True, with_java = True, - with_java_deps = [":test_proto_java_lib"], deps = [ ":test2", + ":test_proto_java_lib", "//test/proto2:test", ], ) From 5d8d7d35e716131c0a2d36e6ddb0bee6b37a5677 Mon Sep 17 00:00:00 2001 From: simuons Date: Thu, 28 Feb 2019 16:26:45 +0200 Subject: [PATCH 14/15] Try to remove some duplication --- scala_proto/scala_proto.bzl | 39 +++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/scala_proto/scala_proto.bzl b/scala_proto/scala_proto.bzl index 87d9fa451..c2f1ee7dc 100644 --- a/scala_proto/scala_proto.bzl +++ b/scala_proto/scala_proto.bzl @@ -596,6 +596,19 @@ def scalapb_proto_library( visibility = visibility, ) +def _scala_proto_gen_attrs(deps_providers): + return { + "deps": attr.label_list(mandatory = True, providers = deps_providers), + "blacklisted_protos" : attr.label_list(providers = [["proto"]]), + "flags": attr.string_list(default = []), + "plugin": attr.label(executable = True, cfg = "host"), + "_protoc": attr.label(executable = True, cfg = "host", default = "@com_google_protobuf//:protoc") + } + +_scala_proto_gen_outputs = { + "srcjar": "lib%{name}.srcjar", +} + def _scalapb_proto_gen_with_jvm_deps_impl(ctx): _scala_proto_gen_impl(ctx) @@ -621,16 +634,8 @@ def _scalapb_proto_gen_with_jvm_deps_impl(ctx): _scalapb_proto_gen_with_jvm_deps = rule( _scalapb_proto_gen_with_jvm_deps_impl, - attrs = { - "deps": attr.label_list(mandatory = True, providers = [["proto"], [JavaInfo]]), - "blacklisted_protos" : attr.label_list(providers = [["proto"]]), - "flags": attr.string_list(default = []), - "plugin": attr.label(executable = True, cfg = "host"), - "_protoc": attr.label(executable = True, cfg = "host", default = "@com_google_protobuf//:protoc") - }, - outputs = { - "srcjar": "lib%{name}.srcjar", - }, + attrs = _scala_proto_gen_attrs([["proto"], [JavaInfo]]), + outputs = _scala_proto_gen_outputs, ) def _strip_root(file, roots): @@ -641,7 +646,7 @@ def _strip_root(file, roots): return file.short_path def _scala_proto_gen_impl(ctx): - protos = [p for p in ctx.attr.deps if hasattr(p, "proto")] # because scalapb can pass JavaInfo as well + protos = [p for p in ctx.attr.deps if hasattr(p, "proto")] # because scalapb passes JavaInfo as well descriptors = depset([f for dep in protos for f in dep.proto.transitive_descriptor_sets]).to_list() sources = depset([f for dep in protos for f in dep.proto.transitive_sources]).to_list() roots = depset([f for dep in protos for f in dep.proto.transitive_proto_path]).to_list() @@ -689,14 +694,6 @@ Outputs: """ scala_proto_gen = rule( _scala_proto_gen_impl, - attrs = { - "deps": attr.label_list(mandatory = True, providers = [["proto"]]), - "blacklisted_protos" : attr.label_list(providers = [["proto"]]), - "flags": attr.string_list(default = []), - "plugin": attr.label(executable = True, cfg = "host"), - "_protoc": attr.label(executable = True, cfg = "host", default = "@com_google_protobuf//:protoc") - }, - outputs = { - "srcjar": "lib%{name}.srcjar", - }, + attrs = _scala_proto_gen_attrs(deps_providers = [["proto"]]), + outputs = _scala_proto_gen_outputs, ) From b1aa1d541565d63437adfbca36af2a88d9618343 Mon Sep 17 00:00:00 2001 From: simuons Date: Fri, 1 Mar 2019 08:48:36 +0200 Subject: [PATCH 15/15] Comments --- scala_proto/scala_proto.bzl | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scala_proto/scala_proto.bzl b/scala_proto/scala_proto.bzl index c2f1ee7dc..a6a9bab90 100644 --- a/scala_proto/scala_proto.bzl +++ b/scala_proto/scala_proto.bzl @@ -482,6 +482,9 @@ def _gen_proto_srcjar_impl(ctx): srcjars = srcjarsattr, ) +""" + Deprecated: use scala_proto_gen instead +""" scala_proto_srcjar = rule( _gen_proto_srcjar_impl, attrs = { @@ -610,13 +613,13 @@ _scala_proto_gen_outputs = { } def _scalapb_proto_gen_with_jvm_deps_impl(ctx): - _scala_proto_gen_impl(ctx) - jvm_deps = [p for p in ctx.attr.deps if hasattr(p, "proto") == False] if "java_conversions" in ctx.attr.flags and len(jvm_deps) == 0: fail("must have at least one jvm dependency if with_java is True (java_conversions is turned on)") + _scala_proto_gen_impl(ctx) + deps_jars = collect_jars(jvm_deps) srcjarsattr = struct(srcjar = ctx.outputs.srcjar) @@ -639,6 +642,10 @@ _scalapb_proto_gen_with_jvm_deps = rule( ) def _strip_root(file, roots): + """Strip first matching root which comes from proto_library(proto_source_root) + It assumes that proto_source_root are unique. + It should go away once generation is moved to aspects and roots can be handled for each proto_library individualy. + """ for root in roots: prefix = root + "/" if file.is_source else file.root.path + "/" + root + "/" if file.path.startswith(prefix): @@ -646,7 +653,7 @@ def _strip_root(file, roots): return file.short_path def _scala_proto_gen_impl(ctx): - protos = [p for p in ctx.attr.deps if hasattr(p, "proto")] # because scalapb passes JavaInfo as well + protos = [p for p in ctx.attr.deps if hasattr(p, "proto")] # because scalapb_proto_library passes JavaInfo as well descriptors = depset([f for dep in protos for f in dep.proto.transitive_descriptor_sets]).to_list() sources = depset([f for dep in protos for f in dep.proto.transitive_sources]).to_list() roots = depset([f for dep in protos for f in dep.proto.transitive_proto_path]).to_list()