diff --git a/scala/private/common.bzl b/scala/private/common.bzl index a97765579..e32a93f6d 100644 --- a/scala/private/common.bzl +++ b/scala/private/common.bzl @@ -1,10 +1,14 @@ def write_manifest(ctx): + main_class = getattr(ctx.attr, "main_class", None) + write_manifest_file(ctx.actions, ctx.outputs.manifest, main_class) + +def write_manifest_file(actions, output_file, main_class): # TODO(bazel-team): I don't think this classpath is what you want manifest = "Class-Path: \n" - if getattr(ctx.attr, "main_class", ""): - manifest += "Main-Class: %s\n" % ctx.attr.main_class + if main_class: + manifest += "Main-Class: %s\n" % main_class - ctx.actions.write(output = ctx.outputs.manifest, content = manifest) + actions.write(output = output_file, content = manifest) def collect_srcjars(targets): srcjars = [] diff --git a/scala/private/rule_impls.bzl b/scala/private/rule_impls.bzl index bb1642d9d..57be73ba9 100644 --- a/scala/private/rule_impls.bzl +++ b/scala/private/rule_impls.bzl @@ -545,28 +545,9 @@ def _lib(ctx, non_macro_lib): scala = scalaattr, providers = [java_provider], runfiles = runfiles, - # This is a free monoid given to the graph for the purpose of - # extensibility. This is necessary when one wants to create - # new targets which want to leverage a scala_library. For example, - # new_target1 -> scala_library -> new_target2. There might be - # information that new_target2 needs to get from new_target1, - # but we do not want to have to change scala_library to pass - # this information through. extra_information allows passing - # this information through, and it is up to the new_targets - # to filter and make sense of this information. - # unfortunately, we need to see this for scrooge and protobuf to work, - # but those are generating srcjar, so they should really come in via srcs - extra_information=_collect_extra_information(ctx.attr.deps + ctx.attr.srcs), jars_to_labels = jars.jars2labels, ) -def _collect_extra_information(targets): - r = [] - for target in targets: - if hasattr(target, "extra_information"): - r.extend(target.extra_information) - return r - def scala_library_impl(ctx): return _lib(ctx, True) diff --git a/src/scala/scripts/TwitterScroogeGenerator.scala b/src/scala/scripts/TwitterScroogeGenerator.scala index f2b53cd65..5b1f01a30 100644 --- a/src/scala/scripts/TwitterScroogeGenerator.scala +++ b/src/scala/scripts/TwitterScroogeGenerator.scala @@ -97,7 +97,7 @@ class ScroogeGenerator extends Processor { val intersect = allFilesInZips(onlyTransitiveThriftSrcJars) .intersect(immediateThriftSrcs) - if (intersect.nonEmpty) + if (intersect.iterator.filter(_.endsWith(".thrift")).nonEmpty) sys.error("onlyTransitiveThriftSrcs and immediateThriftSrcs should " + s"have not intersection, found: ${intersect.mkString(",")}") diff --git a/test/src/main/scala/scalarules/test/twitter_scrooge/BUILD b/test/src/main/scala/scalarules/test/twitter_scrooge/BUILD index 4544c2200..035430847 100644 --- a/test/src/main/scala/scalarules/test/twitter_scrooge/BUILD +++ b/test/src/main/scala/scalarules/test/twitter_scrooge/BUILD @@ -1,22 +1,25 @@ load("//scala:scala.bzl", "scala_binary", "scala_library") load("//twitter_scrooge:twitter_scrooge.bzl", "scrooge_scala_library") +load("//thrift:thrift.bzl", "thrift_library") scrooge_scala_library( name = "scrooge1", visibility = ["//visibility:public"], - deps = [ + exports = [ ":scrooge2_a", ":scrooge2_b", ":scrooge3", - "//test/src/main/scala/scalarules/test/twitter_scrooge/thrift", ], + deps = ["//test/src/main/scala/scalarules/test/twitter_scrooge/thrift"], ) scrooge_scala_library( name = "scrooge2_a", visibility = ["//visibility:public"], - deps = [ + exports = [ ":scrooge3", + ], + deps = [ "//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2:thrift2_a", ], ) @@ -24,12 +27,21 @@ scrooge_scala_library( scrooge_scala_library( name = "scrooge2_b", visibility = ["//visibility:public"], - deps = [ + exports = [ ":scrooge3", + ], + deps = [ "//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2:thrift2_b", ], ) +scrooge_scala_library( + name = "scrooge2_b_imp", + deps = [ + "//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2:thrift2_b_imp", + ], +) + scrooge_scala_library( name = "scrooge3", visibility = ["//visibility:public"], @@ -39,8 +51,10 @@ scrooge_scala_library( scrooge_scala_library( name = "scrooge2", visibility = ["//visibility:public"], - deps = [ + exports = [ ":scrooge3", + ], + deps = [ "//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2:thrift2_a", "//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2:thrift2_b", ], @@ -49,8 +63,10 @@ scrooge_scala_library( scrooge_scala_library( name = "scrooge4", visibility = ["//visibility:public"], - deps = [ + exports = [ ":scrooge2", + ], + deps = [ "//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/thrift4", ], ) @@ -58,8 +74,10 @@ scrooge_scala_library( scrooge_scala_library( name = "scrooge4a", visibility = ["//visibility:public"], - deps = [ + exports = [ ":scrooge4", + ], + deps = [ "//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/thrift4:thrift4a", ], ) @@ -83,21 +101,12 @@ scrooge_scala_library( scrooge_scala_library( name = "bare_thrift_scrooge", visibility = ["//visibility:public"], - deps = [ + exports = [ ":scroogebarejar1", ":scroogebarejar2", - "//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts", ], -) - -scrooge_scala_library( - name = "thrift_with_remote_jar", - remote_jars = [ - "//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts:barejar_java_import", - ], - visibility = ["//visibility:public"], deps = [ - "//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts/bare_jar_2", + "//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts", ], ) @@ -127,6 +136,14 @@ scala_library( deps = [":scrooge3"], ) +scala_library( + name = "justscrooge3_import", + srcs = ["JustScrooge3.scala"], + deps = [ + "//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/thrift3:thrift3_import", + ], +) + scala_library( name = "scrooge2_both", srcs = ["Scrooge2.scala"], diff --git a/test/src/main/scala/scalarules/test/twitter_scrooge/prefix_test/e/f/b/c/d/BUILD b/test/src/main/scala/scalarules/test/twitter_scrooge/prefix_test/e/f/b/c/d/BUILD index bbc8c1937..311e2c649 100644 --- a/test/src/main/scala/scalarules/test/twitter_scrooge/prefix_test/e/f/b/c/d/BUILD +++ b/test/src/main/scala/scalarules/test/twitter_scrooge/prefix_test/e/f/b/c/d/BUILD @@ -15,8 +15,10 @@ thrift_library( scrooge_scala_library( name = "d", visibility = ["//visibility:public"], + exports = [ + "//test/src/main/scala/scalarules/test/twitter_scrooge/prefix_test/a/b/c/d", + ], deps = [ ":b_thrift", - "//test/src/main/scala/scalarules/test/twitter_scrooge/prefix_test/a/b/c/d", ], ) diff --git a/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts/BUILD b/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts/BUILD index 222344c73..7ba56931a 100644 --- a/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts/BUILD +++ b/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts/BUILD @@ -1,20 +1,16 @@ load("//thrift:thrift.bzl", "thrift_library") -filegroup( - name = "barejar", - srcs = ["bare-thrift.jar"], - visibility = ["//visibility:public"], -) - java_import( - name = "barejar_java_import", + name = "barejar", jars = ["bare-thrift.jar"], visibility = ["//visibility:public"], ) thrift_library( name = "bare_jar_thrifts", - external_jars = [":barejar"], + external_jars = [ + ":barejar", + ], visibility = ["//visibility:public"], deps = [ "//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts/bare_jar_1", diff --git a/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts/bare_jar_1/BUILD b/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts/bare_jar_1/BUILD index e9a523805..53dd5f7b2 100644 --- a/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts/bare_jar_1/BUILD +++ b/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts/bare_jar_1/BUILD @@ -1,13 +1,9 @@ load("//thrift:thrift.bzl", "thrift_library") -filegroup( - name = "barejar1", - srcs = ["bare-thrift-1.jar"], - visibility = ["//visibility:public"], -) - thrift_library( name = "bare_jar_1", - external_jars = [":barejar1"], + external_jars = [ + "bare-thrift-1.jar", + ], visibility = ["//visibility:public"], ) diff --git a/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts/bare_jar_2/BUILD b/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts/bare_jar_2/BUILD index 5d794d139..5a461f36d 100644 --- a/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts/bare_jar_2/BUILD +++ b/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/bare_jar_thrifts/bare_jar_2/BUILD @@ -1,13 +1,9 @@ load("//thrift:thrift.bzl", "thrift_library") -filegroup( - name = "barejar2", - srcs = ["bare-thrift-2.jar"], - visibility = ["//visibility:public"], -) - thrift_library( name = "bare_jar_2", - external_jars = [":barejar2"], + external_jars = [ + "bare-thrift-2.jar", + ], visibility = ["//visibility:public"], ) diff --git a/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/BUILD b/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/BUILD index 59aab526f..5fd9913aa 100644 --- a/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/BUILD +++ b/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/BUILD @@ -13,3 +13,10 @@ thrift_library( visibility = ["//visibility:public"], deps = ["//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/thrift3"], ) + +thrift_library( + name = "thrift2_b_imp", + srcs = ["Thrift2_B.thrift"], + visibility = ["//visibility:public"], + deps = ["//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/thrift3:thrift3_import"], +) diff --git a/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/thrift3/BUILD b/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/thrift3/BUILD index b117f4dce..a474c3e50 100644 --- a/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/thrift3/BUILD +++ b/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/thrift3/BUILD @@ -1,7 +1,15 @@ load("//thrift:thrift.bzl", "thrift_library") +load("//twitter_scrooge:twitter_scrooge.bzl", "scrooge_scala_import") thrift_library( name = "thrift3", srcs = ["Thrift3.thrift"], visibility = ["//visibility:public"], ) + +scrooge_scala_import( + name = "thrift3_import", + scala_jars = ["thrift3_scrooge.jar"], + thrift_jars = ["libthrift3.jar"], + visibility = ["//visibility:public"], +) diff --git a/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/thrift3/thrift3_scrooge.jar b/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/thrift3/thrift3_scrooge.jar new file mode 100755 index 000000000..0e0f1c989 Binary files /dev/null and b/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/thrift3/thrift3_scrooge.jar differ diff --git a/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/thrift4/BUILD b/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/thrift4/BUILD index 7eb9a0e16..3b987ef4a 100644 --- a/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/thrift4/BUILD +++ b/test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2/thrift4/BUILD @@ -4,11 +4,17 @@ thrift_library( name = "thrift4", srcs = ["Thrift4.thrift"], visibility = ["//visibility:public"], + deps = [ + "//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2:thrift2_a", + ], ) thrift_library( name = "thrift4a", srcs = ["Thrift4a.thrift"], visibility = ["//visibility:public"], - deps = [":thrift4"], + deps = [ + ":thrift4", + "//test/src/main/scala/scalarules/test/twitter_scrooge/thrift/thrift2:thrift2_a", + ], ) diff --git a/thrift/thrift.bzl b/thrift/thrift.bzl index af4275bfd..ed8a4b59d 100644 --- a/thrift/thrift.bzl +++ b/thrift/thrift.bzl @@ -1,11 +1,14 @@ """Rules for organizing thrift files.""" -ThriftInfo = provider(fields = [ - "srcs", # The source files in this rule - "transitive_srcs", # the transitive version of the above - "external_jars", # external jars of thrift files - "transitive_external_jars" # transitive version of the above -]) +load("@io_bazel_rules_scala//thrift:thrift_info.bzl", "ThriftInfo") + +def empty_thrift_info(): + return ThriftInfo(srcs = depset(), transitive_srcs = depset()) + +def merge_thrift_infos(tis): + return ThriftInfo( + srcs = depset(transitive = [t.srcs for t in tis]), + transitive_srcs = depset(transitive = [t.transitive_srcs for t in tis])) def _common_prefix(strings): pref = None @@ -30,8 +33,6 @@ def _thrift_library_impl(ctx): ] src_paths = [f.path for f in ctx.files.srcs] - if len(src_paths) <= 0 and len(ctx.attr.external_jars) <= 0: - fail("we require at least one thrift file in a target") zipper_args = "\n".join(src_paths) + "\n" if len(prefixes) > 0: @@ -56,11 +57,17 @@ def _thrift_library_impl(ctx): zipper_args = "\n".join( ["%s=%s" % (src[endpos + 1:], src) for src in src_paths]) + "\n" + # external jars are references to things srcs may depend on + # ARE built as part of this target, but are not combined + # into the output jar. They are included in the ThriftInfo provider + externals = [] + for f in ctx.attr.external_jars: + externals.extend(f.files.to_list()) + if len(src_paths) > 0: zipper_arg_path = ctx.actions.declare_file( "%s_zipper_args" % ctx.outputs.libarchive.path) ctx.actions.write(zipper_arg_path, zipper_args) - _valid_thrift_deps(ctx.attr.deps) # We move the files and touch them so that the output file is a purely deterministic # product of the _content_ of the inputs cmd = """ @@ -79,6 +86,7 @@ rm -f {out} progress_message = "making thrift archive %s (%s files)" % (ctx.label, len(src_paths)), ) + srcs_depset = depset([ctx.outputs.libarchive] + externals) else: # we still have to create the output we declared ctx.actions.run_shell( @@ -93,41 +101,22 @@ rm {out}.contents zipper = ctx.executable._zipper.path), progress_message = "making empty thrift archive %s" % ctx.label, ) + srcs_depset = depset(externals) transitive_srcs = depset( - [ctx.outputs.libarchive], - transitive = _collect_thrift_srcs(ctx.attr.deps)) - jarfiles = _collect_thrift_external_jars(ctx.attr.deps) - for jar in ctx.attr.external_jars: - jarfiles.append(depset(jar.files)) - transitive_external_jars = depset(transitive = jarfiles) + transitive = _collect_thrift_srcs(ctx.attr.deps, srcs_depset)) - return [ - ThriftInfo( - srcs = ctx.outputs.libarchive, - transitive_srcs = transitive_srcs, - external_jars = ctx.attr.external_jars, - transitive_external_jars = transitive_external_jars, - ) - ] + return [ThriftInfo( + srcs = srcs_depset, + transitive_srcs = transitive_srcs, + )] -def _collect_thrift_srcs(targets): - ds = [] +def _collect_thrift_srcs(targets, init): + ds = [init] for target in targets: ds.append(target[ThriftInfo].transitive_srcs) return ds -def _collect_thrift_external_jars(targets): - ds = [] - for target in targets: - ds.append(target[ThriftInfo].transitive_external_jars) - return ds - -def _valid_thrift_deps(targets): - for target in targets: - if not ThriftInfo in target: - fail("thrift_library can only depend on thrift_library", target) - # Some notes on the raison d'etre of thrift_library vs. code gen specific # targets. The idea is to be able to separate concerns -- thrift_library is # concerned purely with the ownership and organization of thrift files. It @@ -140,7 +129,7 @@ thrift_library = rule( implementation = _thrift_library_impl, attrs = { "srcs": attr.label_list(allow_files = [".thrift"]), - "deps": attr.label_list(), + "deps": attr.label_list(providers = [ThriftInfo]), #TODO this is not necessarily the best way to do this... the goal # is that we want thrifts to be able to be imported via an absolute # path. But the thrift files have no clue what part of their path @@ -157,8 +146,9 @@ thrift_library = rule( # created by this is created in such a way that absolute imports work... "absolute_prefix": attr.string(default = '', mandatory = False), "absolute_prefixes": attr.string_list(), - # This is a list of JARs which only contain Thrift files - "external_jars": attr.label_list(), + # This is a list of JARs which only Thrift files + # these files WILL be compiled as part of the current target + "external_jars": attr.label_list(allow_files = [".jar"]), "_zipper": attr.label( executable = True, cfg = "host", @@ -166,4 +156,5 @@ thrift_library = rule( allow_files = True) }, outputs = {"libarchive": "lib%{name}.jar"}, + provides = [ThriftInfo], ) diff --git a/thrift/thrift_info.bzl b/thrift/thrift_info.bzl new file mode 100644 index 000000000..3602614f5 --- /dev/null +++ b/thrift/thrift_info.bzl @@ -0,0 +1,4 @@ +ThriftInfo = provider(fields = [ + "srcs", # The source files in this rule + "transitive_srcs", # the transitive version of the above +]) diff --git a/twitter_scrooge/twitter_scrooge.bzl b/twitter_scrooge/twitter_scrooge.bzl index 7aaa19d78..e582be7c9 100644 --- a/twitter_scrooge/twitter_scrooge.bzl +++ b/twitter_scrooge/twitter_scrooge.bzl @@ -8,10 +8,15 @@ load( "scala_mvn_artifact", ) -load("//scala/private:common.bzl", "write_manifest", "collect_srcjars", +load("//scala/private:common.bzl", "write_manifest_file", "collect_srcjars", "collect_jars") -load("//thrift:thrift.bzl", "ThriftInfo") +load("//scala/private:rule_impls.bzl", "compile_scala") + +load("@io_bazel_rules_scala//thrift:thrift_info.bzl", "ThriftInfo") + +load("@io_bazel_rules_scala//thrift:thrift.bzl", "merge_thrift_infos", + "empty_thrift_info") _jar_extension = ".jar" @@ -72,139 +77,57 @@ def twitter_scrooge(): name = 'io_bazel_rules_scala/dependency/thrift/util_logging', actual = '@util_logging//jar') -def _collect_transitive_srcs(targets): - r = [] - for target in targets: - if ThriftInfo in target: - r.append(target[ThriftInfo].transitive_srcs) - return depset(transitive = r) - -def _collect_owned_srcs(targets): - r = [] - for _target in targets: - if hasattr(_target, "extra_information"): - for target in _target.extra_information: - if hasattr(target, "scrooge_srcjar"): - r.append(target.scrooge_srcjar.transitive_owned_srcs) - return depset(transitive = r) - -def _collect_external_jars(targets): - r = [] - for target in targets: - if ThriftInfo in target: - thrift = target[ThriftInfo] - for jar in thrift.external_jars: - r.extend([f for f in jar.files if f.basename.endswith(_jar_extension)]) - r.extend([ - f for f in thrift.transitive_external_jars - if f.basename.endswith(_jar_extension) - ]) - return depset(r) - -def collect_extra_srcjars(targets): - srcjar = [] - srcjars = [] - for target in targets: - if hasattr(target, "extra_information"): - for _target in target.extra_information: - srcjar.append(_target.srcjars.srcjar) - srcjars.append(_target.srcjars.transitive_srcjars) - return depset(srcjar, transitive = srcjars) - -def _collect_immediate_srcs(targets): - srcs = [] - for target in targets: - if ThriftInfo in target: - srcs.append(target[ThriftInfo].srcs) - return depset(srcs) - -def _assert_set_is_subset(want, have): - missing = [] - for e in want: - if e not in have: - missing.append(e) - if len(missing) > 0: - fail( - 'scrooge_srcjar target must depend on scrooge_srcjar targets sufficient to ' - + 'cover the transitive graph of thrift files. Uncovered sources: ' + - str(missing)) - def _colon_paths(data): return ':'.join([f.path for f in sorted(data)]) -def _list_to_map(items): - map_result = {} - for item in items: - map_result[item] = None - return map_result - -def _gen_scrooge_srcjar_impl(ctx): - remote_jars = [] - for target in ctx.attr.remote_jars: - remote_jars.append( - depset( - [f for f in target.files if f.basename.endswith(_jar_extension)])) - - # deduplicate these - remote_jars = depset(transitive = remote_jars).to_list() - - # These are JARs that are declared externally and only have Thrift files - # in them. - external_jars = _collect_external_jars(ctx.attr.deps).to_list() - - # These are the thrift sources whose generated code we will "own" as a target - immediate_thrift_srcs = _collect_immediate_srcs(ctx.attr.deps).to_list() - - # This is the set of sources which is covered by any scala_library - # or scala_scrooge_gen targets that are depended on by this. This is - # necessary as we only compile the sources we own, and rely on other - # targets compiling the rest (for the benefit of caching and correctness). - transitive_owned_srcs = _collect_owned_srcs(ctx.attr.deps) - - # These are the thrift sources in the dependency graph. They are necessary - # to generate the code, but are not "owned" by this target and will not - # be in the resultant source jar - transitive_thrift_srcs = depset(transitive = [ - transitive_owned_srcs, - _collect_transitive_srcs(ctx.attr.deps) - ]).to_list() - - only_transitive_thrift_srcs = [] - for src in transitive_thrift_srcs: - if src not in _list_to_map(immediate_thrift_srcs): - only_transitive_thrift_srcs.append(src) - - # We want to ensure that the thrift sources which we do not own (but need - # in order to generate code) have targets which will compile them. - _assert_set_is_subset( - _list_to_map(only_transitive_thrift_srcs), - _list_to_map(transitive_owned_srcs.to_list())) - +ScroogeAspectInfo = provider(fields = [ + "thrift_info", + "src_jars", + "output_files", + "java_info", +]) + +ScroogeInfo = provider(fields = [ + "aspect_info", +]) + +ScroogeImport = provider(fields = [ + "java_info", + "thrift_info", +]) + +def merge_scrooge_aspect_info(scrooges): + return ScroogeAspectInfo( + src_jars = depset(transitive = [s.src_jars for s in scrooges]), + output_files = depset(transitive = [s.output_files for s in scrooges]), + thrift_info = merge_thrift_infos([s.thrift_info for s in scrooges]), + java_info = java_common.merge([s.java_info for s in scrooges])) + +def _compile_to_scala(ctx, label, compile_thrifts, include_thrifts, jar_output): # bazel worker arguments cannot be empty so we pad to ensure non-empty # and drop it off on the other side # https://github.com/bazelbuild/bazel/issues/3329 worker_arg_pad = "_" path_content = "\n".join([ - worker_arg_pad + _colon_paths(ps) for ps in [ - immediate_thrift_srcs, only_transitive_thrift_srcs, remote_jars, - external_jars - ] + worker_arg_pad + _colon_paths(ps) + for ps in [compile_thrifts, include_thrifts, [], []] ]) worker_content = "{output}\n{paths}\n{flags}".format( - output = ctx.outputs.srcjar.path, + output = jar_output.path, paths = path_content, flags = worker_arg_pad + ':'.join([ - '--with-finagle' if ctx.attr.with_finagle else '', + # always add finagle option which is a no-op if there are no services + # we could put "include_services" on thrift_info, if needed + '--with-finagle', ])) argfile = ctx.actions.declare_file( - "%s_worker_input" % ctx.label.name, sibling = ctx.outputs.srcjar) + "%s_worker_input" % label.name, sibling = jar_output) ctx.actions.write(output = argfile, content = worker_content) ctx.actions.run( executable = ctx.executable._pluck_scrooge_scala, - inputs = remote_jars + only_transitive_thrift_srcs + external_jars + - immediate_thrift_srcs + [argfile], - outputs = [ctx.outputs.srcjar], + inputs = compile_thrifts + include_thrifts + [argfile], + outputs = [jar_output], mnemonic = "ScroogeRule", progress_message = "creating scrooge files %s" % ctx.label, execution_requirements = {"supports-workers": "1"}, @@ -216,96 +139,214 @@ def _gen_scrooge_srcjar_impl(ctx): # In either case (worker or not), they will be jvm flags which will # be correctly handled since the executable is a jvm app that will # consume the flags on startup. - arguments = ["--jvm_flag=%s" % flag for flag in ctx.attr.jvm_flags] + - ["@" + argfile.path], - ) - - deps_jars = collect_jars(ctx.attr.deps) - - scalaattr = struct( - outputs = None, - compile_jars = deps_jars.compile_jars, - transitive_runtime_jars = deps_jars.transitive_runtime_jars, - ) - - transitive_srcjars = depset(transitive = [ - collect_srcjars(ctx.attr.deps), - collect_extra_srcjars(ctx.attr.deps) - ]) - - srcjarsattr = struct( - srcjar = ctx.outputs.srcjar, - transitive_srcjars = transitive_srcjars, - ) - - return struct( - scala = scalaattr, - srcjars = srcjarsattr, - extra_information = [ - struct( - srcjars = srcjarsattr, - scrooge_srcjar = struct( - transitive_owned_srcs = depset( - immediate_thrift_srcs, - transitive = [transitive_owned_srcs])), - ) - ], + #arguments = ["--jvm_flag=%s" % flag for flag in ctx.attr.jvm_flags] + + arguments = ["@" + argfile.path], ) -scrooge_scala_srcjar = rule( - _gen_scrooge_srcjar_impl, +def _compiled_jar_file(actions, scrooge_jar): + scrooge_jar_name = scrooge_jar.basename + # ends with .srcjar, so remove last 6 characters + without_suffix = scrooge_jar_name[0:len(scrooge_jar_name) - 6] + # this already ends with _scrooge because that is how scrooge_jar is named + compiled_jar = without_suffix + "jar" + return actions.declare_file(compiled_jar, sibling = scrooge_jar) + +def _compile_scala(ctx, label, output, scrooge_jar, deps_java_info, + implicit_deps): + + manifest = ctx.actions.declare_file( + label.name + "_MANIFEST.MF", sibling = scrooge_jar) + write_manifest_file(ctx.actions, manifest, None) + statsfile = ctx.actions.declare_file( + label.name + "_scalac.statsfile", sibling = scrooge_jar) + merged_deps = java_common.merge(deps_java_info + implicit_deps) + + # this only compiles scala, not the ijar, but we don't + # want the ijar for generated code anyway: any change + # in the thrift generally will change the interface and + # method bodies + compile_scala( + ctx, + label, + output, + manifest, + statsfile, + sources = [], + cjars = merged_deps.transitive_compile_time_jars, + all_srcjars = depset([scrooge_jar]), + transitive_compile_jars = merged_deps.transitive_compile_time_jars, + plugins = [], + resource_strip_prefix = "", + resources = [], + resource_jars = [], + labels = {}, + in_scalacopts = [], + print_compile_time = False, + expect_java_output = False, + scalac_jvm_flags = []) + + return java_common.create_provider( + use_ijar = False, + source_jars = [scrooge_jar], + compile_time_jars = depset( + [output], transitive = [merged_deps.compile_jars]), + transitive_compile_time_jars = depset( + [output], transitive = [merged_deps.transitive_compile_time_jars]), + transitive_runtime_jars = depset( + [output], transitive = [merged_deps.transitive_runtime_jars])) + +def _empty_java_info(deps_java_info, implicit_deps): + merged_deps = java_common.merge(deps_java_info + implicit_deps) + return java_common.create_provider( + use_ijar = False, + compile_time_jars = depset(transitive = [merged_deps.compile_jars]), + transitive_compile_time_jars = depset( + transitive = [merged_deps.transitive_compile_time_jars]), + transitive_runtime_jars = depset( + transitive = [merged_deps.transitive_runtime_jars])) + +#### +# This is applied to the DAG of thrift_librarys reachable from a deps +# or a scrooge_scala_library. Each thrift_library will be one scrooge +# invocation assuming it has some sources. +def _scrooge_aspect_impl(target, ctx): + if ScroogeImport in target: + target_import = target[ScroogeImport] + target_ti = target_import.thrift_info + deps = [target_import.java_info] + transitive_ti = target_ti + else: + target_ti = target[ThriftInfo] + deps = [d[ScroogeAspectInfo].java_info for d in ctx.rule.attr.deps] + transitive_ti = merge_thrift_infos( + [d[ScroogeAspectInfo].thrift_info + for d in ctx.rule.attr.deps] + [target_ti]) + + # we sort so the inputs are always the same for caching + compile_thrifts = sorted(target_ti.srcs.to_list()) + imps = [j[JavaInfo] for j in ctx.attr._implicit_compile_deps] + if compile_thrifts: + # we sort so the inputs are always the same for caching + compile_thrift_map = {} + for ct in compile_thrifts: + compile_thrift_map[ct] = True + include_thrifts = sorted([ + trans for trans in transitive_ti.transitive_srcs.to_list() + if trans not in compile_thrift_map + ]) + scrooge_file = ctx.actions.declare_file( + target.label.name + "_scrooge.srcjar") + _compile_to_scala(ctx, target.label, compile_thrifts, include_thrifts, + scrooge_file) + + src_jars = depset([scrooge_file]) + output = _compiled_jar_file(ctx.actions, scrooge_file) + outs = depset([output]) + java_info = _compile_scala(ctx, target.label, output, scrooge_file, deps, + imps) + + else: + # this target is only an aggregation target + src_jars = depset() + outs = depset() + java_info = _empty_java_info(deps, imps) + + return [ + ScroogeAspectInfo( + src_jars = src_jars, + output_files = outs, + thrift_info = transitive_ti, + java_info = java_info) + ] + +scrooge_aspect = aspect( + implementation = _scrooge_aspect_impl, + attr_aspects = ['deps'], attrs = { - "deps": attr.label_list(mandatory = True), - #TODO we should think more about how we want to deal - # with these sorts of things... this basically - # is saying that we have a jar with a bunch - # of thrifts that we want to depend on. Seems like - # that should be a concern of thrift_library? we have - # it here through because we need to show that it is - # "covered," as well as needing the thrifts to - # do the code gen. - "remote_jars": attr.label_list(), - "jvm_flags": attr. - string_list(), # the jvm flags to use with the generator - "with_finagle": attr.bool(default = False), "_pluck_scrooge_scala": attr.label( executable = True, cfg = "host", default = Label("//src/scala/scripts:generator"), allow_files = True), + "_scalac": attr.label( + executable = True, + cfg = "host", + default = Label("//src/java/io/bazel/rulesscala/scalac"), + allow_files = True), + "_implicit_compile_deps": attr.label_list( + providers = [JavaInfo], + default = [ + Label( + "//external:io_bazel_rules_scala/dependency/scala/scala_library" + ), + Label( + "//external:io_bazel_rules_scala/dependency/thrift/libthrift" + ), + Label( + "//external:io_bazel_rules_scala/dependency/thrift/scrooge_core" + ), + ]), }, - outputs = { - "srcjar": "lib%{name}.srcjar", - }, + required_aspect_providers = [[ThriftInfo], + [ScroogeImport]], + toolchains = ['@io_bazel_rules_scala//scala:toolchain_type'], ) -def scrooge_scala_library(name, - deps = [], - remote_jars = [], - jvm_flags = [], - visibility = None, - with_finagle = False): - srcjar = name + '_srcjar' - scrooge_scala_srcjar( - name = srcjar, - deps = deps, - remote_jars = remote_jars, - visibility = visibility, - with_finagle = with_finagle, - ) +def _scrooge_scala_library_impl(ctx): + aspect_info = merge_scrooge_aspect_info( + [dep[ScroogeAspectInfo] for dep in ctx.attr.deps]) + if ctx.attr.exports: + exports = [exp[JavaInfo] for exp in ctx.attr.exports] + all_java = java_common.merge(exports + [aspect_info.java_info]) + else: + all_java = aspect_info.java_info + return [ + DefaultInfo(files = aspect_info.output_files), + ScroogeInfo(aspect_info = aspect_info), all_java + ] + +scrooge_scala_library = rule( + implementation = _scrooge_scala_library_impl, + attrs = { + 'deps': attr.label_list(aspects = [scrooge_aspect]), + 'exports': attr.label_list(providers = [JavaInfo]), + }, + provides = [DefaultInfo, ScroogeInfo, JavaInfo], +) - scala_library( - name = name, - srcs = [srcjar], - deps = deps + remote_jars + [ - "//external:io_bazel_rules_scala/dependency/thrift/libthrift", - "//external:io_bazel_rules_scala/dependency/thrift/scrooge_core" - ], - exports = deps + remote_jars + [ - "//external:io_bazel_rules_scala/dependency/thrift/libthrift", - "//external:io_bazel_rules_scala/dependency/thrift/scrooge_core", - ], - jvm_flags = jvm_flags, - visibility = visibility, - expect_java_output = False, - ) +def _scrooge_scala_import_impl(ctx): + scala_jars = depset(ctx.files.scala_jars) + jars_ji = java_common.create_provider( + use_ijar = False, + compile_time_jars = scala_jars, + transitive_compile_time_jars = scala_jars, + transitive_runtime_jars = scala_jars) + java_info = java_common.merge( + [imp[JavaInfo] for imp in ctx.attr._implicit_compile_deps] + [jars_ji]) + # to make the thrift_info, we only put this in the + # transitive part + ti = ThriftInfo( + srcs = depset(), transitive_srcs = depset(ctx.files.thrift_jars)) + return [java_info, ti, ScroogeImport(java_info = java_info, thrift_info = ti)] + +# Allows you to consume thrifts and compiled jars from external repos +scrooge_scala_import = rule( + implementation = _scrooge_scala_import_impl, + attrs = { + "thrift_jars": attr.label_list(allow_files = [".jar"]), + "scala_jars": attr.label_list(allow_files = [".jar"]), + "_implicit_compile_deps": attr.label_list( + providers = [JavaInfo], + default = [ + Label( + "//external:io_bazel_rules_scala/dependency/scala/scala_library" + ), + Label( + "//external:io_bazel_rules_scala/dependency/thrift/libthrift" + ), + Label( + "//external:io_bazel_rules_scala/dependency/thrift/scrooge_core" + ), + ]), + }, + provides = [ThriftInfo, JavaInfo, ScroogeImport])