Wire go_sdk and go_toolchain together (bazelbuild#1607)

* go_toolchain has a new mandatory attribute, "sdk", which be
  something that provides GoSDK.
* go_host_sdk, go_local_sdk, and go_download_sdk are now macros that
  wrap the old rules. Each rule declares toolchains in its BUILD.bazel
  file that work on the host architecture. The macro calls
  register_toolchains with these.
* go_register_toolchains no longer calls register_toolchains, but it
  will an SDK rule if "go_sdk" isn't defined. This is a step toward
  allowing multiple SDKs to support multiple execution platforms.
* Action inputs are narrowed to use go.sdk.tools and go.stdlib.libs
  rather than larger sets of files.

go/private/BUILD.sdk.bazel

@@ -1,23 +1,64 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_sdk")
+load("@io_bazel_rules_go//go/toolchain:toolchains.bzl", "declare_toolchains")
+load("@io_bazel_rules_go//go/private:rules/sdk.bzl", "package_list")
+
 package(default_visibility = ["//visibility:public"])
 
 filegroup(
-    name = "files",
-    srcs = glob([
-        "bin/go*",
-        "src/**",
-        "pkg/**",
-    ]),
+    name = "libs",
+    srcs = glob(
+        ["pkg/{goos}_{goarch}/**/*.a"],
+        exclude = ["pkg/{goos}_{goarch}/**/cmd/**"],
+    ),
+)
+
+filegroup(
+    name = "headers",
+    srcs = glob(["pkg/include/*.h"]),
+)
+
+filegroup(
+    name = "srcs",
+    srcs = glob(["src/**"]),
 )
 
 filegroup(
     name = "tools",
+    srcs = glob(["pkg/tool/**"]),
+)
+
+go_sdk(
+    name = "go_sdk",
+    goos = "{goos}",
+    goarch = "{goarch}",
+    root_file = "ROOT",
+    package_list = ":package_list",
+    libs = [":libs"],
+    headers = [":headers"],
+    srcs = [":srcs"],
+    tools = [":tools"],
+    go = "bin/go{exe}",
+)
+
+# TODO(jayconrod): Gazelle depends on this file directly. This dependency
+# should be broken, and this rule should be folded into go_sdk.
+package_list(
+    name = "package_list",
+    srcs = [":srcs"],
+    root_file = "ROOT",
+    out = "packages.txt",
+)
+
+declare_toolchains(
+    host = "{goos}_{goarch}",
+    sdk = ":go_sdk",
+)
+
+filegroup(
+    name = "files",
     srcs = glob([
         "bin/go*",
-        "pkg/tool/**",
-    ]) + select({
-        "@io_bazel_rules_go//go/platform:darwin_amd64": ["@local_config_cc//:cc_wrapper"],
-        "//conditions:default": [],
-    }),
+        "src/**",
+        "pkg/**",
+    ]),
 )
-
-exports_files(["packages.txt", "ROOT"])

go/private/actions/asm.bzl

@@ -33,11 +33,11 @@ def emit_asm(
         fail("source is a required parameter")
 
     out_obj = go.declare_file(go, path = source.basename[:-2], ext = ".o")
-    inputs = hdrs + go.sdk_tools + go.stdlib.files + [source]
+    inputs = hdrs + go.sdk.tools + go.sdk.headers + go.stdlib.libs + [source]
 
     args = go.args(go)
     args.add_all([source, "--"])
-    includes = ([go.stdlib.root_file.dirname + "/pkg/include"] +
+    includes = ([go.sdk.root_file.dirname + "/pkg/include"] +
                 [f.dirname for f in hdrs])
 
     # TODO(#1463): use uniquify=True when available.

go/private/actions/compile.bzl

@@ -55,7 +55,7 @@ def emit_compile(
 
     inputs = (sources + [go.package_list] +
               [archive.data.file for archive in archives] +
-              go.sdk_tools + go.stdlib.files)
+              go.sdk.tools + go.stdlib.libs)
     outputs = [out_lib]
 
     builder_args = go.args(go)
@@ -105,7 +105,7 @@ def _bootstrap_compile(go, sources, out_lib, gc_goopts):
     args.extend(gc_goopts)
     args.extend([s.path for s in sources])
     go.actions.run_shell(
-        inputs = sources + go.sdk_files + go.sdk_tools,
+        inputs = sources + go.sdk.libs + go.sdk.tools + [go.go],
         outputs = [out_lib],
         mnemonic = "GoCompile",
         command = "export GOROOT=$(pwd)/{} && export GOROOT_FINAL=GOROOT && {} {}".format(go.root, go.go.path, " ".join(args)),

go/private/actions/cover.bzl

@@ -56,7 +56,7 @@ def emit_cover(go, source):
             "-mode=set",
         ])
         go.actions.run(
-            inputs = [src] + go.sdk_tools,
+            inputs = [src] + go.sdk.tools,
             outputs = [out],
             mnemonic = "GoCover",
             executable = go.builders.cover,

go/private/actions/link.bzl

@@ -141,8 +141,8 @@ def emit_link(
             archive.cgo_deps,
             go.crosstool,
             stamp_inputs,
-            go.sdk_tools,
-            go.stdlib.files,
+            go.sdk.tools,
+            go.stdlib.libs,
         ),
         outputs = [executable],
         mnemonic = "GoLink",
@@ -154,7 +154,7 @@ def emit_link(
 def _bootstrap_link(go, archive, executable, gc_linkopts):
     """See go/toolchains.rst#link for full documentation."""
 
-    inputs = [archive.data.file] + go.sdk_files + go.sdk_tools
+    inputs = [archive.data.file] + go.sdk.libs + go.sdk.tools + [go.go]
     args = ["tool", "link", "-s", "-o", executable.path]
     args.extend(gc_linkopts)
     args.append(archive.data.file.path)

go/private/actions/pack.bzl

@@ -25,7 +25,7 @@ def emit_pack(
     if out_lib == None:
         fail("out_lib is a required parameter")
 
-    inputs = [in_lib] + go.sdk_tools + objects + archives
+    inputs = [in_lib] + go.sdk.tools + objects + archives
 
     args = go.args(go)
     args.add_all(["-in", in_lib])

go/private/context.bzl

@@ -79,7 +79,7 @@ def _declare_directory(go, path = "", ext = "", name = ""):
 
 def _new_args(go):
     args = go.actions.args()
-    args.add_all(["-sdk", go.sdk_root.dirname])
+    args.add_all(["-sdk", go.sdk.root_file.dirname])
     if go.tags:
         args.add("-tags")
         args.add_joined(go.tags, join_with = ",")
@@ -192,20 +192,6 @@ def _infer_importpath(ctx):
         importpath = importpath[1:]
     return importpath, importpath, INFERRED_PATH
 
-def _get_go_binary(context_data):
-    for f in context_data.sdk_tools:
-        parent = paths.dirname(f.path)
-        sdk = paths.dirname(parent)
-        parent = paths.basename(parent)
-        if parent != "bin":
-            continue
-        basename = paths.basename(f.path)
-        name, ext = paths.split_extension(basename)
-        if name != "go":
-            continue
-        return f
-    fail("Could not find go executable in go_sdk")
-
 def go_context(ctx, attr = None):
     toolchain = ctx.toolchains["@io_bazel_rules_go//go:toolchain"]
 
@@ -230,14 +216,14 @@ def go_context(ctx, attr = None):
         tags.append("race")
     if mode.msan:
         tags.append("msan")
-    binary = _get_go_binary(context_data)
+    binary = toolchain.sdk.go
 
     stdlib = getattr(attr, "_stdlib", None)
     if stdlib:
         stdlib = get_source(stdlib).stdlib
         goroot = stdlib.root_file.dirname
     else:
-        goroot = context_data.sdk_root.dirname
+        goroot = toolchain.sdk.root_file.dirname
 
     env = dict(context_data.env)
     env.update({
@@ -249,22 +235,31 @@ def go_context(ctx, attr = None):
         "PATH": context_data.cgo_tools.compiler_path,
     })
 
+    # TODO(jayconrod): remove this. It's way too broad. Everything should
+    # depend on more specific lists.
+    sdk_files = ([toolchain.sdk.go] +
+                 toolchain.sdk.srcs +
+                 toolchain.sdk.headers +
+                 toolchain.sdk.libs +
+                 toolchain.sdk.tools)
+
     importpath, importmap, pathtype = _infer_importpath(ctx)
     return GoContext(
         # Fields
         toolchain = toolchain,
+        sdk = toolchain.sdk,
         mode = mode,
         root = goroot,
         go = binary,
         stdlib = stdlib,
-        sdk_root = context_data.sdk_root,
-        sdk_files = context_data.sdk_files,
-        sdk_tools = context_data.sdk_tools,
+        sdk_root = toolchain.sdk.root_file,
+        sdk_files = sdk_files,
+        sdk_tools = toolchain.sdk.tools,
         actions = ctx.actions,
         exe_extension = goos_to_extension(mode.goos),
         shared_extension = goos_to_shared_extension(mode.goos),
         crosstool = context_data.crosstool,
-        package_list = context_data.package_list,
+        package_list = toolchain.sdk.package_list,
         importpath = importpath,
         importmap = importmap,
         pathtype = pathtype,
@@ -316,10 +311,6 @@ def _go_context_data(ctx):
     return struct(
         strip = ctx.attr.strip,
         crosstool = ctx.files._crosstool,
-        package_list = ctx.file._package_list,
-        sdk_root = ctx.file._sdk_root,
-        sdk_files = ctx.files._sdk_files,
-        sdk_tools = ctx.files._sdk_tools,
         tags = tags,
         env = env,
         cgo_tools = struct(
@@ -337,26 +328,7 @@ go_context_data = rule(
     _go_context_data,
     attrs = {
         "strip": attr.string(mandatory = True),
-        # Hidden internal attributes
         "_crosstool": attr.label(default = Label("@bazel_tools//tools/cpp:current_cc_toolchain")),
-        "_package_list": attr.label(
-            allow_files = True,
-            single_file = True,
-            default = "@go_sdk//:packages.txt",
-        ),
-        "_sdk_root": attr.label(
-            allow_single_file = True,
-            default = "@go_sdk//:ROOT",
-        ),
-        "_sdk_files": attr.label(
-            allow_files = True,
-            default = "@go_sdk//:files",
-        ),
-        "_sdk_tools": attr.label(
-            allow_files = True,
-            cfg = "host",
-            default = "@go_sdk//:tools",
-        ),
         "_xcode_config": attr.label(
             default = Label("@bazel_tools//tools/osx:current_xcode_config"),
         ),

go/private/go_toolchain.bzl

@@ -15,6 +15,8 @@
 Toolchain rules used by go.
 """
 
+load("@io_bazel_rules_go//go/platform:list.bzl", "GOOS_GOARCH")
+load("@io_bazel_rules_go//go/private:providers.bzl", "GoSDK")
 load("@io_bazel_rules_go//go/private:actions/archive.bzl", "emit_archive")
 load("@io_bazel_rules_go//go/private:actions/asm.bzl", "emit_asm")
 load("@io_bazel_rules_go//go/private:actions/binary.bzl", "emit_binary")
@@ -24,9 +26,11 @@ load("@io_bazel_rules_go//go/private:actions/link.bzl", "emit_link")
 load("@io_bazel_rules_go//go/private:actions/pack.bzl", "emit_pack")
 
 def _go_toolchain_impl(ctx):
+    sdk = ctx.attr.sdk[GoSDK]
+    cross_compile = ctx.attr.goos != sdk.goos or ctx.attr.goarch != sdk.goarch
     return [platform_common.ToolchainInfo(
         name = ctx.label.name,
-        cross_compile = ctx.attr.cross_compile,
+        cross_compile = cross_compile,
         default_goos = ctx.attr.goos,
         default_goarch = ctx.attr.goarch,
         actions = struct(
@@ -43,27 +47,43 @@ def _go_toolchain_impl(ctx):
             link = ctx.attr.link_flags,
             link_cgo = ctx.attr.cgo_link_flags,
         ),
+        sdk = sdk,
     )]
 
 _go_toolchain = rule(
     _go_toolchain_impl,
     attrs = {
         # Minimum requirements to specify a toolchain
-        "goos": attr.string(mandatory = True),
-        "goarch": attr.string(mandatory = True),
-        "cross_compile": attr.bool(default = False),
+        "goos": attr.string(
+            mandatory = True,
+            doc = "Default target OS",
+        ),
+        "goarch": attr.string(
+            mandatory = True,
+            doc = "Default target architecture",
+        ),
+        "sdk": attr.label(
+            mandatory = True,
+            providers = [GoSDK],
+            doc = "The SDK this toolchain is based on",
+        ),
         # Optional extras to a toolchain
-        "link_flags": attr.string_list(default = []),
-        "cgo_link_flags": attr.string_list(default = []),
+        "link_flags": attr.string_list(
+            doc = "Flags passed to the Go internal linker",
+        ),
+        "cgo_link_flags": attr.string_list(
+            doc = "Flags passed to the external linker (if it is used)",
+        ),
     },
+    doc = "Defines a Go toolchain based on an SDK",
+    provides = [platform_common.ToolchainInfo],
 )
 
-def go_toolchain(name, target, host = None, constraints = [], **kwargs):
+def go_toolchain(name, target, sdk, host = None, constraints = [], **kwargs):
     """See go/toolchains.rst#go-toolchain for full documentation."""
 
     if not host:
         host = target
-    cross = host != target
     goos, _, goarch = target.partition("_")
     target_constraints = constraints + [
         "@io_bazel_rules_go//go/toolchain:" + goos,
@@ -80,7 +100,7 @@ def go_toolchain(name, target, host = None, constraints = [], **kwargs):
         name = impl_name,
         goos = goos,
         goarch = goarch,
-        cross_compile = cross,
+        sdk = sdk,
         tags = ["manual"],
         visibility = ["//visibility:public"],
         **kwargs
@@ -92,3 +112,32 @@ def go_toolchain(name, target, host = None, constraints = [], **kwargs):
         target_compatible_with = target_constraints,
         toolchain = ":" + impl_name,
     )
+
+def generate_toolchains(host, sdk):
+    host_goos, _, host_goarch = host.partition("_")
+    toolchains = []
+    for target_goos, target_goarch in GOOS_GOARCH:
+        target = "{}_{}".format(target_goos, target_goarch)
+        toolchain_name = "go_" + target
+        link_flags = []
+        cgo_link_flags = []
+        if "darwin" in host:
+            cgo_link_flags.extend(["-shared", "-Wl,-all_load"])
+        if "linux" in host:
+            cgo_link_flags.append("-Wl,-whole-archive")
+
+        # Add the primary toolchain
+        toolchains.append(dict(
+            name = toolchain_name,
+            host = host,
+            target = target,
+            sdk = sdk,
+            link_flags = link_flags,
+            cgo_link_flags = cgo_link_flags,
+        ))
+    return toolchains
+
+def generate_toolchain_names():
+    # Keep in sync with generate_toolchains
+    return ["go_{}_{}".format(target_goos, target_goarch)
+            for target_goos, target_goarch in GOOS_GOARCH]