go/tools/gopackagesdriver/pkgjson: Construct pkg json from file input (#4371)

When testing the bazel driver, we ran into an error that flaged that the
argument list as too long.

```
ERROR: ...json failed: (Exit -1): pkgjson failed: error executing Action command (from target //...) bazel-out/k8-opt-exec-ST-a828a81199fe/bin/external/io_bazel_rules_go/go/tools/gopackagesdriver/pkgjson/reset_pkgjson/pkgjson --id @//src/... --pkg_path ... (remaining 15 arguments skipped)

Use --sandbox_debug to see verbose messages from the sandbox and retain the sandbox build root for debugging
Action failed to execute: java.io.IOException: Cannot run program "/home/user/.cache/bazel/_bazel_rhang/install/7e4ce7b0d69e79cb6bd84c7f9dfefe6b/process-wrapper" (in directory "/home/user/.cache/pkgdrv/0ddf1c72b811bee41d29991c732306ef72553747/sandbox/processwrapper-sandbox/29913/execroot/__main__"): error=7, Argument list too long
ERROR: Build did NOT complete successfully
```

Digging into this issue, the cause is that the pkgjson command takes in
all of the fields of package archive data as arguments.

To work around this, we should preserve the original approach of writing
a pkg json, before #4338,
which used Skylark builtins to write the package content directly to
disk.

The pkgjson command is updated to parse ths json file directly and write
out a transformed pkg json with the cgo related corrections in order to
avoid limits regarding argument size.

Note:
    
This diff also makes changes to undo a breaking change (i.e. changing
the signature of the make_pkg_json function) that was made in
#4338.

I revert changes to the functionalty of make_pkg_json and add a new
replacement function make_pkg_json_with_archive.

Author: r-hang

go/tools/gopackagesdriver/aspect.bzl

@@ -46,9 +46,20 @@ def is_file_external(f):
 def file_path(f):
     return file_path_lib(f)
 
-def make_pkg_json(ctx, pkg_json_tool, name, archive):
+# make_pkg_json_with_archive generates a pkg.json file from an archive
+# and supports cgo generated code.
+#
+# This function was created to avoid breaking the signature of make_pkg_json
+# and avoid adding an explicit field for cgo output files in the pkg.json.
+def make_pkg_json_with_archive(ctx, name, archive):
+    pkg_json_file = ctx.actions.declare_file(name + ".pkg.json")
+    write_pkg_json(ctx, ctx.executable._pkgjson, archive, pkg_json_file)
+    return pkg_json_file
+
+# deprecated: use make_pkg_json_with_archive instead
+def make_pkg_json(ctx, name, pkg_info):
     pkg_json_file = ctx.actions.declare_file(name + ".pkg.json")
-    write_pkg_json(ctx, pkg_json_tool, archive, pkg_json_file)
+    ctx.actions.write(pkg_json_file, content = json.encode(pkg_info))
     return pkg_json_file
 
 def _go_pkg_info_aspect_impl(target, ctx):
@@ -87,13 +98,13 @@ def _go_pkg_info_aspect_impl(target, ctx):
         if archive.data.cgo_out_dir:
             compiled_go_files.append(archive.data.cgo_out_dir)
         export_files.append(archive.data.export_file)
-        pkg_json_files.append(make_pkg_json(ctx, ctx.executable._pkgjson, archive.data.name, archive))
+        pkg_json_files.append(make_pkg_json_with_archive(ctx, archive.data.name, archive))
 
         if ctx.rule.kind == "go_test":
             for dep_archive in archive.direct:
                 # find the archive containing the test sources
                 if archive.data.label == dep_archive.data.label:
-                    pkg_json_files.append(make_pkg_json(ctx, ctx.executable._pkgjson, dep_archive.data.name, dep_archive))
+                    pkg_json_files.append(make_pkg_json_with_archive(ctx, dep_archive.data.name, dep_archive))
                     compiled_go_files.extend(dep_archive.source.srcs)
                     export_files.append(dep_archive.data.export_file)
                     break

go/tools/gopackagesdriver/pkgjson/main.go

@@ -27,15 +27,8 @@ func main() {
 }
 
 type params struct {
-	id string
-	pkgPath string
-	exportFile string
-	goFiles []string
-	compiledGoFiles []string
-	otherFiles []string
-	imports map[string]string
+	pkgJSONPath string
 	cgoOutDir string
-	// path to write the pkgjson
 	output string
 }
 
@@ -53,32 +46,16 @@ func parseArgs(args []string) (*params, error) {
 	var params params
 
 	fs := flag.NewFlagSet("pkgjson", flag.ContinueOnError)
-	fs.StringVar(&params.id, "id", "", "id")
-	fs.StringVar(&params.pkgPath, "pkg_path", "", "package path")
-	fs.StringVar(&params.exportFile, "export_file", "", "export file")
+
+	fs.StringVar(&params.pkgJSONPath, "pkg_json", "", "pkg json file")
 	fs.StringVar(&params.cgoOutDir, "cgo_out_dir", "", "cgo out dir")
-	rawGoFiles := fs.String("go_files", "", "a comma separated list of go files")
-	rawCompiledGoFiles := fs.String("compiled_go_files", "", "a comma separated list of compiled go files")
-	rawOtherFiles := fs.String("other_files", "", "a comma separated list of other files")
-	rawImports := fs.String("imports", "", "comma separate pairs of importpath=label")
-	fs.StringVar(&params.output, "output", "", "path to write the pkgjson")
+	fs.StringVar(&params.output, "output", "", "output file")
 
 	err := fs.Parse(args)
 	if err != nil {
 		return nil, err
 	}
 
-	params.goFiles = strings.Split(*rawGoFiles, ",")
-	params.compiledGoFiles = strings.Split(*rawCompiledGoFiles, ",")
-	params.otherFiles = strings.Split(*rawOtherFiles, ",")
-	params.imports = make(map[string]string)
-	if len(*rawImports) > 0 {
-		for _, rawImport := range strings.Split(*rawImports, ",") {
-			parts := strings.Split(rawImport, "=")
-			params.imports[parts[0]] = parts[1]
-		}
-	}
-
 	return &params, nil
 }
 
@@ -87,14 +64,23 @@ func run(args []string) error {
 	if err != nil {
 		return err
 	}
+
+	b, err := os.ReadFile(params.pkgJSONPath)
+	if err != nil {
+		return err
+	}
+
+	var pjson pkgJson
+	json.Unmarshal(b, &pjson)
+
 	data := pkgJson{
-		ID: params.id,
-		PkgPath: params.pkgPath,
-		ExportFile: params.exportFile,
-		GoFiles: params.goFiles,
-		CompiledGoFiles: params.compiledGoFiles,
-		OtherFiles: params.otherFiles,
-		Imports: params.imports,
+		ID: pjson.ID,
+		PkgPath: pjson.PkgPath,
+		ExportFile: pjson.ExportFile,
+		GoFiles: pjson.GoFiles,
+		CompiledGoFiles: pjson.CompiledGoFiles,
+		OtherFiles: pjson.OtherFiles,
+		Imports: pjson.Imports,
 	}
 	if err = processCgoFiles(params.cgoOutDir, &data, resolvePath); err != nil {
 		return err

go/tools/gopackagesdriver/pkgjson/pkg_json.bzl

@@ -3,38 +3,26 @@ load(
     "paths",
 )
 
-def write_pkg_json(ctx, pkg_json_tool, archive, output_json):
+def write_pkg_json(ctx, pkg_json_tool, archive, pkg_json):
+    args = ctx.actions.args()
     inputs = [src for src in archive.data.srcs if src.path.endswith(".go")]
-    input_paths = [file_path(src) for src in inputs]
-    cgo_out_dir = ""
+
+    tmp_json = ctx.actions.declare_file(pkg_json.path + ".tmp")
+    pkg_info = _go_archive_to_pkg(archive)
+    ctx.actions.write(tmp_json, content = json.encode(pkg_info))
+    inputs.append(tmp_json)
+    args.add("--pkg_json", tmp_json.path)
+
     if archive.data.cgo_out_dir:
         inputs.append(archive.data.cgo_out_dir)
-        cgo_out_dir = file_path(archive.data.cgo_out_dir)
+        args.add("--cgo_out_dir", file_path(archive.data.cgo_out_dir))
 
+    args.add("--output", pkg_json.path)
     ctx.actions.run(
         inputs = inputs,
-        outputs = [output_json],
-        executable = pkg_json_tool,
-        arguments = [
-            "--id",
-            str(archive.data.label),
-            "--pkg_path",
-            archive.data.importpath,
-            "--export_file",
-            file_path(archive.data.export_file),
-            "--go_files",
-            ",".join(input_paths),
-            "--compiled_go_files",
-            ",".join(input_paths),
-            "--cgo_out_dir",
-            cgo_out_dir,
-            "--other_files",
-            ",".join([file_path(src) for src in archive.data.srcs if not src.path.endswith(".go")]),
-            "--imports",
-            ",".join([pkg.data.importpath + "=" + str(pkg.data.label) for pkg in archive.direct]),
-            "--output",
-            output_json.path,
-        ],
+        outputs = [pkg_json],
+        executable = pkg_json_tool.path,
+        arguments = [args],
         tools = [pkg_json_tool],
     )
 
@@ -48,3 +36,26 @@ def file_path(f):
 
 def is_file_external(f):
     return f.owner.workspace_root != ""
+
+def _go_archive_to_pkg(archive):
+    go_files = [
+        file_path(src)
+        for src in archive.data.srcs
+        if src.path.endswith(".go")
+    ]
+    return struct(
+        ID = str(archive.data.label),
+        PkgPath = archive.data.importpath,
+        ExportFile = file_path(archive.data.export_file),
+        GoFiles = go_files,
+        CompiledGoFiles = go_files,
+        OtherFiles = [
+            file_path(src)
+            for src in archive.data.srcs
+            if not src.path.endswith(".go")
+        ],
+        Imports = {
+            pkg.data.importpath: str(pkg.data.label)
+            for pkg in archive.direct
+        },
+    )