dev: implement dev generate docs

build/teamcity/cockroach/ci/builds/build_impl.sh

@@ -14,7 +14,7 @@ CONFIG="$1"
 DOC_TARGETS=
 if [ "$CONFIG" == "crosslinux" ]
 then
-   DOC_TARGETS="//docs/generated:gen-logging-md //docs/generated:gen-logsinks-md //docs/generated:gen-eventlog-md //docs/generated:gen-logformats-md //docs/generated/settings:settings //docs/generated/settings:settings_for_tenants //docs/generated/sql //docs/generated/sql/bnf"
+   DOC_TARGETS=$(grep '^//' docs/generated/bazel_targets.txt)
 fi
 
 bazel build //pkg/cmd/bazci --config=ci

docs/generated/bazel_targets.txt

@@ -0,0 +1,11 @@
+This file lists all the targets you need to build to build all the generated
+documentation. Lines not beginning with // should be ignored.
+
+//docs/generated:gen-logging-md
+//docs/generated:gen-logsinks-md
+//docs/generated:gen-eventlog-md
+//docs/generated:gen-logformats-md
+//docs/generated/settings
+//docs/generated/settings:settings_for_tenants
+//docs/generated/sql
+//docs/generated/sql/bnf

pkg/BUILD.bazel

@@ -7,6 +7,7 @@ ALL_TESTS = [
     "//pkg/bench/rttanalysis:rttanalysis_test",
     "//pkg/bench:bench_test",
     "//pkg/blobs:blobs_test",
+    "//pkg/build/util:util_test",
     "//pkg/ccl/backupccl/backupresolver:backupresolver_test",
     "//pkg/ccl/backupccl:backupccl_test",
     "//pkg/ccl/baseccl:baseccl_test",

pkg/build/util/BUILD.bazel

@@ -0,0 +1,15 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "util",
+    srcs = ["util.go"],
+    importpath = "github.com/cockroachdb/cockroach/pkg/build/util",
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "util_test",
+    srcs = ["util_test.go"],
+    embed = [":util"],
+    deps = ["@com_github_stretchr_testify//require"],
+)

pkg/build/util/util.go

@@ -0,0 +1,75 @@
+// Copyright 2015 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+//
+// This file contains assorted utilities for working with Bazel internals.
+
+package util
+
+import (
+	"fmt"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+// OutputOfBinaryRule returns the path of the binary produced by the
+// given build target, relative to bazel-bin. That is,
+//    filepath.Join(bazelBin, OutputOfBinaryRule(target)) is the absolute
+// path to the build binary for the target.
+func OutputOfBinaryRule(target string) string {
+	colon := strings.Index(target, ":")
+	var bin string
+	if colon >= 0 {
+		bin = target[colon+1:]
+	} else {
+		bin = target[strings.LastIndex(target, "/")+1:]
+	}
+	var head string
+	if strings.HasPrefix(target, "@") {
+		doubleSlash := strings.Index(target, "//")
+		head = filepath.Join("external", target[1:doubleSlash])
+	} else if colon >= 0 {
+		head = strings.TrimPrefix(target[:colon], "//")
+	} else {
+		head = strings.TrimPrefix(target, "//")
+	}
+	return filepath.Join(head, bin+"_", bin)
+}
+
+// OutputsOfGenrule lists the outputs of a genrule. The first argument
+// is the name of the target (e.g. //docs/generated/sql), and the second
+// should be the output of `bazel query --output=xml $TARGET`. The
+// returned slice is the list of outputs, all of which are relative
+// paths atop `bazel-bin` as in `OutputOfBinaryRule`.
+func OutputsOfGenrule(target, xmlQueryOutput string) ([]string, error) {
+	// XML parsing is a bit heavyweight here, and encoding/xml
+	// refuses to parse the query output since it's XML 1.1 instead
+	// of 1.0. Have fun with regexes instead.
+	colon := strings.LastIndex(target, ":")
+	if colon < 0 {
+		colon = len(target)
+	}
+	regexStr := fmt.Sprintf("^<rule-output name=\"%s:(?P<Filename>.*)\"/>$", regexp.QuoteMeta(target[:colon]))
+	re, err := regexp.Compile(regexStr)
+	if err != nil {
+		return nil, err
+	}
+	var ret []string
+	for _, line := range strings.Split(xmlQueryOutput, "\n") {
+		line = strings.TrimSpace(line)
+		submatch := re.FindStringSubmatch(line)
+		if submatch == nil {
+			continue
+		}
+		relBinPath := filepath.Join(strings.TrimPrefix(target[:colon], "//"), submatch[1])
+		ret = append(ret, relBinPath)
+	}
+	return ret, nil
+}

pkg/build/util/util_test.go

@@ -0,0 +1,62 @@
+// Copyright 2021 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package util
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestOutputOfBinaryRule(t *testing.T) {
+	require.Equal(t, OutputOfBinaryRule("//pkg/cmd/cockroach-short"), "pkg/cmd/cockroach-short/cockroach-short_/cockroach-short")
+	require.Equal(t, OutputOfBinaryRule("//pkg/cmd/cockroach-short:cockroach-short"), "pkg/cmd/cockroach-short/cockroach-short_/cockroach-short")
+	require.Equal(t, OutputOfBinaryRule("pkg/cmd/cockroach-short"), "pkg/cmd/cockroach-short/cockroach-short_/cockroach-short")
+
+	require.Equal(t, OutputOfBinaryRule("@com_github_cockroachdb_stress//:stress"), "external/com_github_cockroachdb_stress/stress_/stress")
+}
+
+func TestOutputsOfGenrule(t *testing.T) {
+	xmlQueryOutput := `<?xml version="1.1" encoding="UTF-8" standalone="no"?>
+<query version="2">
+    <rule class="genrule" location="/Users/ricky/go/src/github.com/cockroachdb/cockroach/docs/generated/sql/BUILD.bazel:1:8" name="//docs/generated/sql:sql">
+        <string name="name" value="sql"/>
+        <list name="exec_tools">
+            <label value="//pkg/cmd/docgen:docgen"/>
+        </list>
+        <list name="outs">
+            <output value="//docs/generated/sql:aggregates.md"/>
+            <output value="//docs/generated/sql:functions.md"/>
+            <output value="//docs/generated/sql:operators.md"/>
+            <output value="//docs/generated/sql:window_functions.md"/>
+        </list>
+        <string name="cmd" value="&#10;$(location //pkg/cmd/docgen) functions $(RULEDIR) --quiet&#10;"/>
+        <rule-input name="//pkg/cmd/docgen:docgen"/>
+        <rule-input name="@bazel_tools//tools/genrule:genrule-setup.sh"/>
+        <rule-output name="//docs/generated/sql:aggregates.md"/>
+        <rule-output name="//docs/generated/sql:functions.md"/>
+        <rule-output name="//docs/generated/sql:operators.md"/>
+        <rule-output name="//docs/generated/sql:window_functions.md"/>
+    </rule>
+</query>`
+	expected := []string{
+		"docs/generated/sql/aggregates.md",
+		"docs/generated/sql/functions.md",
+		"docs/generated/sql/operators.md",
+		"docs/generated/sql/window_functions.md",
+	}
+	out, err := OutputsOfGenrule("//docs/generated/sql:sql", xmlQueryOutput)
+	require.NoError(t, err)
+	require.Equal(t, out, expected)
+	out, err = OutputsOfGenrule("//docs/generated/sql", xmlQueryOutput)
+	require.NoError(t, err)
+	require.Equal(t, out, expected)
+}

pkg/cmd/bazci/BUILD.bazel

@@ -10,6 +10,7 @@ go_library(
     importpath = "github.com/cockroachdb/cockroach/pkg/cmd/bazci",
     visibility = ["//visibility:private"],
     deps = [
+        "//pkg/build/util",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_spf13_cobra//:cobra",
     ],

pkg/cmd/bazci/watch.go

@@ -19,10 +19,10 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"regexp"
 	"strings"
 	"time"
 
+	bazelutil "github.com/cockroachdb/cockroach/pkg/build/util"
 	"github.com/cockroachdb/errors"
 )
 
@@ -216,22 +216,11 @@ func (w watcher) stageBinaryArtifacts() error {
 		if err != nil {
 			return err
 		}
-		// XML parsing is a bit heavyweight here, and encoding/xml
-		// refuses to parse the query output since it's XML 1.1 instead
-		// of 1.0. Have fun with regexes instead.
-		colon := strings.LastIndex(bin, ":")
-		regexStr := fmt.Sprintf("^<rule-output name=\"%s:(?P<Filename>.*)\"/>$", regexp.QuoteMeta(bin[:colon]))
-		re, err := regexp.Compile(regexStr)
+		outs, err := bazelutil.OutputsOfGenrule(bin, query)
 		if err != nil {
 			return err
 		}
-		for _, line := range strings.Split(query, "\n") {
-			line = strings.TrimSpace(line)
-			submatch := re.FindStringSubmatch(line)
-			if submatch == nil {
-				continue
-			}
-			relBinPath := filepath.Join(strings.TrimPrefix(bin[:colon], "//"), submatch[1])
+		for _, relBinPath := range outs {
 			err := w.maybeStageArtifact(binSourceDir, relBinPath, 0666, finalizePhase, copyContentTo)
 			if err != nil {
 				return err

pkg/cmd/dev/BUILD.bazel

@@ -16,6 +16,7 @@ go_library(
     importpath = "github.com/cockroachdb/cockroach/pkg/cmd/dev",
     visibility = ["//visibility:private"],
     deps = [
+        "//pkg/build/util",
         "//pkg/cmd/dev/io/exec",
         "//pkg/cmd/dev/io/os",
         "@com_github_cockroachdb_errors//:errors",

pkg/cmd/dev/build.go

@@ -18,6 +18,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	bazelutil "github.com/cockroachdb/cockroach/pkg/build/util"
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/errors/oserror"
 	"github.com/spf13/cobra"
@@ -103,7 +104,7 @@ func (d *dev) build(cmd *cobra.Command, targets []string) error {
 	script.WriteString(fmt.Sprintf("bazel %s\n", strings.Join(args, " ")))
 	script.WriteString(fmt.Sprintf("BAZELBIN=`bazel info bazel-bin --color=no --config=%s`\n", cross))
 	for _, target := range fullTargets {
-		script.WriteString(fmt.Sprintf("cp $BAZELBIN/%s /artifacts\n", targetToRelativeBinPath(target)))
+		script.WriteString(fmt.Sprintf("cp $BAZELBIN/%s /artifacts\n", bazelutil.OutputOfBinaryRule(target)))
 		script.WriteString(fmt.Sprintf("chmod +w /artifacts/%s\n", targetToBinBasename(target)))
 	}
 	_, err = d.exec.CommandContextWithInput(ctx, script.String(), "docker", dockerArgs...)
@@ -158,28 +159,6 @@ func (d *dev) symlinkBinaries(ctx context.Context, targets []string) error {
 	return nil
 }
 
-// targetToRelativeBinPath returns the path of the binary produced by this build
-// target relative to bazel-bin. That is,
-//    filepath.Join(bazelBin, targetToRelativeBinPath(target)) is the absolute
-// path to the build binary for the target.
-func targetToRelativeBinPath(target string) string {
-	var head string
-	if strings.HasPrefix(target, "@") {
-		doubleSlash := strings.Index(target, "//")
-		head = filepath.Join("external", target[1:doubleSlash])
-	} else {
-		head = strings.TrimPrefix(target, "//")
-	}
-	var bin string
-	colon := strings.Index(target, ":")
-	if colon >= 0 {
-		bin = target[colon+1:]
-	} else {
-		bin = target[strings.LastIndex(target, "/")+1:]
-	}
-	return filepath.Join(head, bin+"_", bin)
-}
-
 func targetToBinBasename(target string) string {
 	base := filepath.Base(strings.TrimPrefix(target, "//"))
 	// If there's a colon, the actual name of the executable is
@@ -192,14 +171,11 @@ func targetToBinBasename(target string) string {
 }
 
 func (d *dev) getPathToBin(ctx context.Context, target string) (string, error) {
-	args := []string{"info", "bazel-bin", "--color=no"}
-	args = append(args, getConfigFlags()...)
-	out, err := d.exec.CommandContextSilent(ctx, "bazel", args...)
+	bazelBin, err := d.getBazelBin(ctx)
 	if err != nil {
 		return "", err
 	}
-	bazelBin := strings.TrimSpace(string(out))
-	rel := targetToRelativeBinPath(target)
+	rel := bazelutil.OutputOfBinaryRule(target)
 	return filepath.Join(bazelBin, rel), nil
 }
 

pkg/cmd/dev/generate.go

@@ -12,7 +12,9 @@ package main
 
 import (
 	"path/filepath"
+	"strings"
 
+	bazelutil "github.com/cockroachdb/cockroach/pkg/build/util"
 	"github.com/cockroachdb/errors"
 	"github.com/spf13/cobra"
 )
@@ -42,6 +44,7 @@ func (d *dev) generate(cmd *cobra.Command, targets []string) error {
 	// TODO(irfansharif): Flesh out the remaining targets.
 	var generatorTargetMapping = map[string]func(cmd *cobra.Command) error{
 		"bazel": d.generateBazel,
+		"docs":  d.generateDocs,
 	}
 
 	if len(targets) == 0 {
@@ -74,3 +77,55 @@ func (d *dev) generateBazel(cmd *cobra.Command) error {
 	_, err = d.exec.CommandContext(ctx, filepath.Join(workspace, "build", "bazelutil", "bazel-generate.sh"))
 	return err
 }
+
+func (d *dev) generateDocs(cmd *cobra.Command) error {
+	ctx := cmd.Context()
+	workspace, err := d.getWorkspace(ctx)
+	if err != nil {
+		return err
+	}
+	// List targets we need to build.
+	targetsFile, err := d.os.ReadFile(filepath.Join(workspace, "docs/generated/bazel_targets.txt"))
+	if err != nil {
+		return err
+	}
+	var targets []string
+	for _, line := range strings.Split(targetsFile, "\n") {
+		line = strings.TrimSpace(line)
+		if strings.HasPrefix(line, "//") {
+			targets = append(targets, line)
+		}
+	}
+	// Build targets.
+	var args []string
+	args = append(args, "build", "--color=yes", "--experimental_convenience_symlinks=ignore")
+	args = append(args, mustGetRemoteCacheArgs(remoteCacheAddr)...)
+	args = append(args, getConfigFlags()...)
+	args = append(args, targets...)
+	err = d.exec.CommandContextInheritingStdStreams(ctx, "bazel", args...)
+	if err != nil {
+		return err
+	}
+	// Copy docs from bazel-bin to workspace.
+	bazelBin, err := d.getBazelBin(ctx)
+	if err != nil {
+		return err
+	}
+	for _, target := range targets {
+		query, err := d.exec.CommandContextSilent(ctx, "bazel", "query", "--output=xml", target)
+		if err != nil {
+			return err
+		}
+		outputs, err := bazelutil.OutputsOfGenrule(target, string(query))
+		if err != nil {
+			return err
+		}
+		for _, output := range outputs {
+			err = d.os.CopyFile(filepath.Join(bazelBin, output), filepath.Join(workspace, output))
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}