Merge #123974

123974: roachprod: add install cmd for side-eye r=dt a=dt

This also required a new optional extra map of local -- to the client running roachprod -- commands that can be run to replace arguments in the install cmd, so that we can get the key without checking it in.

Alternateively we could special-case the side-eye install in go code but this felt more generalized.

Release note: none.
Epic: none.

Co-authored-by: David Taylor <tinystatemachine@gmail.com>

pkg/roachprod/install/install.go

@@ -15,10 +15,14 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"os"
+	"os/exec"
 	"sort"
+	"strings"
 
 	rperrors "github.com/cockroachdb/cockroach/pkg/roachprod/errors"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/logger"
+	"github.com/cockroachdb/errors"
 )
 
 var installCmds = map[string]string{
@@ -93,6 +97,19 @@ echo "deb [signed-by=/etc/apt/keyrings/fluent-bit.gpg] https://packages.fluentbi
 sudo apt-get update;
 sudo apt-get install -y fluent-bit;
 `,
+
+	"side-eye": `
+	curl https://sh.side-eye.io/ | SIDE_EYE_API_TOKEN=%API_KEY% SIDE_EYE_ENVIRONMENT="%ROACHPROD_CLUSTER_NAME%" sh
+	`,
+}
+
+// installLocalCmds is a map from software name to a map of strings that
+// are replaced in the installCmd for that software with the stdout of executing
+// a command locally.
+var installLocalCmds = map[string]map[string]*exec.Cmd{
+	"side-eye": {
+		"%API_KEY%": exec.Command("gcloud", "secrets", "versions", "access", "latest", "--secret", "side-eye-key"),
+	},
 }
 
 // SortedCmds TODO(peter): document
@@ -129,6 +146,18 @@ func InstallTool(
 	if !ok {
 		return fmt.Errorf("unknown tool %q", softwareName)
 	}
+	cmd = strings.ReplaceAll(cmd, "%ROACHPROD_CLUSTER_NAME%", c.Name)
+
+	for replace, localCmd := range installLocalCmds[softwareName] {
+		copy := *localCmd
+		copy.Stderr = os.Stderr
+		out, err := copy.Output()
+		if err != nil {
+			return errors.Wrapf(err, "running local command to derive install argument %s, command %s, failed", replace, copy.String())
+		}
+		cmd = strings.ReplaceAll(cmd, replace, string(out))
+	}
+
 	// Ensure that we early exit if any of the shell statements fail.
 	cmd = "set -exuo pipefail;" + cmd
 	if err := c.Run(ctx, l, stdout, stderr, WithNodes(nodes), "installing "+softwareName, cmd); err != nil {