Break flagutil.go up so yaml stuff is separate (#2068)

enterprise/server/auth/BUILD

@@ -18,7 +18,7 @@ go_library(
         "//server/tables",
         "//server/util/alert",
         "//server/util/capabilities",
-        "//server/util/flagutil",
+        "//server/util/flagutil/types",
         "//server/util/log",
         "//server/util/lru",
         "//server/util/random",

enterprise/server/auth/auth.go

@@ -19,7 +19,6 @@ import (
 	"github.com/buildbuddy-io/buildbuddy/server/tables"
 	"github.com/buildbuddy-io/buildbuddy/server/util/alert"
 	"github.com/buildbuddy-io/buildbuddy/server/util/capabilities"
-	"github.com/buildbuddy-io/buildbuddy/server/util/flagutil"
 	"github.com/buildbuddy-io/buildbuddy/server/util/log"
 	"github.com/buildbuddy-io/buildbuddy/server/util/lru"
 	"github.com/buildbuddy-io/buildbuddy/server/util/random"
@@ -34,14 +33,15 @@ import (
 	"google.golang.org/grpc/peer"
 
 	akpb "github.com/buildbuddy-io/buildbuddy/proto/api_key"
+	flagtypes "github.com/buildbuddy-io/buildbuddy/server/util/flagutil/types"
 	burl "github.com/buildbuddy-io/buildbuddy/server/util/url"
 	oidc "github.com/coreos/go-oidc"
 )
 
 var (
 	adminGroupID         = flag.String("auth.admin_group_id", "", "ID of a group whose members can perform actions only accessible to server admins.")
 	enableAnonymousUsage = flag.Bool("auth.enable_anonymous_usage", false, "If true, unauthenticated build uploads will still be allowed but won't be associated with your organization.")
-	oauthProviders       = flagutil.Slice("auth.oauth_providers", []OauthProvider{}, "The list of oauth providers to use to authenticate.")
+	oauthProviders       = flagtypes.Slice("auth.oauth_providers", []OauthProvider{}, "The list of oauth providers to use to authenticate.")
 	jwtKey               = flag.String("auth.jwt_key", "set_the_jwt_in_config", "The key to use when signing JWT tokens.")
 	apiKeyGroupCacheTTL  = flag.Duration("auth.api_key_group_cache_ttl", 5*time.Minute, "TTL for API Key to Group caching. Set to '0' to disable cache.")
 	httpsOnlyCookies     = flag.Bool("auth.https_only_cookies", false, "If true, cookies will only be set over https connections.")

enterprise/server/backends/distributed/BUILD

@@ -20,7 +20,7 @@ go_library(
         "//server/resources",
         "//server/util/background",
         "//server/util/consistent_hash",
-        "//server/util/flagutil",
+        "//server/util/flagutil/types",
         "//server/util/log",
         "//server/util/peerset",
         "//server/util/status",

enterprise/server/backends/distributed/distributed.go

@@ -20,7 +20,6 @@ import (
 	"github.com/buildbuddy-io/buildbuddy/server/resources"
 	"github.com/buildbuddy-io/buildbuddy/server/util/background"
 	"github.com/buildbuddy-io/buildbuddy/server/util/consistent_hash"
-	"github.com/buildbuddy-io/buildbuddy/server/util/flagutil"
 	"github.com/buildbuddy-io/buildbuddy/server/util/log"
 	"github.com/buildbuddy-io/buildbuddy/server/util/peerset"
 	"github.com/buildbuddy-io/buildbuddy/server/util/status"
@@ -29,13 +28,14 @@ import (
 
 	dcpb "github.com/buildbuddy-io/buildbuddy/proto/distributed_cache"
 	repb "github.com/buildbuddy-io/buildbuddy/proto/remote_execution"
+	flagtypes "github.com/buildbuddy-io/buildbuddy/server/util/flagutil/types"
 )
 
 var (
 	listenAddr        = flag.String("cache.distributed_cache.listen_addr", "", "The address to listen for local BuildBuddy distributed cache traffic on.")
 	redisTarget       = flag.String("cache.distributed_cache.redis_target", "", "A redis target for improved Caching/RBE performance. Target can be provided as either a redis connection URI or a host:port pair. URI schemas supported: redis[s]://[[USER][:PASSWORD]@][HOST][:PORT][/DATABASE] or unix://[[USER][:PASSWORD]@]SOCKET_PATH[?db=DATABASE] ** Enterprise only **")
 	groupName         = flag.String("cache.distributed_cache.group_name", "", "A unique name for this distributed cache group. ** Enterprise only **")
-	nodes             = flagutil.Slice("cache.distributed_cache.nodes", []string{}, "The hardcoded list of peer distributed cache nodes. If this is set, redis_target will be ignored. ** Enterprise only **")
+	nodes             = flagtypes.Slice("cache.distributed_cache.nodes", []string{}, "The hardcoded list of peer distributed cache nodes. If this is set, redis_target will be ignored. ** Enterprise only **")
 	replicationFactor = flag.Int("cache.distributed_cache.replication_factor", 0, "How many total servers the data should be replicated to. Must be >= 1. ** Enterprise only **")
 	clusterSize       = flag.Int("cache.distributed_cache.cluster_size", 0, "The total number of nodes in this cluster. Required for health checking. ** Enterprise only **")
 	enableLocalWrites = flag.Bool("cache.distributed_cache.enable_local_writes", false, "If enabled, shortcuts distributed writes that belong to the local shard to local cache instead of making an RPC.")

enterprise/server/backends/memcache/BUILD

@@ -14,7 +14,7 @@ go_library(
         "//server/environment",
         "//server/interfaces",
         "//server/remote_cache/digest",
-        "//server/util/flagutil",
+        "//server/util/flagutil/types",
         "//server/util/log",
         "//server/util/prefix",
         "//server/util/status",

enterprise/server/backends/memcache/memcache.go

@@ -12,16 +12,16 @@ import (
 	"github.com/buildbuddy-io/buildbuddy/server/environment"
 	"github.com/buildbuddy-io/buildbuddy/server/interfaces"
 	"github.com/buildbuddy-io/buildbuddy/server/remote_cache/digest"
-	"github.com/buildbuddy-io/buildbuddy/server/util/flagutil"
 	"github.com/buildbuddy-io/buildbuddy/server/util/log"
 	"github.com/buildbuddy-io/buildbuddy/server/util/prefix"
 	"github.com/buildbuddy-io/buildbuddy/server/util/status"
 
 	repb "github.com/buildbuddy-io/buildbuddy/proto/remote_execution"
+	flagtypes "github.com/buildbuddy-io/buildbuddy/server/util/flagutil/types"
 	"golang.org/x/sync/errgroup"
 )
 
-var memcacheTargets = flagutil.Slice("cache.memcache_targets", []string{}, "Deprecated. Use Redis Target instead.")
+var memcacheTargets = flagtypes.Slice("cache.memcache_targets", []string{}, "Deprecated. Use Redis Target instead.")
 
 const (
 	mcCutoffSizeBytes = 134217728 - 1 // 128 MB

enterprise/server/backends/pebble_cache/BUILD

@@ -15,7 +15,7 @@ go_library(
         "//server/interfaces",
         "//server/remote_cache/digest",
         "//server/util/disk",
-        "//server/util/flagutil",
+        "//server/util/flagutil/types",
         "//server/util/log",
         "//server/util/status",
         "@com_github_cespare_xxhash_v2//:xxhash",

enterprise/server/backends/pebble_cache/pebble_cache.go

@@ -21,7 +21,6 @@ import (
 	"github.com/buildbuddy-io/buildbuddy/server/interfaces"
 	"github.com/buildbuddy-io/buildbuddy/server/remote_cache/digest"
 	"github.com/buildbuddy-io/buildbuddy/server/util/disk"
-	"github.com/buildbuddy-io/buildbuddy/server/util/flagutil"
 	"github.com/buildbuddy-io/buildbuddy/server/util/log"
 	"github.com/buildbuddy-io/buildbuddy/server/util/status"
 	"github.com/cespare/xxhash/v2"
@@ -32,13 +31,14 @@ import (
 	rfpb "github.com/buildbuddy-io/buildbuddy/proto/raft"
 	repb "github.com/buildbuddy-io/buildbuddy/proto/remote_execution"
 	cache_config "github.com/buildbuddy-io/buildbuddy/server/cache/config"
+	flagtypes "github.com/buildbuddy-io/buildbuddy/server/util/flagutil/types"
 )
 
 var (
 	rootDirectory       = flag.String("cache.pebble.root_directory", "", "The root directory to store the database in.")
 	blockCacheSizeBytes = flag.Int64("cache.pebble.block_cache_size_bytes", 1000*megabyte, "How much ram to give the block cache")
-	partitions          = flagutil.Slice("cache.pebble.partitions", []disk.Partition{}, "")
-	partitionMappings   = flagutil.Slice("cache.pebble.partition_mappings", []disk.PartitionMapping{}, "")
+	partitions          = flagtypes.Slice("cache.pebble.partitions", []disk.Partition{}, "")
+	partitionMappings   = flagtypes.Slice("cache.pebble.partition_mappings", []disk.PartitionMapping{}, "")
 )
 
 // TODO:

enterprise/server/backends/redis_client/BUILD

@@ -9,7 +9,7 @@ go_library(
         "//enterprise/server/remote_execution/config",
         "//enterprise/server/util/redisutil",
         "//server/environment",
-        "//server/util/flagutil",
+        "//server/util/flagutil/types",
         "//server/util/status",
     ],
 )

enterprise/server/backends/redis_client/redis_client.go

@@ -6,29 +6,29 @@ import (
 
 	"github.com/buildbuddy-io/buildbuddy/enterprise/server/util/redisutil"
 	"github.com/buildbuddy-io/buildbuddy/server/environment"
-	"github.com/buildbuddy-io/buildbuddy/server/util/flagutil"
 	"github.com/buildbuddy-io/buildbuddy/server/util/status"
 
 	remote_execution_config "github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_execution/config"
+	flagtypes "github.com/buildbuddy-io/buildbuddy/server/util/flagutil/types"
 )
 
 var (
 	defaultRedisTarget          = flag.String("app.default_redis_target", "", "A Redis target for storing remote shared state. To ease migration, the redis target from the remote execution config will be used if this value is not specified.")
-	defaultRedisShards          = flagutil.Slice("app.default_sharded_redis.shards", []string{}, "Ordered list of Redis shard addresses.")
+	defaultRedisShards          = flagtypes.Slice("app.default_sharded_redis.shards", []string{}, "Ordered list of Redis shard addresses.")
 	defaultShardedRedisUsername = flag.String("app.default_sharded_redis.username", "", "Redis username")
 	defaultShardedRedisPassword = flag.String("app.default_sharded_redis.password", "", "Redis password")
 
 	// Cache Redis
 	// TODO: We need to deprecate one of the redis targets here or distinguish them
 	cacheRedisTargetFallback  = flag.String("cache.redis_target", "", "A redis target for improved Caching/RBE performance. Target can be provided as either a redis connection URI or a host:port pair. URI schemas supported: redis[s]://[[USER][:PASSWORD]@][HOST][:PORT][/DATABASE] or unix://[[USER][:PASSWORD]@]SOCKET_PATH[?db=DATABASE] ** Enterprise only **")
 	cacheRedisTarget          = flag.String("cache.redis.redis_target", "", "A redis target for improved Caching/RBE performance. Target can be provided as either a redis connection URI or a host:port pair. URI schemas supported: redis[s]://[[USER][:PASSWORD]@][HOST][:PORT][/DATABASE] or unix://[[USER][:PASSWORD]@]SOCKET_PATH[?db=DATABASE] ** Enterprise only **")
-	cacheRedisShards          = flagutil.Slice("cache.redis.sharded.shards", []string{}, "Ordered list of Redis shard addresses.")
+	cacheRedisShards          = flagtypes.Slice("cache.redis.sharded.shards", []string{}, "Ordered list of Redis shard addresses.")
 	cacheShardedRedisUsername = flag.String("cache.redis.sharded.username", "", "Redis username")
 	cacheShardedRedisPassword = flag.String("cache.redis.sharded.password", "", "Redis password")
 
 	// Remote Execution Redis
 	remoteExecRedisTarget          = flag.String("remote_execution.redis_target", "", "A Redis target for storing remote execution state. Falls back to app.default_redis_target if unspecified. Required for remote execution. To ease migration, the redis target from the cache config will be used if neither this value nor app.default_redis_target are specified.")
-	remoteExecRedisShards          = flagutil.Slice("remote_execution.sharded_redis.shards", []string{}, "Ordered list of Redis shard addresses.")
+	remoteExecRedisShards          = flagtypes.Slice("remote_execution.sharded_redis.shards", []string{}, "Ordered list of Redis shard addresses.")
 	remoteExecShardedRedisUsername = flag.String("remote_execution.sharded_redis.username", "", "Redis username")
 	remoteExecShardedRedisPassword = flag.String("remote_execution.sharded_redis.password", "", "Redis password")
 )

enterprise/server/cmd/ci_runner/BUILD

@@ -22,7 +22,7 @@ go_library(
         "//server/remote_cache/digest",
         "//server/util/bazel",
         "//server/util/bazelisk",
-        "//server/util/flagutil",
+        "//server/util/flagutil/types",
         "//server/util/git",
         "//server/util/grpc_client",
         "//server/util/lockingbuffer",

enterprise/server/cmd/ci_runner/main.go

@@ -25,7 +25,6 @@ import (
 	"github.com/buildbuddy-io/buildbuddy/server/remote_cache/digest"
 	"github.com/buildbuddy-io/buildbuddy/server/util/bazel"
 	"github.com/buildbuddy-io/buildbuddy/server/util/bazelisk"
-	"github.com/buildbuddy-io/buildbuddy/server/util/flagutil"
 	"github.com/buildbuddy-io/buildbuddy/server/util/grpc_client"
 	"github.com/buildbuddy-io/buildbuddy/server/util/lockingbuffer"
 	"github.com/buildbuddy-io/buildbuddy/server/util/log"
@@ -42,6 +41,7 @@ import (
 
 	bespb "github.com/buildbuddy-io/buildbuddy/proto/build_event_stream"
 	repb "github.com/buildbuddy-io/buildbuddy/proto/remote_execution"
+	flagtypes "github.com/buildbuddy-io/buildbuddy/server/util/flagutil/types"
 	gitutil "github.com/buildbuddy-io/buildbuddy/server/util/git"
 	bspb "google.golang.org/genproto/googleapis/bytestream"
 	gstatus "google.golang.org/grpc/status"
@@ -113,7 +113,7 @@ var (
 	invocationID       = flag.String("invocation_id", "", "If set, use the specified invocation ID for the workflow action. Ignored if action_name is not set.")
 	visibility         = flag.String("visibility", "", "If set, use the specified value for VISIBILITY build metadata for the workflow invocation.")
 	bazelSubCommand    = flag.String("bazel_sub_command", "", "If set, run the bazel command specified by these args and ignore all triggering and configured actions.")
-	patchDigests       = flagutil.Slice("patch_digest", []string{}, "Digests of patches to apply to the repo after checkout. Can be specified multiple times to apply multiple patches.")
+	patchDigests       = flagtypes.Slice("patch_digest", []string{}, "Digests of patches to apply to the repo after checkout. Can be specified multiple times to apply multiple patches.")
 	recordRunMetadata  = flag.Bool("record_run_metadata", false, "Instead of running a target, extract metadata about it and report it in the build event stream.")
 
 	shutdownAndExit = flag.Bool("shutdown_and_exit", false, "If set, runs bazel shutdown with the configured bazel_command, and exits. No other commands are run.")

enterprise/server/cmd/executor/BUILD

@@ -31,7 +31,7 @@ go_library(
         "//server/remote_cache/content_addressable_storage_server",
         "//server/resources",
         "//server/util/fileresolver",
-        "//server/util/flagutil",
+        "//server/util/flagutil/yaml",
         "//server/util/grpc_client",
         "//server/util/grpc_server",
         "//server/util/healthcheck",

enterprise/server/cmd/executor/executor.go

@@ -29,7 +29,6 @@ import (
 	"github.com/buildbuddy-io/buildbuddy/server/remote_cache/content_addressable_storage_server"
 	"github.com/buildbuddy-io/buildbuddy/server/resources"
 	"github.com/buildbuddy-io/buildbuddy/server/util/fileresolver"
-	"github.com/buildbuddy-io/buildbuddy/server/util/flagutil"
 	"github.com/buildbuddy-io/buildbuddy/server/util/grpc_client"
 	"github.com/buildbuddy-io/buildbuddy/server/util/grpc_server"
 	"github.com/buildbuddy-io/buildbuddy/server/util/healthcheck"
@@ -48,6 +47,7 @@ import (
 	remote_executor "github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_execution/executor"
 	repb "github.com/buildbuddy-io/buildbuddy/proto/remote_execution"
 	scpb "github.com/buildbuddy-io/buildbuddy/proto/scheduler"
+	flagyaml "github.com/buildbuddy-io/buildbuddy/server/util/flagutil/yaml"
 	bspb "google.golang.org/genproto/googleapis/bytestream"
 	_ "google.golang.org/grpc/encoding/gzip" // imported for side effects; DO NOT REMOVE.
 )
@@ -200,7 +200,7 @@ func main() {
 	rootContext := context.Background()
 
 	flag.Parse()
-	if err := flagutil.PopulateFlagsFromFile(config.Path()); err != nil {
+	if err := flagyaml.PopulateFlagsFromFile(config.Path()); err != nil {
 		log.Fatalf("Error loading config from file: %s", err)
 	}
 

enterprise/server/cmd/executor/yaml_doc/BUILD

@@ -13,7 +13,7 @@ go_library(
     visibility = ["//visibility:private"],
     deps = [
         "//enterprise/server/cmd/executor:executor_lib",
-        "//server/util/flagutil",
+        "//server/util/flagutil/yaml",
     ],
 )
 

enterprise/server/cmd/executor/yaml_doc/main.go

@@ -6,21 +6,21 @@ import (
 	"os"
 
 	_ "github.com/buildbuddy-io/buildbuddy/enterprise/server/cmd/executor"
-	"github.com/buildbuddy-io/buildbuddy/server/util/flagutil"
+	flagyaml "github.com/buildbuddy-io/buildbuddy/server/util/flagutil/yaml"
 )
 
 const flagName = "yaml_documented_defaults_out_file"
 
 var yamlDefaultsOutFile = flag.String(flagName, "buildbuddy_executor_documented_defaults.yaml", "Path to a file to write the default YAML config (with docs) to.")
 
 func init() {
-	flagutil.IgnoreFlagForYAML(flagName)
+	flagyaml.IgnoreFlagForYAML(flagName)
 }
 
 func main() {
 	flag.Parse()
 
-	b, err := flagutil.SplitDocumentedYAMLFromFlags()
+	b, err := flagyaml.SplitDocumentedYAMLFromFlags()
 	if err != nil {
 		log.Fatalf("Encountered error generating documented default YAML file: %s", err)
 	}

enterprise/server/cmd/server/BUILD

@@ -49,6 +49,7 @@ go_library(
         "//server/telemetry",
         "//server/util/fileresolver",
         "//server/util/flagutil",
+        "//server/util/flagutil/yaml",
         "//server/util/healthcheck",
         "//server/util/log",
         "//server/util/tracing",

enterprise/server/cmd/server/main.go

@@ -51,6 +51,7 @@ import (
 	remote_execution_redis_client "github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_execution/redis_client"
 	telserver "github.com/buildbuddy-io/buildbuddy/enterprise/server/telemetry"
 	workflow "github.com/buildbuddy-io/buildbuddy/enterprise/server/workflow/service"
+	flagyaml "github.com/buildbuddy-io/buildbuddy/server/util/flagutil/yaml"
 )
 
 var serverType = flag.String("server_type", "buildbuddy-server", "The server type to match on health checks")
@@ -151,7 +152,7 @@ func main() {
 	version.Print()
 
 	flag.Parse()
-	if err := flagutil.PopulateFlagsFromFile(config.Path()); err != nil {
+	if err := flagyaml.PopulateFlagsFromFile(config.Path()); err != nil {
 		log.Fatalf("Error loading config from file: %s", err)
 	}
 	healthChecker := healthcheck.NewHealthChecker(*serverType)

enterprise/server/cmd/server/yaml_doc/BUILD

@@ -13,7 +13,7 @@ go_library(
     visibility = ["//visibility:private"],
     deps = [
         "//enterprise/server/cmd/server:server_lib",
-        "//server/util/flagutil",
+        "//server/util/flagutil/yaml",
     ],
 )
 

enterprise/server/cmd/server/yaml_doc/main.go

@@ -6,21 +6,21 @@ import (
 	"os"
 
 	_ "github.com/buildbuddy-io/buildbuddy/enterprise/server/cmd/server"
-	"github.com/buildbuddy-io/buildbuddy/server/util/flagutil"
+	flagyaml "github.com/buildbuddy-io/buildbuddy/server/util/flagutil/yaml"
 )
 
 const flagName = "yaml_documented_defaults_out_file"
 
 var yamlDefaultsOutFile = flag.String(flagName, "buildbuddy_enterprise_server_documented_defaults.yaml", "Path to a file to write the default YAML config (with docs) to.")
 
 func init() {
-	flagutil.IgnoreFlagForYAML(flagName)
+	flagyaml.IgnoreFlagForYAML(flagName)
 }
 
 func main() {
 	flag.Parse()
 
-	b, err := flagutil.SplitDocumentedYAMLFromFlags()
+	b, err := flagyaml.SplitDocumentedYAMLFromFlags()
 	if err != nil {
 		log.Fatalf("Encountered error generating documented default YAML file: %s", err)
 	}

enterprise/server/raft/cache/BUILD

@@ -25,7 +25,7 @@ go_library(
         "//server/interfaces",
         "//server/remote_cache/digest",
         "//server/util/disk",
-        "//server/util/flagutil",
+        "//server/util/flagutil/types",
         "//server/util/log",
         "//server/util/network",
         "//server/util/status",

enterprise/server/raft/cache/cache.go

@@ -26,7 +26,6 @@ import (
 	"github.com/buildbuddy-io/buildbuddy/server/interfaces"
 	"github.com/buildbuddy-io/buildbuddy/server/remote_cache/digest"
 	"github.com/buildbuddy-io/buildbuddy/server/util/disk"
-	"github.com/buildbuddy-io/buildbuddy/server/util/flagutil"
 	"github.com/buildbuddy-io/buildbuddy/server/util/log"
 	"github.com/buildbuddy-io/buildbuddy/server/util/network"
 	"github.com/buildbuddy-io/buildbuddy/server/util/status"
@@ -37,13 +36,14 @@ import (
 	rfpb "github.com/buildbuddy-io/buildbuddy/proto/raft"
 	rfspb "github.com/buildbuddy-io/buildbuddy/proto/raft_service"
 	repb "github.com/buildbuddy-io/buildbuddy/proto/remote_execution"
+	flagtypes "github.com/buildbuddy-io/buildbuddy/server/util/flagutil/types"
 	dbConfig "github.com/lni/dragonboat/v3/config"
 )
 
 var (
 	rootDirectory = flag.String("cache.raft.root_directory", "", "The root directory to use for storing cached data.")
 	listenAddr    = flag.String("cache.raft.listen_addr", "", "The address to listen for local gossip traffic on. Ex. 'localhost:1991")
-	join          = flagutil.Slice("cache.raft.join", []string{}, "The list of nodes to use when joining clusters Ex. '1.2.3.4:1991,2.3.4.5:1991...'")
+	join          = flagtypes.Slice("cache.raft.join", []string{}, "The list of nodes to use when joining clusters Ex. '1.2.3.4:1991,2.3.4.5:1991...'")
 	httpPort      = flag.Int("cache.raft.http_port", 0, "The address to listen for HTTP raft traffic. Ex. '1992'")
 	gRPCPort      = flag.Int("cache.raft.grpc_port", 0, "The address to listen for internal API traffic on. Ex. '1993'")
 )