From 714dfd2c0b9775251036905cb2f803e81d14f861 Mon Sep 17 00:00:00 2001 From: Stuart Douglas Date: Tue, 26 Mar 2024 07:49:29 +1100 Subject: [PATCH] alllow for database connection, grpc worker thread pool, and k8s client level tuning of the watcher and api servers - add open and idle database connection pool settings to the config.env file - but don't change golang's database/sql defaults - add the ablity to increase GRPC worker thread poll count - add the ability set set K8s client QPS and burst settings to the api sever via config.env and to the watcher via command line arguments rh-pre-commit.version: 2.2.0 rh-pre-commit.check-secrets: ENABLED --- cmd/api/README.md | 67 ++++++++++++++++++--------------- cmd/api/main.go | 45 +++++++++++++++++++++- cmd/watcher/main.go | 18 ++++++++- config/base/env/config | 5 +++ pkg/api/server/config/config.go | 6 +++ 5 files changed, 107 insertions(+), 34 deletions(-) diff --git a/cmd/api/README.md b/cmd/api/README.md index 6a9d07c82..cd1e6f556 100644 --- a/cmd/api/README.md +++ b/cmd/api/README.md @@ -2,37 +2,42 @@ ## Variables -| Environment Variable | Description | Example | -|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------| -| DB_USER | Postgres Database user | user | -| DB_PASSWORD | Postgres Database Password | hunter2 | -| DB_HOST | Postgres Database host | /cloudsql/my-project:us-east1:tekton-results | -| DB_NAME | Postgres Database name | tekton_results | -| DB_SSLMODE | Database SSL mode | verify-full | -| DB_SSLROOTCERT | Path to CA cert used to validate Database cert | /etc/tls/db/ca.crt | -| DB_ENABLE_AUTO_MIGRATION | Auto-migrate the database on startup (create/update schemas). For further details, refer to | true (default) | -| PROFILING | Enable profiling server | false (default) | -| PROFILING_PORT | Profiling Server Port | 6060 (default) | -| SERVER_PORT | gRPC and REST Server Port | 8080 (default) | -| PROMETHEUS_PORT | Prometheus Port | 9090 (default) | -| PROMETHEUS_HISTOGRAM | Enable Prometheus histogram metrics to measure latency distributions of RPCs | false (default) | -| TLS_PATH | Path to TLS files | /etc/tls | -| AUTH_DISABLE | Disable RBAC check for resources | false (default) | -| AUTH_IMPERSONATE | Enable RBAC impersonation | true (default) | -| LOG_LEVEL | Log level for api server | info (default) | -| LOGS_API | Enable logs storage service | false (default) | -| LOGS_TYPE | Determine Logs storage backend type | File (default) | -| LOGS_BUFFER_SIZE | Buffer for streaming logs | 32768 (default) | -| LOGS_PATH | Logs storage path | logs (default) | -| S3_BUCKET_NAME | S3 Bucket name | | -| S3_ENDPOINT | S3 Endpoint | https://s3.ap-south-1.amazonaws.com | -| S3_HOSTNAME_IMMUTABLE | S3 Hostname immutable | false (default) | -| S3_REGION | S3 Region | ap-south-1 | -| S3_ACCESS_KEY_ID | S3 Access Key ID | | -| S3_SECRET_ACCESS_KEY | S3 Secret Access Key | | -| S3_MULTI_PART_SIZE | S3 Multi part size | 5242880 (default) | -| GCS_BUCKET_NAME | GCS Bucket Name | | -| STORAGE_EMULATOR_HOST | GCS Storage Emulator Server | http://localhost:9004 | +| Environment Variable | Description | Example | +|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------| +| DB_USER | Postgres Database user | user | +| DB_PASSWORD | Postgres Database Password | hunter2 | +| DB_HOST | Postgres Database host | /cloudsql/my-project:us-east1:tekton-results | +| DB_NAME | Postgres Database name | tekton_results | +| DB_SSLMODE | Database SSL mode | verify-full | +| DB_SSLROOTCERT | Path to CA cert used to validate Database cert | /etc/tls/db/ca.crt | +| DB_ENABLE_AUTO_MIGRATION | Auto-migrate the database on startup (create/update schemas). For further details, refer to | true (default) | +| PROFILING | Enable profiling server | false (default) | +| PROFILING_PORT | Profiling Server Port | 6060 (default) | +| DB_MAX_IDLE_CONNECTIONS | The number of idle database connections to keep open | 2 (default for golang, but specific database drivers may have settings for this too) | +| DB_MAX_OPEN_CONNECTIONS | The maximum number of database connections, for best performance it should equal DB_MAX_IDLE_CONNECTIONS | unlimited (default for golang, but specific database drivers may have settings for this too) | +| GRPC_WORKER_POOL | The maximum number of goroutines pre-allocated for process GRPC requests. The GRPC server will also dynamically create threads. | 2 (default) | +| K8S_QPS | The QPS setting for the kubernetes client created. | 5 (default) | +| K8S_BURST | The burst setting for the kubernetes client created. | 10 (default) | +| SERVER_PORT | gRPC and REST Server Port | 8080 (default) | +| PROMETHEUS_PORT | Prometheus Port | 9090 (default) | +| PROMETHEUS_HISTOGRAM | Enable Prometheus histogram metrics to measure latency distributions of RPCs | false (default) | +| TLS_PATH | Path to TLS files | /etc/tls | +| AUTH_DISABLE | Disable RBAC check for resources | false (default) | +| AUTH_IMPERSONATE | Enable RBAC impersonation | true (default) | +| LOG_LEVEL | Log level for api server | info (default) | +| LOGS_API | Enable logs storage service | false (default) | +| LOGS_TYPE | Determine Logs storage backend type | File (default) | +| LOGS_BUFFER_SIZE | Buffer for streaming logs | 32768 (default) | +| LOGS_PATH | Logs storage path | logs (default) | +| S3_BUCKET_NAME | S3 Bucket name | | +| S3_ENDPOINT | S3 Endpoint | https://s3.ap-south-1.amazonaws.com | +| S3_HOSTNAME_IMMUTABLE | S3 Hostname immutable | false (default) | +| S3_REGION | S3 Region | ap-south-1 | +| S3_ACCESS_KEY_ID | S3 Access Key ID | | +| S3_SECRET_ACCESS_KEY | S3 Secret Access Key | | +| S3_MULTI_PART_SIZE | S3 Multi part size | 5242880 (default) | +| GCS_BUCKET_NAME | GCS Bucket Name | | +| STORAGE_EMULATOR_HOST | GCS Storage Emulator Server | http://localhost:9004 | These values can also be set in the config file located in the `config/env/config` directory. diff --git a/cmd/api/main.go b/cmd/api/main.go index bf3b45fc2..26acbedfc 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -19,6 +19,7 @@ package main import ( "context" "crypto/tls" + "database/sql" "fmt" "net/http" "path" @@ -119,6 +120,33 @@ func main() { log.Fatalf("Failed to connect to database: %v", err) } + var sqlDB *sql.DB + + // Set DB connection limits + maxIdle := serverConfig.DB_MAX_IDLE_CONNECTIONS + maxOpen := serverConfig.DB_MAX_OPEN_CONNECTIONS + if maxOpen > 0 { + sqlDB, err = db.DB() + if err != nil { + log.Fatalf("Error getting database configuration for updating max open connections: %s", err.Error()) + } + sqlDB.SetMaxOpenConns(maxOpen) + } + if maxIdle > 0 { + sqlDB, err = db.DB() + if err != nil { + log.Fatalf("Error getting database configuration for updating max open connections: %s", err.Error()) + } + sqlDB.SetMaxIdleConns(maxIdle) + } + + // Set grpc worker pool + grpcWorkers := serverConfig.GRPC_WORKER_POOL + var streamWorkers grpc.ServerOption + if grpcWorkers > 2 { + streamWorkers = grpc.NumStreamWorkers((uint32)(grpcWorkers)) + } + // Create the authorization authCheck var authCheck auth.Checker var serverMuxOptions []runtime.ServeMuxOption @@ -132,6 +160,15 @@ func main() { if err != nil { log.Fatal("Error getting kubernetes client config:", err) } + // Override k8s client qps/burts settings + qps := serverConfig.K8S_QPS + burst := serverConfig.K8S_BURST + if qps > 0 { + k8sConfig.QPS = (float32)(qps) + } + if burst > 0 { + k8sConfig.Burst = burst + } k8s, err := kubernetes.NewForConfig(k8sConfig) if err != nil { log.Fatal("Error creating kubernetes clientset:", err) @@ -163,7 +200,7 @@ func main() { // Customize logger, so it can be passed to the gRPC interceptors grpcLogger := log.Desugar().With(zap.Bool("grpc.auth_disabled", serverConfig.AUTH_DISABLE)) - gs := grpc.NewServer( + svrOpts := []grpc.ServerOption{ grpc.Creds(creds), grpc_middleware.WithUnaryServerChain( // The grpc_ctxtags context updater should be before everything else @@ -181,7 +218,11 @@ func main() { prometheus.StreamServerInterceptor, recovery.StreamServerInterceptor(recovery.WithRecoveryHandler(recoveryHandler)), ), - ) + } + if streamWorkers != nil { + svrOpts = append(svrOpts, streamWorkers) + } + gs := grpc.NewServer(svrOpts...) v1alpha2pb.RegisterResultsServer(gs, v1a2) if serverConfig.LOGS_API { v1alpha2pb.RegisterLogsServer(gs, v1a2) diff --git a/cmd/watcher/main.go b/cmd/watcher/main.go index bbb686daa..bdafa4ed1 100644 --- a/cmd/watcher/main.go +++ b/cmd/watcher/main.go @@ -40,6 +40,7 @@ import ( "google.golang.org/grpc/credentials/oauth" corev1 "k8s.io/api/core/v1" _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/client-go/rest" "k8s.io/client-go/transport" "knative.dev/pkg/configmap" "knative.dev/pkg/controller" @@ -62,6 +63,8 @@ var ( authToken = flag.String("token", "", "Authentication token to use in requests. If not specified, on-cluster configuration is assumed.") completedRunGracePeriod = flag.Duration("completed_run_grace_period", 0, "Grace period duration before Runs should be deleted. If 0, Runs will not be deleted. If < 0, Runs will be deleted immediately.") threadiness = flag.Int("threadiness", controller.DefaultThreadsPerController, "Number of threads (Go routines) allocated to each controller") + qps = flag.Float64("qps", float64(rest.DefaultQPS), "Kubernetes client QPS setting") + burst = flag.Int("burst", rest.DefaultBurst, "Kubernetes client Burst setting") logsAPI = flag.Bool("logs_api", true, "Disable sending logs. If not set, the logs will be sent only if server support API for it") labelSelector = flag.String("label_selector", "", "Selector (label query) to filter objects to be deleted. Matching objects must satisfy all labels requirements to be eligible for deletion") requeueInterval = flag.Duration("requeue_interval", 10*time.Minute, "How long the Watcher waits to reprocess keys on certain events (e.g. an object doesn't match the provided selectors)") @@ -109,12 +112,25 @@ func main() { } } - sharedmain.MainWithContext(injection.WithNamespaceScope(ctx, *namespace), "watcher", + ctors := []injection.ControllerConstructor{ func(ctx context.Context, cmw configmap.Watcher) *controller.Impl { return pipelinerun.NewControllerWithConfig(ctx, results, cfg, cmw) }, func(ctx context.Context, cmw configmap.Watcher) *controller.Impl { return taskrun.NewControllerWithConfig(ctx, results, cfg, cmw) }, + } + + // This parses flags. + k8scfg := injection.ParseAndGetRESTConfigOrDie() + + if qps != nil { + k8scfg.QPS = float32(*qps) * float32(len(ctors)) + } + if burst != nil { + k8scfg.Burst = *burst * len(ctors) + } + + sharedmain.MainWithConfig(injection.WithNamespaceScope(ctx, *namespace), "watcher", k8scfg, ctors..., ) } diff --git a/config/base/env/config b/config/base/env/config index 87d83aec8..d45307dc3 100644 --- a/config/base/env/config +++ b/config/base/env/config @@ -6,6 +6,11 @@ DB_NAME=tekton-results DB_SSLMODE=disable DB_SSLROOTCERT= DB_ENABLE_AUTO_MIGRATION=true +DB_MAX_IDLE_CONNECTIONS=10 +DB_MAX_OPEN_CONNECTIONS=10 +GRPC_WORKER_POOL=2 +K8S_QPS=5 +K8S_BURST=10 PROFILING=false PROFILING_PORT=6060 SERVER_PORT=8080 diff --git a/pkg/api/server/config/config.go b/pkg/api/server/config/config.go index 1068ed808..49c788bdc 100644 --- a/pkg/api/server/config/config.go +++ b/pkg/api/server/config/config.go @@ -15,12 +15,18 @@ type Config struct { DB_SSLMODE string `mapstructure:"DB_SSLMODE"` DB_SSLROOTCERT string `mapstructure:"DB_SSLROOTCERT"` DB_ENABLE_AUTO_MIGRATION bool `mapstructure:"DB_ENABLE_AUTO_MIGRATION"` + DB_MAX_IDLE_CONNECTIONS int `mapstructure:"DB_MAX_IDLE_CONNECTIONS"` + DB_MAX_OPEN_CONNECTIONS int `mapstructure:"DB_MAX_OPEN_CONNECTIONS"` SERVER_PORT string `mapstructure:"SERVER_PORT"` PROMETHEUS_PORT string `mapstructure:"PROMETHEUS_PORT"` PROMETHEUS_HISTOGRAM bool `mapstructure:"PROMETHEUS_HISTOGRAM"` LOG_LEVEL string `mapstructure:"LOG_LEVEL"` TLS_PATH string `mapstructure:"TLS_PATH"` + GRPC_WORKER_POOL int `mapstructure:"GRPC_WORKER_POOL"` + K8S_QPS int `mapstructure:"K8S_QPS"` + K8S_BURST int `mapstructure:"K8S_BURST"` + AUTH_DISABLE bool `mapstructure:"AUTH_DISABLE"` AUTH_IMPERSONATE bool `mapstructure:"AUTH_IMPERSONATE"`