From 37420f7d12e4bde9bca9b658ae054223e0796571 Mon Sep 17 00:00:00 2001 From: ulya-sidorina Date: Tue, 28 Oct 2025 18:22:06 +0100 Subject: [PATCH] refactor(config): remove old format fields --- cmd/ydbcp/config.yaml | 5 +- cmd/ydbcp/main.go | 14 ++-- internal/config/config.go | 66 ------------------- internal/handlers/delete_backup_test.go | 10 +-- internal/handlers/restore_backup_test.go | 14 ++-- internal/handlers/take_backup_test.go | 20 +++--- internal/handlers/utils.go | 2 +- .../backup_schedule_service.go | 6 +- local_config.yaml | 16 +++-- 9 files changed, 46 insertions(+), 107 deletions(-) diff --git a/cmd/ydbcp/config.yaml b/cmd/ydbcp/config.yaml index a619d878..42459a03 100644 --- a/cmd/ydbcp/config.yaml +++ b/cmd/ydbcp/config.yaml @@ -1,5 +1,3 @@ -operation_ttl_seconds: 86400 # 24 hours - db_connection: connection_string: "grpcs://localhost:2135/domain/database" insecure: true @@ -37,3 +35,6 @@ metrics_server: bind_address: 127.0.0.1 #tls_certificate_path: path/to/tls.crt #tls_key_path: path/to/tls.key + +operation_processor: + operation_ttl_seconds: 86400 # 24 hours diff --git a/cmd/ydbcp/main.go b/cmd/ydbcp/main.go index 82c67529..f9b86981 100644 --- a/cmd/ydbcp/main.go +++ b/cmd/ydbcp/main.go @@ -60,10 +60,10 @@ func main() { var wg sync.WaitGroup var logger *xlog.LogConfig - if configInstance.GetDuplicateLogToFile() != "" { - logger, err = xlog.SetupLoggingWithFile(configInstance.GetLogLevel(), configInstance.GetDuplicateLogToFile()) + if configInstance.Log.DuplicateToFile != "" { + logger, err = xlog.SetupLoggingWithFile(configInstance.Log.Level, configInstance.Log.DuplicateToFile) } else { - logger, err = xlog.SetupLogging(configInstance.GetLogLevel()) + logger, err = xlog.SetupLogging(configInstance.Log.Level) } if err != nil { log.Error(err) @@ -116,7 +116,7 @@ func main() { xlog.Info(ctx, "Initialized AuthProvider") metrics.InitializeMetricsRegistry(ctx, &wg, &configInstance.MetricsServer, clockwork.NewRealClock()) xlog.Info(ctx, "Initialized metrics registry") - audit.EventsDestination = configInstance.GetAuditEventsDestination() + audit.EventsDestination = configInstance.Audit.EventsDestination server, err := server.NewServer(&configInstance.GRPCServer, authProvider) if err != nil { xlog.Error(ctx, "failed to initialize GRPC server", zap.Error(err)) @@ -199,11 +199,11 @@ func main() { } processor.NewOperationProcessor( - ctx, &wg, configInstance.GetProcessorIntervalSeconds(), dbConnector, handlersRegistry, + ctx, &wg, configInstance.OperationProcessor.ProcessorIntervalSeconds, dbConnector, handlersRegistry, ) xlog.Info(ctx, "Initialized OperationProcessor") - if configInstance.GetDisableTTLDeletion() { + if configInstance.FeatureFlags.DisableTTLDeletion { xlog.Info(ctx, "TtlWatcher is disabled, old backups won't be deleted") } else { ttl_watcher.NewTtlWatcher(ctx, &wg, dbConnector, queries.NewWriteTableQuery) @@ -213,7 +213,7 @@ func main() { backupScheduleHandler := handlers.NewBackupScheduleHandler(queries.NewWriteTableQuery, clockwork.NewRealClock()) schedule_watcher.NewScheduleWatcher( - ctx, &wg, configInstance.GetProcessorIntervalSeconds(), dbConnector, + ctx, &wg, configInstance.OperationProcessor.ProcessorIntervalSeconds, dbConnector, backupScheduleHandler, clockwork.NewRealClock(), ) xlog.Info(ctx, "Created ScheduleWatcher") diff --git a/internal/config/config.go b/internal/config/config.go index ad7d8a7a..331a9638 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -55,7 +55,6 @@ type GRPCServerConfig struct { BindPort uint16 `yaml:"bind_port" default:"2135"` TLSCertificatePath string `yaml:"tls_certificate_path"` TLSKeyPath string `yaml:"tls_key_path"` - LogLevel string `yaml:"log_level" default:"DEBUG"` } type MetricsServerConfig struct { @@ -104,71 +103,6 @@ type Config struct { Log LogConfig `yaml:"log"` Quota QuotaConfig `yaml:"quota"` FeatureFlags FeatureFlagsConfig `yaml:"feature_flags"` - - // TODO: remove these fields and their getters after migration to the new config format - OperationTtlSeconds int64 `yaml:"operation_ttl_seconds" default:"86400"` - SchedulesLimitPerDB int `yaml:"schedules_limit_per_db" default:"10"` - ProcessorIntervalSeconds int64 `yaml:"processor_interval_seconds" default:"10"` - DisableTTLDeletion bool `yaml:"disable_ttl_deletion" default:"false"` - AuditEventsDestination string `yaml:"audit_events_destination"` - DuplicateLogToFile string `yaml:"duplicate_log_to_file"` -} - -func (c Config) GetOperationTtlSeconds() int64 { - if c.OperationTtlSeconds == 0 { - return c.OperationProcessor.OperationTtlSeconds - } - - return c.OperationTtlSeconds -} - -func (c *Config) SetOperationTtlSeconds(val int64) { - c.OperationTtlSeconds = val - c.OperationProcessor.OperationTtlSeconds = val -} - -func (c Config) GetSchedulesLimitPerDB() int { - if c.SchedulesLimitPerDB == 0 { - return c.Quota.SchedulesPerDB - } - - return c.SchedulesLimitPerDB -} - -func (c Config) GetProcessorIntervalSeconds() int64 { - if c.ProcessorIntervalSeconds == 0 { - return c.OperationProcessor.ProcessorIntervalSeconds - } - - return c.ProcessorIntervalSeconds -} - -func (c Config) GetDisableTTLDeletion() bool { - return c.DisableTTLDeletion || c.FeatureFlags.DisableTTLDeletion -} - -func (c Config) GetAuditEventsDestination() string { - if len(c.AuditEventsDestination) == 0 { - return c.Audit.EventsDestination - } - - return c.AuditEventsDestination -} - -func (c Config) GetDuplicateLogToFile() string { - if len(c.DuplicateLogToFile) == 0 { - return c.Log.DuplicateToFile - } - - return c.DuplicateLogToFile -} - -func (c Config) GetLogLevel() string { - if len(c.GRPCServer.LogLevel) == 0 { - return c.Log.Level - } - - return c.GRPCServer.LogLevel } type ClusterConnectionConfig struct { diff --git a/internal/handlers/delete_backup_test.go b/internal/handlers/delete_backup_test.go index 69e1e9f3..961d695a 100644 --- a/internal/handlers/delete_backup_test.go +++ b/internal/handlers/delete_backup_test.go @@ -52,7 +52,7 @@ func TestDBOperationHandlerDeadlineExceededForRunningOperation(t *testing.T) { s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap) config := config.Config{} - config.SetOperationTtlSeconds(0) + config.OperationProcessor.OperationTtlSeconds = 0 handler := NewDBOperationHandler( dbConnector, s3Connector, config, queries.NewWriteTableQueryMock, ) @@ -124,7 +124,7 @@ func TestDBOperationHandlerPendingOperationCompletedSuccessfully(t *testing.T) { metrics.InitializeMockMetricsRegistry() config := config.Config{} - config.SetOperationTtlSeconds(1000) + config.OperationProcessor.OperationTtlSeconds = 1000 handler := NewDBOperationHandler( dbConnector, s3Connector, config, queries.NewWriteTableQueryMock, ) @@ -204,7 +204,7 @@ func TestDBOperationHandlerRunningOperationCompletedSuccessfully(t *testing.T) { metrics.InitializeMockMetricsRegistry() config := config.Config{} - config.SetOperationTtlSeconds(1000) + config.OperationProcessor.OperationTtlSeconds = 1000 handler := NewDBOperationHandler( dbConnector, s3Connector, config, queries.NewWriteTableQueryMock, ) @@ -281,7 +281,7 @@ func TestDBOperationHandlerUnexpectedBackupStatus(t *testing.T) { s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap) config := config.Config{} - config.SetOperationTtlSeconds(1000) + config.OperationProcessor.OperationTtlSeconds = 1000 handler := NewDBOperationHandler( dbConnector, s3Connector, config, queries.NewWriteTableQueryMock, ) @@ -351,7 +351,7 @@ func TestDBOperationHandlerDeleteMoreThanAllowedLimit(t *testing.T) { metrics.InitializeMockMetricsRegistry() config := config.Config{} - config.SetOperationTtlSeconds(1000) + config.OperationProcessor.OperationTtlSeconds = 1000 handler := NewDBOperationHandler( dbConnector, s3Connector, config, queries.NewWriteTableQueryMock, ) diff --git a/internal/handlers/restore_backup_test.go b/internal/handlers/restore_backup_test.go index 3cf51aeb..e9f4e966 100644 --- a/internal/handlers/restore_backup_test.go +++ b/internal/handlers/restore_backup_test.go @@ -133,7 +133,7 @@ func TestRBOperationHandlerRunningOperationInProgress(t *testing.T) { // try to handle pending rb operation with ttl config := config.Config{} - config.SetOperationTtlSeconds(1000) + config.OperationProcessor.OperationTtlSeconds = 1000 handler := NewRBOperationHandler( dbConnector, clientConnector, @@ -187,7 +187,7 @@ func TestRBOperationHandlerRunningOperationCompletedSuccessfully(t *testing.T) { dbConnector := db.NewMockDBConnector(db.WithOperations(opMap)) config := config.Config{} - config.SetOperationTtlSeconds(1000) + config.OperationProcessor.OperationTtlSeconds = 1000 handler := NewRBOperationHandler( dbConnector, clientConnector, @@ -240,7 +240,7 @@ func TestRBOperationHandlerRunningOperationCancelled(t *testing.T) { dbConnector := db.NewMockDBConnector(db.WithOperations(opMap)) config := config.Config{} - config.SetOperationTtlSeconds(10) + config.OperationProcessor.OperationTtlSeconds = 10 handler := NewRBOperationHandler( dbConnector, clientConnector, @@ -344,7 +344,7 @@ func TestRBOperationHandlerCancellingOperationInProgress(t *testing.T) { dbConnector := db.NewMockDBConnector(db.WithOperations(opMap)) config := config.Config{} - config.SetOperationTtlSeconds(1000) + config.OperationProcessor.OperationTtlSeconds = 1000 handler := NewRBOperationHandler( dbConnector, clientConnector, @@ -398,7 +398,7 @@ func TestRBOperationHandlerCancellingOperationCompletedSuccessfully(t *testing.T dbConnector := db.NewMockDBConnector(db.WithOperations(opMap)) config := config.Config{} - config.SetOperationTtlSeconds(10) + config.OperationProcessor.OperationTtlSeconds = 10 handler := NewRBOperationHandler( dbConnector, clientConnector, @@ -452,7 +452,7 @@ func TestRBOperationHandlerCancellingOperationCancelled(t *testing.T) { dbConnector := db.NewMockDBConnector(db.WithOperations(opMap)) config := config.Config{} - config.SetOperationTtlSeconds(10) + config.OperationProcessor.OperationTtlSeconds = 10 handler := NewRBOperationHandler( dbConnector, clientConnector, @@ -506,7 +506,7 @@ func TestRBOperationHandlerRetriableErrorForRunningOperation(t *testing.T) { dbConnector := db.NewMockDBConnector(db.WithOperations(opMap)) config := config.Config{} - config.SetOperationTtlSeconds(10) + config.OperationProcessor.OperationTtlSeconds = 10 handler := NewRBOperationHandler( dbConnector, clientConnector, diff --git a/internal/handlers/take_backup_test.go b/internal/handlers/take_backup_test.go index 1fd3e5f3..ee6aecc2 100644 --- a/internal/handlers/take_backup_test.go +++ b/internal/handlers/take_backup_test.go @@ -115,7 +115,7 @@ func TestTBOperationHandlerDeadlineExceededForRunningOperation(t *testing.T) { s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap) config := config.Config{} - config.SetOperationTtlSeconds(0) + config.OperationProcessor.OperationTtlSeconds = 0 handler := NewTBOperationHandler( dbConnector, clientConnector, s3Connector, config, queries.NewWriteTableQueryMock, ) @@ -187,7 +187,7 @@ func TestTBOperationHandlerRunningOperationInProgress(t *testing.T) { s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap) config := config.Config{} - config.SetOperationTtlSeconds(10000) + config.OperationProcessor.OperationTtlSeconds = 10000 handler := NewTBOperationHandler( dbConnector, clientConnector, s3Connector, config, queries.NewWriteTableQueryMock, ) @@ -276,7 +276,7 @@ func TestTBOperationHandlerRunningOperationCompletedSuccessfully(t *testing.T) { s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap) metrics.InitializeMockMetricsRegistry() config := config.Config{} - config.SetOperationTtlSeconds(10000) + config.OperationProcessor.OperationTtlSeconds = 10000 handler := NewTBOperationHandler( dbConnector, clientConnector, s3Connector, config, queries.NewWriteTableQueryMock, ) @@ -353,7 +353,7 @@ func TestTBOperationHandlerRunningOperationCancelled(t *testing.T) { s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap) metrics.InitializeMockMetricsRegistry() config := config.Config{} - config.SetOperationTtlSeconds(10000) + config.OperationProcessor.OperationTtlSeconds = 10000 handler := NewTBOperationHandler( dbConnector, clientConnector, s3Connector, config, queries.NewWriteTableQueryMock, ) @@ -429,7 +429,7 @@ func TestTBOperationHandlerDeadlineExceededForCancellingOperation(t *testing.T) s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap) metrics.InitializeMockMetricsRegistry() config := config.Config{} - config.SetOperationTtlSeconds(0) + config.OperationProcessor.OperationTtlSeconds = 0 handler := NewTBOperationHandler( dbConnector, clientConnector, s3Connector, config, queries.NewWriteTableQueryMock, ) @@ -507,7 +507,7 @@ func TestTBOperationHandlerCancellingOperationInProgress(t *testing.T) { s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap) config := config.Config{} - config.SetOperationTtlSeconds(10000) + config.OperationProcessor.OperationTtlSeconds = 10000 handler := NewTBOperationHandler( dbConnector, clientConnector, s3Connector, config, queries.NewWriteTableQueryMock, ) @@ -595,7 +595,7 @@ func TestTBOperationHandlerCancellingOperationCompletedSuccessfully(t *testing.T s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap) metrics.InitializeMockMetricsRegistry() config := config.Config{} - config.SetOperationTtlSeconds(10000) + config.OperationProcessor.OperationTtlSeconds = 10000 handler := NewTBOperationHandler( dbConnector, clientConnector, s3Connector, config, queries.NewWriteTableQueryMock, ) @@ -672,7 +672,7 @@ func TestTBOperationHandlerCancellingOperationCancelled(t *testing.T) { s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap) metrics.InitializeMockMetricsRegistry() config := config.Config{} - config.SetOperationTtlSeconds(10000) + config.OperationProcessor.OperationTtlSeconds = 10000 handler := NewTBOperationHandler( dbConnector, clientConnector, s3Connector, config, queries.NewWriteTableQueryMock, ) @@ -764,7 +764,7 @@ func TestTBOperationHandlerCancellingOperationCancelledWithRemovingDataFromS3(t s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap) metrics.InitializeMockMetricsRegistry() config := config.Config{} - config.SetOperationTtlSeconds(10000) + config.OperationProcessor.OperationTtlSeconds = 10000 handler := NewTBOperationHandler( dbConnector, clientConnector, s3Connector, config, queries.NewWriteTableQueryMock, ) @@ -845,7 +845,7 @@ func TestTBOperationHandlerRetriableErrorForRunningOperation(t *testing.T) { s3Connector := s3Client.NewMockS3Connector(s3ObjectsMap) config := config.Config{} - config.SetOperationTtlSeconds(10000) + config.OperationProcessor.OperationTtlSeconds = 10000 handler := NewTBOperationHandler( dbConnector, clientConnector, s3Connector, config, queries.NewWriteTableQueryMock, ) diff --git a/internal/handlers/utils.go b/internal/handlers/utils.go index a55b43eb..f26afad0 100644 --- a/internal/handlers/utils.go +++ b/internal/handlers/utils.go @@ -18,7 +18,7 @@ import ( ) func deadlineExceeded(createdAt *timestamppb.Timestamp, config config.Config) bool { - return time.Since(createdAt.AsTime()) > time.Duration(config.GetOperationTtlSeconds())*time.Second + return time.Since(createdAt.AsTime()) > time.Duration(config.OperationProcessor.OperationTtlSeconds)*time.Second } func isValidStatus(status Ydb.StatusIds_StatusCode) bool { return status == Ydb.StatusIds_SUCCESS || status == Ydb.StatusIds_CANCELLED diff --git a/internal/server/services/backup_schedule/backup_schedule_service.go b/internal/server/services/backup_schedule/backup_schedule_service.go index 747413b8..fe677931 100644 --- a/internal/server/services/backup_schedule/backup_schedule_service.go +++ b/internal/server/services/backup_schedule/backup_schedule_service.go @@ -94,12 +94,12 @@ func (s *BackupScheduleService) CreateBackupSchedule( return nil, status.Error(codes.Internal, "error getting backup schedules") } - if len(schedules)+1 > s.config.GetSchedulesLimitPerDB() { + if len(schedules)+1 > s.config.Quota.SchedulesPerDB { xlog.Error( ctx, "can't create backup schedule, limit exceeded for database", zap.String("database", request.DatabaseName), zap.String("container", request.ContainerId), - zap.Int("limit", s.config.GetSchedulesLimitPerDB()), + zap.Int("limit", s.config.Quota.SchedulesPerDB), ) s.IncApiCallsCounter(methodName, codes.FailedPrecondition) return nil, status.Errorf( @@ -107,7 +107,7 @@ func (s *BackupScheduleService) CreateBackupSchedule( "can't create backup schedule, limit exceeded for database: %s, container: %s, limit: %d", request.DatabaseName, request.ContainerId, - s.config.GetSchedulesLimitPerDB(), + s.config.Quota.SchedulesPerDB, ) } diff --git a/local_config.yaml b/local_config.yaml index f79b8c2c..37ac78de 100644 --- a/local_config.yaml +++ b/local_config.yaml @@ -1,5 +1,3 @@ -operation_ttl_seconds: 86400 # 24 hours - db_connection: connection_string: "grpcs://${YDB_NAME}:2135/local" insecure: true @@ -24,15 +22,21 @@ s3: grpc_server: bind_port: 50051 - log_level: INFO metrics_server: bind_address: 127.0.0.1 bind_port: 9090 -schedules_limit_per_db: 1 -processor_interval_seconds: 2 -duplicate_log_to_file: /var/log/main.log +operation_processor: + operation_ttl_seconds: 86400 # 24 hours + processor_interval_seconds: 2 + +quota: + schedules_per_db: 1 + +log: + duplicate_to_file: /var/log/main.log + level: INFO feature_flags: enable_new_paths_format: ${ENABLE_NEW_PATHS_FORMAT} \ No newline at end of file