From 2d27ae5cd3fe55737c2fa02b46616ec09ade47c5 Mon Sep 17 00:00:00 2001 From: Hank Donnay Date: Fri, 23 Oct 2020 11:54:41 -0500 Subject: [PATCH] initialize: add standalone initialization functions Signed-off-by: Hank Donnay --- initialize/logging.go | 13 ++ initialize/services.go | 362 ++++++++++++++++++++--------------------- 2 files changed, 187 insertions(+), 188 deletions(-) diff --git a/initialize/logging.go b/initialize/logging.go index 0fe298ae73..a510616d82 100644 --- a/initialize/logging.go +++ b/initialize/logging.go @@ -7,6 +7,8 @@ import ( "github.com/rs/zerolog" "github.com/rs/zerolog/log" + + "github.com/quay/clair/v4/config" ) // Logging will set the global logging level for Clair, @@ -26,6 +28,7 @@ func (i *Init) Logging() error { return nil } +// LogLevel does a string-to-level mapping. func LogLevel(level string) zerolog.Level { level = strings.ToLower(level) switch level { @@ -49,3 +52,13 @@ func LogLevel(level string) zerolog.Level { return zerolog.InfoLevel } } + +// Logging configures a logger according to the provided configuration and returns +// a configured Context. +func Logging(ctx context.Context, cfg *config.Config) (context.Context, error) { + zerolog.SetGlobalLevel(LogLevel(cfg.LogLevel)) + l := log.With().Timestamp().Logger() + ctx = l.WithContext(ctx) + l.Debug().Str("component", "initialize/Logging").Msg("logging initialized") + return ctx, nil +} diff --git a/initialize/services.go b/initialize/services.go index 5fbdea1853..84c4c9b2f4 100644 --- a/initialize/services.go +++ b/initialize/services.go @@ -1,6 +1,8 @@ package initialize import ( + "context" + "errors" "fmt" "time" @@ -14,233 +16,217 @@ import ( "github.com/quay/clair/v4/config" "github.com/quay/clair/v4/httptransport" "github.com/quay/clair/v4/httptransport/client" + "github.com/quay/clair/v4/indexer" + "github.com/quay/clair/v4/matcher" notifier "github.com/quay/clair/v4/notifier/service" ) const ( - // DefaultUpdatePeriod is the default period used for running updaters - // within matcher processes. - DefaultUpdatePeriod = 30 * time.Minute // NotifierIssuer is the value used for the issuer claim of any outgoing // HTTP requests the notifier makes, if PSK auth is configured. NotifierIssuer = `clair-notifier` ) -// Services will initialize the correct ClairCore services -// dependent on operation mode. +var ( + intraserviceClaim = jwt.Claims{Issuer: httptransport.IntraserviceIssuer} + notifierClaim = jwt.Claims{Issuer: NotifierIssuer} +) + +// Srv is a bundle of configured Services. // -// Services maybe local or remote (over a network). -func (i *Init) Services() error { - log := zerolog.Ctx(i.GlobalCTX).With().Str("component", "init/Init.Services").Logger() +// The members are populated according to the configuration that was passed to +// Services. +type Srv struct { + Indexer indexer.Service + Matcher matcher.Service + Notifier notifier.Service +} + +// Services configures the services needed for a given mode according to the +// provided configuration. +func Services(ctx context.Context, cfg *config.Config) (*Srv, error) { + log := zerolog.Ctx(ctx).With().Str("component", "init/Services").Logger() log.Info().Msg("begin service initialization") + defer log.Info().Msg("end service initialization") - switch i.conf.Mode { + var srv Srv + var err error + switch cfg.Mode { case config.ComboMode: - // configure two local services via claircore libraries - opts := libindex.Opts{ - ConnString: i.conf.Indexer.ConnString, - ScanLockRetry: time.Duration(i.conf.Indexer.ScanLockRetry) * time.Second, - LayerScanConcurrency: i.conf.Indexer.LayerScanConcurrency, - Migrations: i.conf.Indexer.Migrations, - Airgap: i.conf.Indexer.Airgap, - } - if i.conf.Indexer.Scanner.Package != nil { - opts.ScannerConfig.Package = make(map[string]func(interface{}) error, len(i.conf.Indexer.Scanner.Package)) - for name, node := range i.conf.Indexer.Scanner.Package { - opts.ScannerConfig.Package[name] = node.Decode - } - } - if i.conf.Indexer.Scanner.Dist != nil { - opts.ScannerConfig.Dist = make(map[string]func(interface{}) error, len(i.conf.Indexer.Scanner.Dist)) - for name, node := range i.conf.Indexer.Scanner.Dist { - opts.ScannerConfig.Dist[name] = node.Decode - } - } - if i.conf.Indexer.Scanner.Repo != nil { - opts.ScannerConfig.Repo = make(map[string]func(interface{}) error, len(i.conf.Indexer.Scanner.Repo)) - for name, node := range i.conf.Indexer.Scanner.Repo { - opts.ScannerConfig.Repo[name] = node.Decode - } - } - libI, err := libindex.New(i.GlobalCTX, &opts) - if err != nil { - return clairerror.ErrNotInitialized{Msg: "failed to initialize libindex: " + err.Error()} - } - updaterConfigs := make(map[string]driver.ConfigUnmarshaler) - for name, node := range i.conf.Updaters.Config { - updaterConfigs[name] = node.Decode - } - libV, err := libvuln.New(i.GlobalCTX, &libvuln.Opts{ - MaxConnPool: int32(i.conf.Matcher.MaxConnPool), - ConnString: i.conf.Matcher.ConnString, - Migrations: i.conf.Matcher.Migrations, - UpdaterSets: i.conf.Updaters.Sets, - UpdateInterval: i.conf.Matcher.Period, - UpdaterConfigs: updaterConfigs, - UpdateRetention: i.conf.Matcher.UpdateRetention, - }) + srv.Indexer, err = localIndexer(ctx, cfg) if err != nil { - return fmt.Errorf("failed to initialize libvuln: %v", err) + return nil, err } - - c, _, err := i.conf.Client(nil, notifierClaim) + srv.Matcher, err = localMatcher(ctx, cfg) if err != nil { - return err + return nil, err } - - n, err := notifier.New(i.GlobalCTX, notifier.Opts{ - DeliveryInterval: i.conf.Notifier.DeliveryInterval, - ConnString: i.conf.Notifier.ConnString, - Indexer: libI, - Matcher: libV, - Client: c, - Migrations: i.conf.Notifier.Migrations, - PollInterval: i.conf.Notifier.PollInterval, - DisableSummary: i.conf.Notifier.DisableSummary, - Webhook: i.conf.Notifier.Webhook, - AMQP: i.conf.Notifier.AMQP, - STOMP: i.conf.Notifier.STOMP, - }) + srv.Notifier, err = localNotifier(ctx, cfg, srv.Indexer, srv.Matcher) if err != nil { - return &clairerror.ErrNotInitialized{ - Msg: "notifier failed to initialize: " + err.Error(), - } + return nil, err } - - i.Indexer = libI - i.Matcher = libV - i.Notifier = n case config.IndexerMode: - // configure just a local indexer - opts := libindex.Opts{ - ConnString: i.conf.Indexer.ConnString, - ScanLockRetry: time.Duration(i.conf.Indexer.ScanLockRetry) * time.Second, - LayerScanConcurrency: i.conf.Indexer.LayerScanConcurrency, - Migrations: i.conf.Indexer.Migrations, - Airgap: i.conf.Indexer.Airgap, - } - if i.conf.Indexer.Scanner.Package != nil { - opts.ScannerConfig.Package = make(map[string]func(interface{}) error, len(i.conf.Indexer.Scanner.Package)) - for name, node := range i.conf.Indexer.Scanner.Package { - opts.ScannerConfig.Package[name] = node.Decode - } - } - if i.conf.Indexer.Scanner.Dist != nil { - opts.ScannerConfig.Dist = make(map[string]func(interface{}) error, len(i.conf.Indexer.Scanner.Dist)) - for name, node := range i.conf.Indexer.Scanner.Dist { - opts.ScannerConfig.Dist[name] = node.Decode - } - } - if i.conf.Indexer.Scanner.Repo != nil { - opts.ScannerConfig.Repo = make(map[string]func(interface{}) error, len(i.conf.Indexer.Scanner.Repo)) - for name, node := range i.conf.Indexer.Scanner.Repo { - opts.ScannerConfig.Repo[name] = node.Decode - } - } - libI, err := libindex.New(i.GlobalCTX, &opts) + srv.Indexer, err = localIndexer(ctx, cfg) if err != nil { - return clairerror.ErrNotInitialized{Msg: "failed to initialize libindex: " + err.Error()} + return nil, err } - i.Indexer = libI - i.Matcher = nil case config.MatcherMode: - updaterConfigs := make(map[string]driver.ConfigUnmarshaler) - for name, node := range i.conf.Updaters.Config { - updaterConfigs[name] = node.Decode - } - // configure a local matcher but a remote indexer - libV, err := libvuln.New(i.GlobalCTX, &libvuln.Opts{ - MaxConnPool: int32(i.conf.Matcher.MaxConnPool), - ConnString: i.conf.Matcher.ConnString, - Migrations: i.conf.Matcher.Migrations, - UpdaterSets: i.conf.Updaters.Sets, - UpdateInterval: i.conf.Matcher.Period, - UpdaterConfigs: updaterConfigs, - UpdateRetention: i.conf.Matcher.UpdateRetention, - }) + srv.Matcher, err = localMatcher(ctx, cfg) if err != nil { - return fmt.Errorf("failed to initialize libvuln: %v", err) - } - // matcher mode needs a remote indexer client - c, auth, err := i.conf.Client(nil, intraserviceClaim) - switch { - case err != nil: - return err - case !auth && i.conf.Auth.Any(): - return &clairerror.ErrNotInitialized{ - Msg: "client authorization required but not provided", - } - default: // OK + return nil, err } - remoteIndexer, err := client.NewHTTP(i.GlobalCTX, - client.WithAddr(i.conf.Matcher.IndexerAddr), - client.WithClient(c)) + srv.Indexer, err = remoteIndexer(ctx, cfg, cfg.Matcher.IndexerAddr) if err != nil { - return err + return nil, err } - i.Indexer = remoteIndexer - i.Matcher = libV case config.NotifierMode: - // notifier uses a remote indexer and matcher - c, auth, err := i.conf.Client(nil, intraserviceClaim) - switch { - case err != nil: - return err - case !auth && i.conf.Auth.Any(): - return &clairerror.ErrNotInitialized{ - Msg: "client authorization required but not provided", - } - default: // OK + srv.Indexer, err = remoteIndexer(ctx, cfg, cfg.Notifier.IndexerAddr) + if err != nil { + return nil, err } - - remoteIndexer, err := client.NewHTTP(i.GlobalCTX, - client.WithAddr(i.conf.Notifier.IndexerAddr), - client.WithClient(c)) + srv.Matcher, err = remoteMatcher(ctx, cfg, cfg.Notifier.MatcherAddr) if err != nil { - return err + return nil, err } - - remoteMatcher, err := client.NewHTTP(i.GlobalCTX, - client.WithAddr(i.conf.Notifier.MatcherAddr), - client.WithClient(c)) + srv.Notifier, err = localNotifier(ctx, cfg, srv.Indexer, srv.Matcher) if err != nil { - return err + return nil, err } + default: + return nil, fmt.Errorf("could not determine passed in mode: %v", cfg.Mode) + } - c, _, err = i.conf.Client(nil, notifierClaim) - if err != nil { - return err + return &srv, nil +} + +func localIndexer(ctx context.Context, cfg *config.Config) (indexer.Service, error) { + const msg = "failed to initialize indexer: " + mkErr := func(err error) *clairerror.ErrNotInitialized { + return &clairerror.ErrNotInitialized{msg + err.Error()} + } + opts := libindex.Opts{ + ConnString: cfg.Indexer.ConnString, + ScanLockRetry: time.Duration(cfg.Indexer.ScanLockRetry) * time.Second, + LayerScanConcurrency: cfg.Indexer.LayerScanConcurrency, + Migrations: cfg.Indexer.Migrations, + Airgap: cfg.Indexer.Airgap, + } + if cfg.Indexer.Scanner.Package != nil { + opts.ScannerConfig.Package = make(map[string]func(interface{}) error, len(cfg.Indexer.Scanner.Package)) + for name, node := range cfg.Indexer.Scanner.Package { + opts.ScannerConfig.Package[name] = node.Decode } + } + if cfg.Indexer.Scanner.Dist != nil { + opts.ScannerConfig.Dist = make(map[string]func(interface{}) error, len(cfg.Indexer.Scanner.Dist)) + for name, node := range cfg.Indexer.Scanner.Dist { + opts.ScannerConfig.Dist[name] = node.Decode + } + } + if cfg.Indexer.Scanner.Repo != nil { + opts.ScannerConfig.Repo = make(map[string]func(interface{}) error, len(cfg.Indexer.Scanner.Repo)) + for name, node := range cfg.Indexer.Scanner.Repo { + opts.ScannerConfig.Repo[name] = node.Decode + } + } + s, err := libindex.New(ctx, &opts) + if err != nil { + return nil, mkErr(err) + } + return s, nil +} - n, err := notifier.New(i.GlobalCTX, notifier.Opts{ - DeliveryInterval: i.conf.Notifier.DeliveryInterval, - ConnString: i.conf.Notifier.ConnString, - Indexer: remoteIndexer, - Matcher: remoteMatcher, - Client: c, - Migrations: i.conf.Notifier.Migrations, - PollInterval: i.conf.Notifier.PollInterval, - Webhook: i.conf.Notifier.Webhook, - AMQP: i.conf.Notifier.AMQP, - STOMP: i.conf.Notifier.STOMP, - }) - if err != nil { - return &clairerror.ErrNotInitialized{ - Msg: "notifier failed to initialize: " + err.Error(), - } +func remoteIndexer(ctx context.Context, cfg *config.Config, addr string) (indexer.Service, error) { + const msg = "failed to initialize indexer client: " + mkErr := func(err error) *clairerror.ErrNotInitialized { + return &clairerror.ErrNotInitialized{msg + err.Error()} + } + rc, err := remoteClient(ctx, cfg, intraserviceClaim, addr) + if err != nil { + return nil, mkErr(err) + } + return rc, nil +} + +func remoteClient(ctx context.Context, cfg *config.Config, claim jwt.Claims, addr string) (*client.HTTP, error) { + c, auth, err := cfg.Client(nil, claim) + switch { + case err != nil: + return nil, err + case !auth && cfg.Auth.Any(): + return nil, errors.New("client authorization required but not provided") + default: // OK + } + return client.NewHTTP(ctx, client.WithAddr(addr), client.WithClient(c)) +} + +func localMatcher(ctx context.Context, cfg *config.Config) (matcher.Service, error) { + const msg = "failed to initialize matcher: " + mkErr := func(err error) *clairerror.ErrNotInitialized { + return &clairerror.ErrNotInitialized{ + Msg: msg + err.Error(), } - i.Indexer = remoteIndexer - i.Matcher = remoteMatcher - i.Notifier = n + } - default: - return fmt.Errorf("could not determine passed in mode: %v", i.conf.Mode) + updaterConfigs := make(map[string]driver.ConfigUnmarshaler) + for name, node := range cfg.Updaters.Config { + updaterConfigs[name] = node.Decode + } + s, err := libvuln.New(ctx, &libvuln.Opts{ + MaxConnPool: int32(cfg.Matcher.MaxConnPool), + ConnString: cfg.Matcher.ConnString, + Migrations: cfg.Matcher.Migrations, + UpdaterSets: cfg.Updaters.Sets, + UpdateInterval: cfg.Matcher.Period, + UpdaterConfigs: updaterConfigs, + UpdateRetention: cfg.Matcher.UpdateRetention, + }) + if err != nil { + return nil, mkErr(err) } + return s, nil +} - return nil +func remoteMatcher(ctx context.Context, cfg *config.Config, addr string) (matcher.Service, error) { + const msg = "failed to initialize matcher client: " + mkErr := func(err error) *clairerror.ErrNotInitialized { + return &clairerror.ErrNotInitialized{msg + err.Error()} + } + rc, err := remoteClient(ctx, cfg, intraserviceClaim, addr) + if err != nil { + return nil, mkErr(err) + } + return rc, nil } -var ( - intraserviceClaim = jwt.Claims{Issuer: httptransport.IntraserviceIssuer} - notifierClaim = jwt.Claims{Issuer: NotifierIssuer} -) +func localNotifier(ctx context.Context, cfg *config.Config, i indexer.Service, m matcher.Service) (notifier.Service, error) { + const msg = "failed to initialize notifier: " + mkErr := func(err error) *clairerror.ErrNotInitialized { + return &clairerror.ErrNotInitialized{ + Msg: msg + err.Error(), + } + } + + c, _, err := cfg.Client(nil, notifierClaim) + if err != nil { + return nil, mkErr(err) + } + + s, err := notifier.New(ctx, notifier.Opts{ + DeliveryInterval: cfg.Notifier.DeliveryInterval, + ConnString: cfg.Notifier.ConnString, + Indexer: i, + Matcher: m, + Client: c, + Migrations: cfg.Notifier.Migrations, + PollInterval: cfg.Notifier.PollInterval, + DisableSummary: cfg.Notifier.DisableSummary, + Webhook: cfg.Notifier.Webhook, + AMQP: cfg.Notifier.AMQP, + STOMP: cfg.Notifier.STOMP, + }) + if err != nil { + return nil, mkErr(err) + } + return s, nil +}