EVG-7779: Make buildlogger setup code general to cedar (#3963)

apimodels/buildlogger_models.go → apimodels/cedar_models.go

@@ -1,6 +1,6 @@
 package apimodels
 
-type BuildloggerInfo struct {
+type CedarConfig struct {
 	BaseURL  string `json:"base_url"`
 	RPCPort  string `json:"rpc_port"`
 	Username string `json:"username"`

config.go

@@ -34,7 +34,7 @@ var (
 	ClientVersion = "2020-09-02"
 
 	// Agent version to control agent rollover.
-	AgentVersion = "2020-09-08"
+	AgentVersion = "2020-09-09"
 )
 
 // ConfigSection defines a sub-document in the evergreen config
@@ -62,6 +62,7 @@ type Settings struct {
 	Banner             string                    `bson:"banner" json:"banner" yaml:"banner"`
 	BannerTheme        BannerTheme               `bson:"banner_theme" json:"banner_theme" yaml:"banner_theme"`
 	Backup             BackupConfig              `bson:"backup" json:"backup" yaml:"backup"`
+	Cedar              CedarConfig               `bson:"cedar" json:"cedar" yaml:"cedar" id:"cedar"`
 	ClientBinariesDir  string                    `yaml:"client_binaries_dir" bson:"client_binaries_dir" json:"client_binaries_dir"`
 	CommitQueue        CommitQueueConfig         `yaml:"commit_queue" bson:"commit_queue" json:"commit_queue" id:"commit_queue"`
 	ConfigDir          string                    `yaml:"configdir" bson:"configdir" json:"configdir"`

config_cedar.go

@@ -0,0 +1,59 @@
+package evergreen
+
+import (
+	"github.com/pkg/errors"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+type CedarConfig struct {
+	BaseURL  string `bson:"base_url" json:"base_url" yaml:"base_url"`
+	RPCPort  string `bson:"rpc_port" json:"rpc_port" yaml:"rpc_port"`
+	User     string `bson:"user" json:"user" yaml:"user"`
+	Password string `bson:"password" json:"password" yaml:"password"`
+	APIKey   string `bson:"api_key" json:"api_key" yaml:"api_key"`
+}
+
+func (*CedarConfig) SectionId() string { return "cedar" }
+
+func (c *CedarConfig) Get(env Environment) error {
+	ctx, cancel := env.Context()
+	defer cancel()
+	coll := env.DB().Collection(ConfigCollection)
+
+	res := coll.FindOne(ctx, byId(c.SectionId()))
+	if err := res.Err(); err != nil {
+		if err == mongo.ErrNoDocuments {
+			*c = CedarConfig{}
+			return nil
+		}
+		return errors.Wrapf(err, "error retrieving section %s", c.SectionId())
+	}
+	if err := res.Decode(c); err != nil {
+		return errors.Wrap(err, "problem decoding result")
+	}
+
+	return nil
+}
+
+func (c *CedarConfig) Set() error {
+	env := GetEnvironment()
+	ctx, cancel := env.Context()
+	defer cancel()
+	coll := env.DB().Collection(ConfigCollection)
+
+	_, err := coll.UpdateOne(ctx, byId(c.SectionId()), bson.M{
+		"$set": bson.M{
+			"base_url": c.BaseURL,
+			"rpc_port": c.RPCPort,
+			"user":     c.User,
+			"password": c.Password,
+			"api_key":  c.APIKey,
+		},
+	}, options.Update().SetUpsert(true))
+
+	return errors.Wrapf(err, "error updating section %s", c.SectionId())
+}
+
+func (c *CedarConfig) ValidateAndDefault() error { return nil }

config_db.go

@@ -20,6 +20,7 @@ var (
 	serviceFlagsKey       = bsonutil.MustHaveTag(Settings{}, "ServiceFlags")
 	configDirKey          = bsonutil.MustHaveTag(Settings{}, "ConfigDir")
 	apiUrlKey             = bsonutil.MustHaveTag(Settings{}, "ApiUrl")
+	cedarKey              = bsonutil.MustHaveTag(Settings{}, "Cedar")
 	clientBinariesDirKey  = bsonutil.MustHaveTag(Settings{}, "ClientBinariesDir")
 	hostJasperKey         = bsonutil.MustHaveTag(Settings{}, "HostJasper")
 	domainNameKey         = bsonutil.MustHaveTag(Settings{}, "DomainName")

config_logger.go

@@ -10,16 +10,11 @@ import (
 )
 
 type LoggerConfig struct {
-	Buffer              LogBuffering `bson:"buffer" json:"buffer" yaml:"buffer"`
-	DefaultLevel        string       `bson:"default_level" json:"default_level" yaml:"default_level"`
-	ThresholdLevel      string       `bson:"threshold_level" json:"threshold_level" yaml:"threshold_level"`
-	LogkeeperURL        string       `bson:"logkeeper_url" json:"logkeeper_url" yaml:"logkeeper_url"`
-	BuildloggerBaseURL  string       `bson:"buildlogger_base_url" json:"buildlogger_base_url" yaml:"buildlogger_base_url"`
-	BuildloggerRPCPort  string       `bson:"buildlogger_rpc_port" json:"buildlogger_rpc_port" yaml:"buildlogger_rpc_port"`
-	BuildloggerUser     string       `bson:"buildlogger_user" json:"buildlogger_user" yaml:"buildlogger_user"`
-	BuildloggerPassword string       `bson:"buildlogger_password" json:"buildlogger_password" yaml:"buildlogger_password"`
-	BuildloggerAPIKey   string       `bson:"buildlogger_api_key" json:"buildlogger_api_key" yaml:"buildlogger_api_key"`
-	DefaultLogger       string       `bson:"default_logger" json:"default_logger" yaml:"default_logger"`
+	Buffer         LogBuffering `bson:"buffer" json:"buffer" yaml:"buffer"`
+	DefaultLevel   string       `bson:"default_level" json:"default_level" yaml:"default_level"`
+	ThresholdLevel string       `bson:"threshold_level" json:"threshold_level" yaml:"threshold_level"`
+	LogkeeperURL   string       `bson:"logkeeper_url" json:"logkeeper_url" yaml:"logkeeper_url"`
+	DefaultLogger  string       `bson:"default_logger" json:"default_logger" yaml:"default_logger"`
 }
 
 func (c LoggerConfig) Info() send.LevelInfo {
@@ -59,16 +54,11 @@ func (c *LoggerConfig) Set() error {
 
 	_, err := coll.UpdateOne(ctx, byId(c.SectionId()), bson.M{
 		"$set": bson.M{
-			"buffer":               c.Buffer,
-			"default_level":        c.DefaultLevel,
-			"threshold_level":      c.ThresholdLevel,
-			"logkeeper_url":        c.LogkeeperURL,
-			"buildlogger_base_url": c.BuildloggerBaseURL,
-			"buildlogger_rpc_port": c.BuildloggerRPCPort,
-			"buildlogger_user":     c.BuildloggerUser,
-			"buildlogger_password": c.BuildloggerPassword,
-			"buildlogger_api_key":  c.BuildloggerAPIKey,
-			"default_logger":       c.DefaultLogger,
+			"buffer":          c.Buffer,
+			"default_level":   c.DefaultLevel,
+			"threshold_level": c.ThresholdLevel,
+			"logkeeper_url":   c.LogkeeperURL,
+			"default_logger":  c.DefaultLogger,
 		},
 	}, options.Update().SetUpsert(true))
 

config_registry.go

@@ -35,6 +35,7 @@ func resetRegistry() error {
 		&AmboyConfig{},
 		&APIConfig{},
 		&AuthConfig{},
+		&CedarConfig{},
 		&CloudProviders{},
 		&CommitQueueConfig{},
 		&ContainerPoolsConfig{},

graphql/resolvers.go

@@ -1068,7 +1068,7 @@ func (r *queryResolver) TaskLogs(ctx context.Context, taskID string) (*RecentTas
 	// get logs from cedar
 	if defaultLogger == model.BuildloggerLogSender {
 		opts := apimodels.GetBuildloggerLogsOptions{
-			BaseURL:       evergreen.GetEnvironment().Settings().LoggerConfig.BuildloggerBaseURL,
+			BaseURL:       evergreen.GetEnvironment().Settings().Cedar.BaseURL,
 			TaskID:        taskID,
 			Execution:     t.Execution,
 			PrintPriority: true,

rest/client/client.go

@@ -315,17 +315,17 @@ func (c *communicatorImpl) makeSender(ctx context.Context, td TaskData, opts []L
 
 func (c *communicatorImpl) createCedarGRPCConn(ctx context.Context) error {
 	if c.cedarGRPCClient == nil {
-		bi, err := c.GetBuildloggerInfo(ctx)
+		cc, err := c.GetCedarConfig(ctx)
 		if err != nil {
 			return errors.Wrap(err, "error setting up buildlogger sender")
 		}
 
 		dialOpts := timber.DialCedarOptions{
-			BaseAddress: bi.BaseURL,
-			RPCPort:     bi.RPCPort,
-			Username:    bi.Username,
-			Password:    bi.Password,
-			APIKey:      bi.APIKey,
+			BaseAddress: cc.BaseURL,
+			RPCPort:     cc.RPCPort,
+			Username:    cc.Username,
+			Password:    cc.Password,
+			APIKey:      cc.APIKey,
 			Retries:     10,
 		}
 		c.cedarGRPCClient, err = timber.DialCedar(ctx, c.httpClient, dialOpts)

rest/client/interface.go

@@ -74,9 +74,9 @@ type Communicator interface {
 	FetchExpansionVars(context.Context, TaskData) (*apimodels.ExpansionVars, error)
 	// GetNextTask returns a next task response by getting the next task for a given host.
 	GetNextTask(context.Context, *apimodels.GetNextTaskDetails) (*apimodels.NextTaskResponse, error)
-	// GetBuildloggerInfo returns buildlogger service information including
-	// the base URL, RPC port, and LDAP credentials.
-	GetBuildloggerInfo(context.Context) (*apimodels.BuildloggerInfo, error)
+	// GetCedarConfig returns the cedar service information including the
+	// base URL, RPC port, and credentials.
+	GetCedarConfig(context.Context) (*apimodels.CedarConfig, error)
 	// GetCedarGRPCConn returns the client connection to cedar if it exists, or
 	// creates it if it doesn't exist.
 	GetCedarGRPCConn(context.Context) (*grpc.ClientConn, error)

rest/client/methods.go

@@ -310,29 +310,29 @@ func (c *communicatorImpl) GetNextTask(ctx context.Context, details *apimodels.G
 	return nextTask, nil
 }
 
-// GetBuildloggerInfo returns buildlogger service information including the
-// base URL, RPC port, and LDAP credentials.
-func (c *communicatorImpl) GetBuildloggerInfo(ctx context.Context) (*apimodels.BuildloggerInfo, error) {
-	bi := &apimodels.BuildloggerInfo{}
+// GetCedarConfig returns the cedar service information including the base URL,
+// URL, RPC port, and credentials.
+func (c *communicatorImpl) GetCedarConfig(ctx context.Context) (*apimodels.CedarConfig, error) {
+	cc := &apimodels.CedarConfig{}
 
 	info := requestInfo{
 		method:  get,
 		version: apiVersion1,
-		path:    "agent/buildlogger_info",
+		path:    "agent/cedar_config",
 	}
 
 	resp, err := c.retryRequest(ctx, info, nil)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to get buildlogger service info")
+		return nil, errors.Wrap(err, "failed to get cedar service info")
 	}
 	defer resp.Body.Close()
 
-	if err = utility.ReadJSON(resp.Body, bi); err != nil {
+	if err = utility.ReadJSON(resp.Body, cc); err != nil {
 		err = errors.Wrap(err, "failed to read next task from response")
 		return nil, err
 	}
 
-	return bi, nil
+	return cc, nil
 }
 
 // GetCedarGRPCConn returns the client connection to cedar if it exists, or

rest/client/mock.go

@@ -255,13 +255,14 @@ func (c *Mock) GetNextTask(ctx context.Context, details *apimodels.GetNextTaskDe
 	}, nil
 }
 
-// GetBuildloggerInfo returns mock buildlogger service information.
-func (c *Mock) GetBuildloggerInfo(ctx context.Context) (*apimodels.BuildloggerInfo, error) {
-	return &apimodels.BuildloggerInfo{
+// GetCedarConfig returns mock cedar service information.
+func (c *Mock) GetCedarConfig(ctx context.Context) (*apimodels.CedarConfig, error) {
+	return &apimodels.CedarConfig{
 		BaseURL:  "base_url",
 		RPCPort:  "1000",
 		Username: "user",
 		Password: "password",
+		APIKey:   "api_key",
 	}, nil
 }
 

rest/model/admin.go

@@ -17,6 +17,7 @@ func NewConfigModel() *APIAdminSettings {
 		Api:               &APIapiConfig{},
 		AuthConfig:        &APIAuthConfig{},
 		Backup:            &APIBackupConfig{},
+		Cedar:             &APICedarConfig{},
 		CommitQueue:       &APICommitQueueConfig{},
 		ContainerPools:    &APIContainerPoolsConfig{},
 		Credentials:       map[string]string{},
@@ -53,6 +54,7 @@ type APIAdminSettings struct {
 	Banner             *string                           `json:"banner,omitempty"`
 	BannerTheme        *string                           `json:"banner_theme,omitempty"`
 	Backup             *APIBackupConfig                  `json:"backup,omitempty"`
+	Cedar              *APICedarConfig                   `json:"cedar,omitempty"`
 	ClientBinariesDir  *string                           `json:"client_binaries_dir,omitempty"`
 	CommitQueue        *APICommitQueueConfig             `json:"commit_queue,omitempty"`
 	ConfigDir          *string                           `json:"configdir,omitempty"`
@@ -541,6 +543,38 @@ func (a *APIBackupConfig) ToService() (interface{}, error) {
 	}, nil
 }
 
+type APICedarConfig struct {
+	BaseURL  *string `json:"base_url"`
+	RPCPort  *string `json:"rpc_port"`
+	User     *string `json:"user"`
+	Password *string `json:"password"`
+	APIKey   *string `json:"api_key"`
+}
+
+func (a *APICedarConfig) BuildFromService(h interface{}) error {
+	switch v := h.(type) {
+	case evergreen.CedarConfig:
+		a.BaseURL = ToStringPtr(v.BaseURL)
+		a.RPCPort = ToStringPtr(v.RPCPort)
+		a.User = ToStringPtr(v.User)
+		a.Password = ToStringPtr(v.Password)
+		a.APIKey = ToStringPtr(v.APIKey)
+	default:
+		return errors.Errorf("%T is not a supported type", h)
+	}
+	return nil
+}
+
+func (a *APICedarConfig) ToService() (interface{}, error) {
+	return evergreen.CedarConfig{
+		BaseURL:  FromStringPtr(a.BaseURL),
+		RPCPort:  FromStringPtr(a.RPCPort),
+		User:     FromStringPtr(a.User),
+		Password: FromStringPtr(a.Password),
+		APIKey:   FromStringPtr(a.APIKey),
+	}, nil
+}
+
 type APILDAPConfig struct {
 	URL                *string `json:"url"`
 	Port               *string `json:"port"`
@@ -934,16 +968,11 @@ func (a *APILDAPRoleMap) ToService() (interface{}, error) {
 }
 
 type APILoggerConfig struct {
-	Buffer              *APILogBuffering `json:"buffer"`
-	DefaultLevel        *string          `json:"default_level"`
-	ThresholdLevel      *string          `json:"threshold_level"`
-	LogkeeperURL        *string          `json:"logkeeper_url"`
-	BuildloggerBaseURL  *string          `json:"buildlogger_base_url"`
-	BuildloggerRPCPort  *string          `json:"buildlogger_rpc_port"`
-	BuildloggerUser     *string          `json:"buildlogger_user"`
-	BuildloggerPassword *string          `json:"buildlogger_password"`
-	BuildloggerAPIKey   *string          `json:"buildlogger_api_key"`
-	DefaultLogger       *string          `json:"default_logger"`
+	Buffer         *APILogBuffering `json:"buffer"`
+	DefaultLevel   *string          `json:"default_level"`
+	ThresholdLevel *string          `json:"threshold_level"`
+	LogkeeperURL   *string          `json:"logkeeper_url"`
+	DefaultLogger  *string          `json:"default_logger"`
 }
 
 func (a *APILoggerConfig) BuildFromService(h interface{}) error {
@@ -952,11 +981,6 @@ func (a *APILoggerConfig) BuildFromService(h interface{}) error {
 		a.DefaultLevel = ToStringPtr(v.DefaultLevel)
 		a.ThresholdLevel = ToStringPtr(v.ThresholdLevel)
 		a.LogkeeperURL = ToStringPtr(v.LogkeeperURL)
-		a.BuildloggerBaseURL = ToStringPtr(v.BuildloggerBaseURL)
-		a.BuildloggerRPCPort = ToStringPtr(v.BuildloggerRPCPort)
-		a.BuildloggerUser = ToStringPtr(v.BuildloggerUser)
-		a.BuildloggerPassword = ToStringPtr(v.BuildloggerPassword)
-		a.BuildloggerAPIKey = ToStringPtr(v.BuildloggerAPIKey)
 		a.DefaultLogger = ToStringPtr(v.DefaultLogger)
 		a.Buffer = &APILogBuffering{}
 		if err := a.Buffer.BuildFromService(v.Buffer); err != nil {
@@ -970,15 +994,10 @@ func (a *APILoggerConfig) BuildFromService(h interface{}) error {
 
 func (a *APILoggerConfig) ToService() (interface{}, error) {
 	config := evergreen.LoggerConfig{
-		DefaultLevel:        FromStringPtr(a.DefaultLevel),
-		ThresholdLevel:      FromStringPtr(a.ThresholdLevel),
-		LogkeeperURL:        FromStringPtr(a.LogkeeperURL),
-		BuildloggerBaseURL:  FromStringPtr(a.BuildloggerBaseURL),
-		BuildloggerRPCPort:  FromStringPtr(a.BuildloggerRPCPort),
-		BuildloggerUser:     FromStringPtr(a.BuildloggerUser),
-		BuildloggerPassword: FromStringPtr(a.BuildloggerPassword),
-		BuildloggerAPIKey:   FromStringPtr(a.BuildloggerAPIKey),
-		DefaultLogger:       FromStringPtr(a.DefaultLogger),
+		DefaultLevel:   FromStringPtr(a.DefaultLevel),
+		ThresholdLevel: FromStringPtr(a.ThresholdLevel),
+		LogkeeperURL:   FromStringPtr(a.LogkeeperURL),
+		DefaultLogger:  FromStringPtr(a.DefaultLogger),
 	}
 	i, err := a.Buffer.ToService()
 	if err != nil {

rest/route/admin_settings.go

@@ -77,7 +77,6 @@ func (h *adminPostHandler) Run(ctx context.Context) gimlet.Responder {
 	if err != nil {
 		return gimlet.MakeJSONErrorResponder(errors.Wrap(err, "error applying new settings"))
 	}
-
 	if err = newSettings.Validate(); err != nil {
 		return gimlet.MakeJSONErrorResponder(errors.Wrap(err, "Validation error"))
 	}

service/api.go

@@ -556,7 +556,8 @@ func (as *APIServer) GetServiceApp() *gimlet.APIApp {
 	// Agent routes
 	app.Route().Version(2).Route("/agent/setup").Wrap(checkHost).Handler(as.agentSetup).Get()
 	app.Route().Version(2).Route("/agent/next_task").Wrap(checkHost).Handler(as.NextTask).Get()
-	app.Route().Version(2).Route("/agent/buildlogger_info").Wrap(checkHost).Handler(as.Buildlogger).Get()
+	app.Route().Version(2).Route("/agent/buildlogger_info").Wrap(checkHost).Handler(as.Cedar).Get()
+	app.Route().Version(2).Route("/agent/cedar_config").Wrap(checkHost).Handler(as.Cedar).Get()
 	app.Route().Version(2).Route("/task/{taskId}/end").Wrap(checkTaskSecret, checkHost).Handler(as.EndTask).Post()
 	app.Route().Version(2).Route("/task/{taskId}/start").Wrap(checkTaskSecret, checkHost).Handler(as.StartTask).Post()
 	app.Route().Version(2).Route("/task/{taskId}/log").Wrap(checkTaskSecret, checkHost).Handler(as.AppendTaskLog).Post()