Skip to content

Commit

Permalink
feat(engine): store config in db
Browse files Browse the repository at this point in the history 
UI will be done in another PR

Signed-off-by: Yvonnick Esnault <yvonnick.esnault@corp.ovh.com>
  • Loading branch information
@yesnault
yesnault committed Sep 25, 2018
1 parent 7abf470 commit 5995379
Show file tree
Hide file tree
Showing 17 changed files with 250 additions and 233 deletions.
188 changes: 94 additions & 94 deletions engine/api/api.go
View file

Large diffs are not rendered by default.

20 changes: 10 additions & 10 deletions engine/api/database/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@ const (

// DBConfiguration is the exposed type for database API configuration
type DBConfiguration struct {
User string `toml:"user" default:"cds"`
Role string `toml:"role" default:"" commented:"true" comment:"Set a specific role to run SET ROLE for each connection"`
Password string `toml:"password" default:"cds"`
Name string `toml:"name" default:"cds"`
Host string `toml:"host" default:"localhost"`
Port int `toml:"port" default:"5432"`
SSLMode string `toml:"sslmode" default:"disable" comment:"DB SSL Mode: require (default), verify-full, or disable"`
MaxConn int `toml:"maxconn" default:"20" comment:"DB Max connection"`
ConnectTimeout int `toml:"connectTimeout" default:"10" comment:"Maximum wait for connection, in seconds"`
Timeout int `toml:"timeout" default:"3000" comment:"Statement timeout value in milliseconds"`
User string `toml:"user" default:"cds" json:"user"`
Role string `toml:"role" default:"" commented:"true" comment:"Set a specific role to run SET ROLE for each connection" json:"role"`
Password string `toml:"password" default:"cds" json:"-"`
Name string `toml:"name" default:"cds" json:"name"`
Host string `toml:"host" default:"localhost" json:"host"`
Port int `toml:"port" default:"5432" json:"port"`
SSLMode string `toml:"sslmode" default:"disable" comment:"DB SSL Mode: require (default), verify-full, or disable" json:"sslmode"`
MaxConn int `toml:"maxconn" default:"20" comment:"DB Max connection" json:"maxconn"`
ConnectTimeout int `toml:"connectTimeout" default:"10" comment:"Maximum wait for connection, in seconds" json:"connectTimeout"`
Timeout int `toml:"timeout" default:"3000" comment:"Statement timeout value in milliseconds" json:"timeout"`
}
14 changes: 7 additions & 7 deletions engine/api/observability/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,16 @@ var DefaultFormat propagation.HTTPFormat = &b3.HTTPFormat{}

// Configuration is the global tracing configuration
type Configuration struct {
Enable bool
Enable bool `json:"enable"`
Exporter struct {
Jaeger struct {
HTTPCollectorEndpoint string `toml:"HTTPCollectorEndpoint" default:"http://localhost:14268"`
}
HTTPCollectorEndpoint string `toml:"HTTPCollectorEndpoint" default:"http://localhost:14268" json:"httpCollectorEndpoint"`
} `json:"jaeger"`
Prometheus struct {
ReporteringPeriod int `toml:"ReporteringPeriod" default:"60"`
}
}
SamplingProbability float64
ReporteringPeriod int `toml:"ReporteringPeriod" default:"60" json:"reporteringPeriod"`
} `json:"prometheus"`
} `json:"exporter"`
SamplingProbability float64 `json:"samplingProbability"`
}

//Options is the options struct for a new tracing span
Expand Down
1 change: 1 addition & 0 deletions engine/api/services.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ func (api *API) serviceAPIHeartbeatUpdate(c context.Context, db *gorp.DbMap, has
Hash: string(hash),
LastHeartbeat: time.Now(),
Type: services.TypeAPI,
Config: api.Config,
}

//Try to find the service, and keep; else generate a new one
Expand Down
2 changes: 1 addition & 1 deletion engine/api/services/awol.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func KillDeadServices(ctx context.Context, dbFunc func() *gorp.DbMap) {
}
for i := range services {
if err := Delete(db, &services[i]); err != nil {
log.Error("KillDeadServices> Unable to find dead services: %v", errdead)
log.Error("KillDeadServices> Unable to find dead services: %v", err)
continue
}
}
Expand Down
29 changes: 18 additions & 11 deletions engine/api/services/dao.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ func FindByType(db gorp.SqlExecutor, t string) ([]sdk.Service, error) {
if ss, ok := servicesCacheByType[t]; ok {
return ss, nil
}

query := `SELECT * FROM services WHERE type = $1`
services, err := findAll(db, query, t)
if err != nil {
Expand Down Expand Up @@ -153,24 +152,28 @@ func Delete(db gorp.SqlExecutor, s *sdk.Service) error {

// PostGet is a dbHook on Select to get json column
func (s *service) PostGet(db gorp.SqlExecutor) error {
query := "SELECT monitoring_status FROM services WHERE name = $1"
var content []byte
if err := db.QueryRow(query, s.Name).Scan(&content); err != nil {
query := "SELECT monitoring_status, config FROM services WHERE name = $1"
var monitoringStatus, config []byte
if err := db.QueryRow(query, s.Name).Scan(&monitoringStatus, &config); err != nil {
return sdk.WrapError(err, "PostGet> error on queryRow")
}

if len(content) > 0 {
if len(monitoringStatus) > 0 {
m := sdk.MonitoringStatus{}
if err := json.Unmarshal(content, &m); err != nil {
return sdk.WrapError(err, "PostGet> error on unmarshal job")
if err := json.Unmarshal(monitoringStatus, &m); err != nil {
return sdk.WrapError(err, "PostGet> error on unmarshal monitoringStatus service")
}
for i := range m.Lines {
m.Lines[i].Component = fmt.Sprintf("%s/%s", s.Name, m.Lines[i].Component)
m.Lines[i].Type = s.Type
}
s.MonitoringStatus = m
}

if len(config) > 0 {
if err := json.Unmarshal(config, &s.Config); err != nil {
return sdk.WrapError(err, "PostGet> error on unmarshal config service")
}
}
return nil
}

Expand All @@ -185,10 +188,14 @@ func (s *service) PostUpdate(db gorp.SqlExecutor) error {
if err != nil {
return err
}
config, errc := json.Marshal(s.Config)
if errc != nil {
return errc
}

query := "update services set monitoring_status = $1 where name = $2"
if _, err := db.Exec(query, content, s.Name); err != nil {
return sdk.WrapError(err, "PostUpdate> err on update sql")
query := "update services set monitoring_status = $1, config = $2 where name = $3"
if _, err := db.Exec(query, content, config, s.Name); err != nil {
return sdk.WrapError(err, "PostUpdate> err on update sql service")
}
return nil
}
Expand Down
24 changes: 12 additions & 12 deletions engine/elasticsearch/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,18 @@ type Service struct {

// Configuration is the vcs configuration structure
type Configuration struct {
Name string `toml:"name" comment:"Name of this CDS elasticsearch Service\n Enter a name to enable this service"`
Name string `toml:"name" comment:"Name of this CDS elasticsearch Service\n Enter a name to enable this service" json:"name"`
HTTP struct {
Addr string `toml:"addr" default:"" commented:"true" comment:"Listen address without port, example: 127.0.0.1"`
Port int `toml:"port" default:"8088"`
} `toml:"http" comment:"######################\n CDS Elasticsearch HTTP Configuration \n######################"`
URL string `default:"http://localhost:8088"`
Addr string `toml:"addr" default:"" commented:"true" comment:"Listen address without port, example: 127.0.0.1" json:"addr"`
Port int `toml:"port" default:"8088" json:"port"`
} `toml:"http" comment:"######################\n CDS Elasticsearch HTTP Configuration \n######################" json:"http"`
URL string `default:"http://localhost:8088" json:"url"`
ElasticSearch struct {
URL string `toml:"url"`
Username string `toml:"username"`
Password string `toml:"password"`
IndexEvents string `toml:"indexEvents" commented:"true" comment:"index to store CDS events"`
IndexMetrics string `toml:"indexMetrics" commented:"true" comment:"index to store CDS metrics"`
} `toml:"elasticsearch" comment:"######################\n CDS ElasticSearch Settings \n######################"`
API service.APIServiceConfiguration `toml:"api" comment:"######################\n CDS API Settings \n######################"`
URL string `toml:"url" json:"url"`
Username string `toml:"username" json:"username"`
Password string `toml:"password" json:"-"`
IndexEvents string `toml:"indexEvents" commented:"true" comment:"index to store CDS events" json:"indexEvents"`
IndexMetrics string `toml:"indexMetrics" commented:"true" comment:"index to store CDS metrics" json:"indexMetrics"`
} `toml:"elasticsearch" comment:"######################\n CDS ElasticSearch Settings \n######################" json:"elasticsearch"`
API service.APIServiceConfiguration `toml:"api" comment:"######################\n CDS API Settings \n######################" json:"api"`
}
32 changes: 16 additions & 16 deletions engine/hooks/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,23 +24,23 @@ type Service struct {

// Configuration is the hooks configuration structure
type Configuration struct {
Name string `toml:"name" comment:"Name of this CDS Hooks Service\n Enter a name to enable this service"`
Name string `toml:"name" comment:"Name of this CDS Hooks Service\n Enter a name to enable this service" json:"name"`
HTTP struct {
Addr string `toml:"addr" default:"" commented:"true" comment:"Listen address without port, example: 127.0.0.1"`
Port int `toml:"port" default:"8083"`
} `toml:"http" comment:"######################\n CDS Hooks HTTP Configuration \n######################"`
URL string `default:"http://localhost:8083"`
URLPublic string `toml:"urlPublic" comment:"Public url for external call (webhook)"`
RetryDelay int64 `toml:"retryDelay" default:"120" comment:"Execution retry delay in seconds"`
RetryError int64 `toml:"retryError" default:"3" comment:"Retry execution while this number of error is not reached"`
ExecutionHistory int `toml:"executionHistory" default:"10" comment:"Number of execution to keep"`
Disable bool `toml:"disable" default:"false" comment:"Disable all hooks executions"`
API service.APIServiceConfiguration `toml:"api" comment:"######################\n CDS API Settings \n######################"`
Addr string `toml:"addr" default:"" commented:"true" comment:"Listen address without port, example: 127.0.0.1" json:"addr"`
Port int `toml:"port" default:"8083" json:"port"`
} `toml:"http" comment:"######################\n CDS Hooks HTTP Configuration \n######################" json:"http"`
URL string `default:"http://localhost:8083" json:"url"`
URLPublic string `toml:"urlPublic" comment:"Public url for external call (webhook)" json:"urlPublic"`
RetryDelay int64 `toml:"retryDelay" default:"120" comment:"Execution retry delay in seconds" json:"retryDelay"`
RetryError int64 `toml:"retryError" default:"3" comment:"Retry execution while this number of error is not reached" json:"retryError"`
ExecutionHistory int `toml:"executionHistory" default:"10" comment:"Number of execution to keep" json:"executionHistory"`
Disable bool `toml:"disable" default:"false" comment:"Disable all hooks executions" json:"disable"`
API service.APIServiceConfiguration `toml:"api" comment:"######################\n CDS API Settings \n######################" json:"api"`
Cache struct {
TTL int `toml:"ttl" default:"60"`
TTL int `toml:"ttl" default:"60" json:"ttl"`
Redis struct {
Host string `toml:"host" default:"localhost:6379" comment:"If your want to use a redis-sentinel based cluster, follow this syntax! <clustername>@sentinel1:26379,sentinel2:26379,sentinel3:26379"`
Password string `toml:"password"`
} `toml:"redis" comment:"Connect CDS to a redis cache If you more than one CDS instance and to avoid losing data at startup"`
} `toml:"cache" comment:"######################\n CDS Hooks Cache Settings \n######################"`
Host string `toml:"host" default:"localhost:6379" comment:"If your want to use a redis-sentinel based cluster, follow this syntax! <clustername>@sentinel1:26379,sentinel2:26379,sentinel3:26379" json:"host"`
Password string `toml:"password" json:"-"`
} `toml:"redis" comment:"Connect CDS to a redis cache If you more than one CDS instance and to avoid losing data at startup" json:"redis"`
} `toml:"cache" comment:"######################\n CDS Hooks Cache Settings \n######################" json:"cache"`
}
8 changes: 4 additions & 4 deletions engine/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -545,25 +545,25 @@ See $ engine config command for more details.
}

func start(c context.Context, s service.Service, cfg interface{}, serviceName string) {
if err := serve(c, s, serviceName); err != nil {
if err := serve(c, s, serviceName, cfg); err != nil {
sdk.Exit("Service has been stopped: %s %v", serviceName, err)
}
}

func serve(c context.Context, s service.Service, serviceName string) error {
func serve(c context.Context, s service.Service, serviceName string, cfg interface{}) error {
ctx, cancel := context.WithCancel(c)
defer cancel()

// first register
if err := s.Register(s.Status); err != nil {
if err := s.Register(s.Status, cfg); err != nil {
log.Error("%s> Unable to register: %v", serviceName, err)
return err
}
log.Info("%s> Service registered", serviceName)

// start the heartbeat goroutine
go func() {
if err := s.Heartbeat(ctx, s.Status); err != nil {
if err := s.Heartbeat(ctx, s.Status, cfg); err != nil {
log.Error("%v", err)
cancel()
}
Expand Down
18 changes: 9 additions & 9 deletions engine/migrateservice/service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,16 @@ type dbmigservice struct {

// Configuration is the exposed type for database API configuration
type Configuration struct {
Name string `toml:"name" comment:"Name of this CDS Database Migrate service\n Enter a name to enable this service"`
URL string `default:"http://localhost:8087"`
Directory string `toml:"directory" comment:"SQL Migration files directory" default:"sql"`
Name string `toml:"name" comment:"Name of this CDS Database Migrate service\n Enter a name to enable this service" json:"name"`
URL string `default:"http://localhost:8087" json:"url"`
Directory string `toml:"directory" comment:"SQL Migration files directory" default:"sql" json:"directory"`
HTTP struct {
Addr string `toml:"addr" default:"" commented:"true" comment:"Listen address without port, example: 127.0.0.1"`
Port int `toml:"port" default:"8087"`
Insecure bool `toml:"insecure" default:"false" commented:"true" comment:"sslInsecureSkipVerify, set to true if you use a self-signed SSL on CDS API"`
} `toml:"http" comment:"######################\n CDS DB Migrate HTTP Configuration \n######################"`
API service.APIServiceConfiguration `toml:"api" comment:"######################\n CDS API Settings \n######################"`
DB database.DBConfiguration `toml:"db" comment:"################################\n Postgresql Database settings \n###############################"`
Addr string `toml:"addr" default:"" commented:"true" comment:"Listen address without port, example: 127.0.0.1" json:"addr"`
Port int `toml:"port" default:"8087" json:"port"`
Insecure bool `toml:"insecure" default:"false" commented:"true" comment:"sslInsecureSkipVerify, set to true if you use a self-signed SSL on CDS API" json:"insecure"`
} `toml:"http" comment:"######################\n CDS DB Migrate HTTP Configuration \n######################" json:"http"`
API service.APIServiceConfiguration `toml:"api" comment:"######################\n CDS API Settings \n######################" json:"api"`
DB database.DBConfiguration `toml:"db" comment:"################################\n Postgresql Database settings \n###############################" json:"db"`
}

// New instanciates a new API object
Expand Down
28 changes: 14 additions & 14 deletions engine/repositories/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,23 +21,23 @@ type Service struct {

// Configuration is the vcs configuration structure
type Configuration struct {
Name string `toml:"name" comment:"Name of this CDS Repositories Service\n Enter a name to enable this service"`
Basedir string `toml:"basedir" comment:"Root directory where the service will store all checked-out repositories"`
OperationRetention int `toml:"operation_retention" comment:"Operation retention in redis store (in days)" default:"5"`
RepositoriesRentention int `toml:"repositories_retention" comment:"Re retention on the filesystem (in days)" default:"10"`
Name string `toml:"name" comment:"Name of this CDS Repositories Service\n Enter a name to enable this service" json:"name"`
Basedir string `toml:"basedir" comment:"Root directory where the service will store all checked-out repositories" json:"basedir"`
OperationRetention int `toml:"operation_retention" comment:"Operation retention in redis store (in days)" default:"5" json:"operation_retention"`
RepositoriesRentention int `toml:"repositories_retention" comment:"Re retention on the filesystem (in days)" default:"10" json:"repositories_retention"`
HTTP struct {
Addr string `toml:"addr" default:"" commented:"true" comment:"Listen address without port, example: 127.0.0.1"`
Port int `toml:"port" default:"8085"`
} `toml:"http" comment:"######################\n CDS Repositories HTTP Configuration \n######################"`
URL string `default:"http://localhost:8085"`
API service.APIServiceConfiguration `toml:"api" comment:"######################\n CDS API Settings \n######################"`
Addr string `toml:"addr" default:"" commented:"true" comment:"Listen address without port, example: 127.0.0.1" json:"addr"`
Port int `toml:"port" default:"8085" json:"port"`
} `toml:"http" comment:"######################\n CDS Repositories HTTP Configuration \n######################" json:"http"`
URL string `default:"http://localhost:8085" json:"url"`
API service.APIServiceConfiguration `toml:"api" comment:"######################\n CDS API Settings \n######################" json:"api"`
Cache struct {
TTL int `toml:"ttl" default:"60"`
TTL int `toml:"ttl" default:"60" json:"ttl"`
Redis struct {
Host string `toml:"host" default:"localhost:6379" comment:"If your want to use a redis-sentinel based cluster, follow this syntax! <clustername>@sentinel1:26379,sentinel2:26379,sentinel3:26379"`
Password string `toml:"password"`
} `toml:"redis"`
} `toml:"cache" comment:"######################\n CDS Repositories Cache Settings \n######################"`
Host string `toml:"host" default:"localhost:6379" comment:"If your want to use a redis-sentinel based cluster, follow this syntax! <clustername>@sentinel1:26379,sentinel2:26379,sentinel3:26379" json:"host"`
Password string `toml:"password" json:"-"`
} `toml:"redis" json:"redis"`
} `toml:"cache" comment:"######################\n CDS Repositories Cache Settings \n######################" json:"cache"`
}

// Repo retiens a sdk.OperationRepo from an sdk.Operation
Expand Down
7 changes: 4 additions & 3 deletions engine/service/heartbeat.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func (c *Common) CommonMonitoring() sdk.MonitoringStatus {
}

// Heartbeat have to be launch as a goroutine, call DoHeartBeat each 30s
func (c *Common) Heartbeat(ctx context.Context, status func() sdk.MonitoringStatus) error {
func (c *Common) Heartbeat(ctx context.Context, status func() sdk.MonitoringStatus, cfg interface{}) error {
// no heartbeat for api
if c.Type == "api" {
return nil
Expand All @@ -50,7 +50,7 @@ func (c *Common) Heartbeat(ctx context.Context, status func() sdk.MonitoringStat
return ctx.Err()
case <-ticker.C:
// try to register, on success reset the failure count
if err := c.Register(status); err != nil {
if err := c.Register(status, cfg); err != nil {
heartbeatFailures++
log.Error("%s> Heartbeat> Register failed %d/%d", c.Name,
heartbeatFailures, c.MaxHeartbeatFailures)
Expand All @@ -67,7 +67,7 @@ func (c *Common) Heartbeat(ctx context.Context, status func() sdk.MonitoringStat
}

// Register the service to CDS api and store session hash.
func (c *Common) Register(status func() sdk.MonitoringStatus) error {
func (c *Common) Register(status func() sdk.MonitoringStatus, cfg interface{}) error {
// no need to register for api
if c.Type == "api" {
return nil
Expand All @@ -80,6 +80,7 @@ func (c *Common) Register(status func() sdk.MonitoringStatus) error {
Token: c.Token,
Type: c.Type,
MonitoringStatus: status(),
Config: cfg,
Version: sdk.VERSION,
})
if err != nil {
Expand Down
Loading

0 comments on commit 5995379

Please sign in to comment.