Merge remote-tracking branch 'origin/master' into CT-2125-event-for-a…

…ction-email

Gopkg.toml

@@ -27,12 +27,13 @@
 
 [[constraint]]
   name = "github.com/cloudtrust/common-service"
-  version = "v1.2.2"
-
+  branch = "master"
+  #version = "v1.2.2"
 
 [[constraint]]
   name = "github.com/cloudtrust/keycloak-client"
-  version = "v1.2.1"
+  branch = "master"
+  #version = "v1.2.1"
 
 [[constraint]]
   name = "github.com/go-kit/kit"

README.md

@@ -86,6 +86,39 @@ It is manadatory to load an authorization JSON file, if no authorization file is
 
 The keycloak event-emitter module sends all events to the bridge's event endpoint. The event emitter use HTTP with flatbuffers.
 
+### Monitoring of keycloak-bridge
+
+An endpoint allows to get a status of the Bridge and its components health.
+URL: ```http://{bridge.host}:{bridge.port}/health/check```
+
+Status example:
+```
+{
+ "name": "keycloak-bridge",
+ "state": "DOWN",
+ "details": [
+  {
+   "name": "Audit R/W",
+   "type": "database",
+   "state": "DOWN",
+   "message": "bad connection"
+  },
+  {
+   "name": "Config RO",
+   "type": "database",
+   "state": "UP",
+   "connection": "established"
+  },
+  {
+   "name": "Keycloak",
+   "type": "http",
+   "state": "DOWN",
+   "message": "Can't hit target: Get http://127.0.0.1:8080: dial tcp 127.0.0.1:8080: connectex: No connection could be made because the target machine actively refused it."
+  }
+ ]
+}
+```
+
 ## About monitoring
 
 Each HTTP request will trigger a set of operations that are going to be logged, measured, tracked and traced. For those information to be usable, we must be able to link the logs, metrics, traces and error report together. We achieve that with a unique correlation ID. For a given request, the same correlation ID will appear on the logs, metrics, traces and error report.

cmd/keycloakb/keycloak_bridge.go

@@ -14,6 +14,9 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/cloudtrust/common-service/database/sqltypes"
+	"github.com/cloudtrust/common-service/healthcheck"
+
 	cs "github.com/cloudtrust/common-service"
 	"github.com/cloudtrust/common-service/database"
 	errorhandler "github.com/cloudtrust/common-service/errors"
@@ -261,46 +264,56 @@ func main() {
 		defer tracer.Close()
 	}
 
-	var eventsDBConn database.CloudtrustDB
+	var eventsDBConn sqltypes.CloudtrustDB
 	{
 		var err error
-		eventsDBConn, err = auditRwDbParams.OpenDatabase()
+		eventsDBConn, err = database.NewReconnectableCloudtrustDB(auditRwDbParams)
 		if err != nil {
 			logger.Error(ctx, "msg", "could not create R/W DB connection for audit events", "error", err)
 			return
 		}
 	}
 
-	var eventsRODBConn database.CloudtrustDB
+	var eventsRODBConn sqltypes.CloudtrustDB
 	{
 		var err error
-		eventsRODBConn, err = auditRoDbParams.OpenDatabase()
+		eventsRODBConn, err = database.NewReconnectableCloudtrustDB(auditRoDbParams)
 		if err != nil {
 			logger.Error(ctx, "msg", "could not create RO DB connection for audit events", "error", err)
 			return
 		}
 	}
 
-	var configurationRwDBConn database.CloudtrustDB
+	var configurationRwDBConn sqltypes.CloudtrustDB
 	{
 		var err error
-		configurationRwDBConn, err = configRwDbParams.OpenDatabase()
+		configurationRwDBConn, err = database.NewReconnectableCloudtrustDB(configRwDbParams)
 		if err != nil {
 			logger.Error(ctx, "msg", "could not create DB connection for configuration storage (RW)", "error", err)
 			return
 		}
 	}
 
-	var configurationRoDBConn database.CloudtrustDB
+	var configurationRoDBConn sqltypes.CloudtrustDB
 	{
 		var err error
-		configurationRoDBConn, err = configRoDbParams.OpenDatabase()
+		configurationRoDBConn, err = database.NewReconnectableCloudtrustDB(configRoDbParams)
 		if err != nil {
 			logger.Error(ctx, "msg", "could not create DB connection for configuration storage (RO)", "error", err)
 			return
 		}
 	}
 
+	// Health check configuration
+	var healthChecker = healthcheck.NewHealthChecker(keycloakb.ComponentName, logger)
+	var healthCheckCacheDuration = c.GetDuration("livenessprobe-cache-duration") * time.Second
+	var httpTimeout = c.GetDuration("livenessprobe-http-timeout") * time.Second
+	healthChecker.AddDatabase("Audit R/W", eventsDBConn, healthCheckCacheDuration)
+	healthChecker.AddDatabase("Audit RO", eventsRODBConn, healthCheckCacheDuration)
+	healthChecker.AddDatabase("Config R/W", configurationRwDBConn, healthCheckCacheDuration)
+	healthChecker.AddDatabase("Config RO", configurationRoDBConn, healthCheckCacheDuration)
+	healthChecker.AddHTTPEndpoint("Keycloak", keycloakConfig.AddrAPI, httpTimeout, 200, healthCheckCacheDuration)
+
 	// Event service.
 	var eventEndpoints = event.Endpoints{}
 	{
@@ -525,6 +538,7 @@ func main() {
 
 		// Version.
 		route.Handle("/", commonhttp.MakeVersionHandler(keycloakb.ComponentName, ComponentID, keycloakb.Version, Environment, GitCommit))
+		route.Handle("/health/check", healthChecker.MakeHandler())
 
 		// Event.
 		var eventSubroute = route.PathPrefix("/event").Subrouter()
@@ -569,6 +583,7 @@ func main() {
 
 		// Version.
 		route.Handle("/", http.HandlerFunc(commonhttp.MakeVersionHandler(keycloakb.ComponentName, ComponentID, keycloakb.Version, Environment, GitCommit)))
+		route.Handle("/health/check", healthChecker.MakeHandler())
 
 		// Rights
 		var rightsHandler = configureRightsHandler(keycloakb.ComponentName, ComponentID, idGenerator, authorizationManager, keycloakClient, audienceRequired, tracer, logger)
@@ -712,6 +727,7 @@ func main() {
 
 		// Version.
 		route.Handle("/", http.HandlerFunc(commonhttp.MakeVersionHandler(keycloakb.ComponentName, ComponentID, keycloakb.Version, Environment, GitCommit)))
+		route.Handle("/health/check", healthChecker.MakeHandler())
 
 		// Account
 		var updatePasswordHandler = configureAccountHandler(keycloakb.ComponentName, ComponentID, idGenerator, keycloakClient, audienceRequired, tracer, logger)(accountEndpoints.UpdatePassword)
@@ -852,6 +868,10 @@ func config(ctx context.Context, logger log.Logger) *viper.Viper {
 	// Debug routes enabled.
 	v.SetDefault("pprof-route-enabled", true)
 
+	// Liveness probe
+	v.SetDefault("livenessprobe-http-timeout", 5)
+	v.SetDefault("livenessprobe-cache-duration", 10)
+
 	// First level of override.
 	pflag.String("config-file", v.GetString("config-file"), "The configuration file path can be relative or absolute.")
 	pflag.String("authorization-file", v.GetString("authorization-file"), "The authorization file path can be relative or absolute.")