feat(sql): ability to override SqlHealthIndicator by dynamic config (#…

…3158)

- SqlHealthActivator is designed to halt queue processing when sql
persistence is in use and the connected database is in a read-only
mode. This is desired behavior when running in a single writer
environment, where write availability is impacted by a failover.
This PR adds a dynamic configuration property,
`sql.overrideSpringHealthCheck` that prevents Spring's Health
Indicator going unhealthy when connected to a read-only database,
allowing Orca to continue serving read-requests for task and pipeline
executions.

orca-sql/src/main/kotlin/com/netflix/spinnaker/config/SqlConfiguration.kt

@@ -128,9 +128,10 @@ class SqlConfiguration {
 
   @Bean("dbHealthIndicator") fun dbHealthIndicator(
     sqlHealthcheckActivator: SqlHealthcheckActivator,
-    sqlProperties: SqlProperties
+    sqlProperties: SqlProperties,
+    dynamicConfigService: DynamicConfigService
   ) =
-    SqlHealthIndicator(sqlHealthcheckActivator, sqlProperties.connectionPool.dialect)
+    SqlHealthIndicator(sqlHealthcheckActivator, sqlProperties.connectionPool.dialect, dynamicConfigService)
 
   @ConditionalOnProperty("execution-repository.sql.enabled")
   @ConditionalOnMissingBean(NotificationClusterLock::class)

orca-sql/src/main/kotlin/com/netflix/spinnaker/orca/sql/SqlHealthIndicator.kt

@@ -15,6 +15,7 @@
  */
 package com.netflix.spinnaker.orca.sql
 
+import com.netflix.spinnaker.kork.dynamicconfig.DynamicConfigService
 import org.jooq.SQLDialect
 import org.springframework.boot.actuate.health.AbstractHealthIndicator
 import org.springframework.boot.actuate.health.Health
@@ -26,11 +27,14 @@ import org.springframework.boot.actuate.health.Health
  */
 class SqlHealthIndicator(
   private val sqlHealthcheckActivator: SqlHealthcheckActivator,
-  private val sqlDialect: SQLDialect
+  private val sqlDialect: SQLDialect,
+  private val dynamicConfigService: DynamicConfigService
 ) : AbstractHealthIndicator() {
 
   override fun doHealthCheck(builder: Health.Builder) {
-    if (sqlHealthcheckActivator.enabled) {
+    val overrideHealth = dynamicConfigService.isEnabled("sql.override-spring-health-check", false)
+
+    if (overrideHealth || sqlHealthcheckActivator.enabled) {
       builder.up().withDetail("database", sqlDialect.name)
     } else {
       builder.down().withDetail("database", sqlDialect.name).let {