Adds a HealthCheckService (#77)

* feat: Adds a HealthCheckService

with an abstract implementation that runs checks periodically in the
background. Adds the /about/health endpoint.

* Add tests for /about/health.

jicoco-kotlin/pom.xml

@@ -44,6 +44,16 @@
             <artifactId>jitsi-utils-kotlin</artifactId>
             <version>${jitsi-utils.version}</version>
         </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>jicoco</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>jitsi-android-osgi</artifactId>
+            <version>1.0-20190327.160432-3</version>
+        </dependency>
         <dependency>
             <groupId>com.typesafe</groupId>
             <artifactId>config</artifactId>
@@ -130,7 +140,17 @@
         </snapshots>
         <url>https://github.com/jitsi/jitsi-maven-repository/raw/master/releases/</url>
     </repository>
+    <repository>
+       <id>jitsi-maven-repository-snapshots</id>
+       <layout>default</layout>
+       <name>Jitsi Maven Repository (Snapshots)</name>
+       <releases>
+           <enabled>false</enabled>
+       </releases>
+       <snapshots>
+           <enabled>true</enabled>
+       </snapshots>
+       <url>https://github.com/jitsi/jitsi-maven-repository/raw/master/snapshots/</url>
+    </repository>
     </repositories>
-
-
 </project>

jicoco-kotlin/src/main/kotlin/org/jitsi/health/AbstractHealthCheckService.kt

@@ -0,0 +1,181 @@
+/*
+ * Copyright @ 2018 - present 8x8, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jitsi.health
+
+import org.jitsi.utils.concurrent.PeriodicRunnable
+import org.jitsi.utils.concurrent.RecurringRunnableExecutor
+import org.jitsi.utils.logging2.Logger
+import org.jitsi.utils.logging2.LoggerImpl
+import org.osgi.framework.BundleActivator
+import org.osgi.framework.BundleContext
+import java.time.Clock
+import java.time.Duration
+import java.time.Instant
+
+abstract class AbstractHealthCheckService @JvmOverloads constructor(
+    val interval: Duration = Duration.ofSeconds(10),
+    /**
+     * If no health checks have been performed in the last {@code timeout}
+     * period, the service is considered unhealthy.
+     */
+    val timeout: Duration = Duration.ofSeconds(30),
+    /**
+     * The maximum duration that a call to {@link #performCheck()} is allowed
+     * to take. If a call takes longer, the service is considered unhealthy. A
+     * value of {@code null} indicates no max check duration.
+     * <p>
+     * Note that if a check never completes, we rely on {@link #timeout} instead.
+     */
+    private val maxCheckDuration: Duration = Duration.ofSeconds(3),
+    /**
+     * If set, a single health check failure after the initial
+     * {@link #STICKY_FAILURES_GRACE_PERIOD}, will be result in the service
+     * being permanently unhealthy.
+     */
+    private val stickyFailures: Boolean = false,
+    /**
+     * Failures in this period (since the start of the service) are not sticky.
+     */
+    private val stickyFailuresGracePeriod: Duration = stickyFailuresGracePeriodDefault,
+    private val clock: Clock = Clock.systemUTC()
+): BundleActivator, HealthCheckService, PeriodicRunnable(interval.toMillis())
+{
+    private val logger: Logger = LoggerImpl(javaClass.name)
+
+    /**
+     * The executor which runs {@link #performCheck()} periodically.
+     */
+    private var executor: RecurringRunnableExecutor? = null
+
+    /**
+     * The exception resulting from the last health check. When the health
+     * check is successful, this is {@code null}.
+     */
+    private var lastResult: Exception? = null
+
+    /**
+     * The time the last health check finished being performed.
+     */
+    private var lastResultTime = Instant.MIN
+
+    /**
+     * The time when this service was started.
+     */
+    private var serviceStartTime = Instant.MAX
+
+    /**
+     * Whether we've seen a health check failure (after the grace period).
+     */
+    private var hasFailed = false
+
+    @Throws(Exception::class)
+    override fun start(bundleContext: BundleContext)
+    {
+        bundleContext.registerService(HealthCheckService::class.java, this, null)
+
+        if (executor == null)
+        {
+            executor = RecurringRunnableExecutor(javaClass.name)
+        }
+        executor!!.registerRecurringRunnable(this)
+        serviceStartTime = clock.instant()
+
+        logger.info("Started with interval=$period, timeout=$timeout, maxDuration=$maxCheckDuration, stickyFailures=$stickyFailures.")
+    }
+
+    @Throws(Exception::class)
+    override fun stop(bundleContext: BundleContext)
+    {
+        executor?.apply {
+            deRegisterRecurringRunnable(this@AbstractHealthCheckService)
+            close()
+        }
+        executor = null
+        logger.info("Stopped")
+    }
+
+    /**
+     * Returns the result of the last performed health check, or a new exception
+     * if no health check has been perform recently.
+     * @return
+     */
+    override fun getResult(): Exception? {
+        val timeSinceLastResult: Duration = Duration.between(lastResultTime, clock.instant())
+        if (timeSinceLastResult > timeout) {
+            return Exception("No health checks performed recently, the last result was $timeSinceLastResult ago.")
+        }
+        return lastResult
+    }
+
+    /**
+     * Performs a check to determine whether this service is healthy.
+     * This is executed periodically in {@link #executor} since it may block.
+     *
+     * @throws Exception if the service is not healthy.
+     */
+    @Throws(Exception::class)
+    protected abstract fun performCheck()
+
+    /**
+     * Performs a health check and updates this instance's state. Runs
+     * periodically in {@link #executor}.
+     */
+    override fun run()
+    {
+        super.run()
+
+        val checkStart = clock.instant()
+        var exception: Exception? = null
+
+        try {
+            performCheck()
+        }
+        catch (e: Exception) {
+            exception = e
+
+            val now = clock.instant()
+            val timeSinceStart = Duration.between(serviceStartTime, now)
+            if (timeSinceStart > stickyFailuresGracePeriod) {
+                hasFailed = true
+            }
+        }
+
+        lastResultTime = clock.instant()
+        val checkDuration = Duration.between(checkStart, lastResultTime)
+        if (checkDuration > maxCheckDuration) {
+            exception = Exception("Performing a health check took too long: $checkDuration")
+        }
+
+        lastResult = if (stickyFailures && hasFailed && exception == null) {
+            // We didn't fail this last test, but we've failed before and
+            // sticky failures are enabled.
+            Exception("Sticky failure.")
+        } else {
+            exception
+        }
+
+        if (exception == null) {
+            logger.info(
+                    "Performed a successful health check in $checkDuration. Sticky failure: ${stickyFailures && hasFailed}")
+        } else {
+            logger.error( "Health check failed in $checkDuration:", exception)
+        }
+    }
+
+    companion object {
+        val stickyFailuresGracePeriodDefault = Duration.ofMinutes(5)
+    }
+}

jicoco-test-kotlin/src/main/kotlin/org/jitsi/config/ConfigTest.kt

@@ -72,6 +72,12 @@ object LongMockConfigValueGenerator : MockConfigValueGenerator<Long, Long> {
     }
 }
 
+object DurationMockConfigValueGenerator : MockConfigValueGenerator<Duration, Duration> {
+    override fun gen(): MockConfigValue<Duration, Duration> {
+        return Random().nextLong().let { MockConfigValue(Duration.ofMillis(it), Duration.ofMillis(it)) }
+    }
+}
+
 /**
  * We don't use a singleton here because collisions would be too common
  */
@@ -89,3 +95,8 @@ object DurationToLongMockConfigValueGenerator : TransformingMockConfigValueGener
     { Duration.ofMillis(Random().nextLong()) },
     { it.toMillis() }
 )
+
+object LongToDurationMockConfigValueGenerator : TransformingMockConfigValueGenerator<Long, Duration>(
+    { Random().nextLong() },
+    { Duration.ofMillis(it) }
+)

jicoco/src/main/java/org/jitsi/health/HealthCheckService.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright @ 2018 - present 8x8, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jitsi.health;
+
+public interface HealthCheckService
+{
+    /**
+     * Returns the result of a health check: either {@code null} indicating
+     * the service is healthy, or the exception which caused the health
+     * check failure otherwise.
+     */
+    Exception getResult();
+}

jicoco/src/main/java/org/jitsi/health/HealthCheckServiceProvider.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright @ 2018 - present 8x8, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jitsi.health;
+
+import org.jitsi.osgi.*;
+import org.osgi.framework.*;
+
+public class HealthCheckServiceProvider extends OsgiServiceProvider<HealthCheckService>
+{
+    public HealthCheckServiceProvider(BundleContext bundleContext)
+    {
+        super(bundleContext, HealthCheckService.class);
+    }
+}

jicoco/src/main/java/org/jitsi/rest/Health.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright @ 2018 - present 8x8, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jitsi.rest;
+
+import org.jitsi.health.*;
+
+import javax.inject.*;
+import javax.servlet.http.*;
+import javax.ws.rs.*;
+import javax.ws.rs.core.*;
+
+/**
+ * A generic health check REST endpoint which checks the health using a
+ * a {@link HealthCheckService}, if one is present.
+ *
+ */
+@Path("/about/health")
+public class Health
+{
+    @Inject
+    protected HealthCheckServiceProvider healthCheckServiceProvider;
+
+    @GET
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response getHealth()
+    {
+        HealthCheckService healthCheckService = healthCheckServiceProvider.get();
+        if (healthCheckService == null)
+        {
+            throw new NotFoundException();
+        }
+
+        Exception status = healthCheckServiceProvider.get().getResult();
+        if (status != null)
+        {
+            return Response
+                    .status(HttpServletResponse.SC_INTERNAL_SERVER_ERROR)
+                    .entity(status.getMessage())
+                    .build();
+        }
+
+        return Response.ok().build();
+    }
+}