probes 3, relies on status

os-eap-probes/3.0/added/livenessProbe.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+
+. "$JBOSS_HOME/bin/probe_common.sh"
+
+if [ true = "${DEBUG}" ] ; then
+    # short circuit liveness check in dev mode
+    exit 0
+fi
+
+OUTPUT=/tmp/liveness-output
+ERROR=/tmp/liveness-error
+LOG=/tmp/liveness-log
+
+DEBUG_SCRIPT=false
+PROBE_IMPL="probe.eap.dmr.EapProbe probe.eap.dmr.HealthCheckProbe"
+
+if [ $# -gt 0 ] ; then
+    DEBUG_SCRIPT=$1
+fi
+
+if [ $# -gt 1 ] ; then
+    PROBE_IMPL=$2
+fi
+
+if [ "$DEBUG_SCRIPT" = "true" ]; then
+    DEBUG_OPTIONS="--debug --logfile $LOG --loglevel DEBUG"
+fi
+
+if python $JBOSS_HOME/bin/probes/runner.py -c READY -c NOT_READY $DEBUG_OPTIONS $PROBE_IMPL; then
+    exit 0
+fi
+
+if [ "$DEBUG_SCRIPT" == "true" ]; then
+    jps -v | grep standalone | awk '{print $1}' | xargs kill -3
+fi
+
+exit 1
+

os-eap-probes/3.0/added/probe_common.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+# common routines for readiness and liveness probes
+
+# jboss-cli.sh sometimes hangs indefinitely. Send SIGTERM after CLI_TIMEOUT has passed
+# and failing that SIGKILL after CLI_KILLTIME, to ensure that it exits
+CLI_TIMEOUT=10s
+CLI_KILLTIME=30s
+
+EAP_7_WARNING="Warning! The CLI is running in a non-modular environment and cannot load commands from management extensions."
+
+run_cli_cmd() {
+    cmd="$1"
+
+    #Default for EAP7
+    cli_port=9990
+
+    if [ -f "$JBOSS_HOME/bin/run.sh" ]; then
+      version=$($JBOSS_HOME/bin/run.sh -V)
+      if [[ "$version" == *"JBoss Enterprise Application Platform 6"* ]]; then
+        cli_port=9999
+      fi
+    fi
+
+    if [ -n "${PORT_OFFSET}" ]; then
+      cli_port=$(($cli_port+$PORT_OFFSET))
+    fi
+
+    timeout --foreground -k "$CLI_KILLTIME" "$CLI_TIMEOUT" java -jar $JBOSS_HOME/bin/client/jboss-cli-client.jar --connect --controller=localhost:${cli_port} "$cmd" | grep -v "$EAP_7_WARNING"
+}
+
+is_eap7() {
+    run_cli_cmd "version" | grep -q "^JBoss AS product: JBoss EAP 7"
+}
+
+# Additional check necessary for EAP7, see CLOUD-615
+deployments_failed() {
+    ls -- /deployments/*failed >/dev/null 2>&1 || (is_eap7 && run_cli_cmd "deployment-info" | grep -q FAILED)
+}
+
+list_failed_deployments() {
+    ls -- /deployments/*failed >/dev/null 2>&1 && \
+        echo /deployments/*.failed | sed "s+^/deployments/\(.*\)\.failed$+\1+"
+}

os-eap-probes/3.0/added/probes/probe/api.py

@@ -0,0 +1,184 @@
+"""
+Copyright 2017 Red Hat, Inc.
+
+Red Hat licenses this file to you under the Apache License, version
+2.0 (the "License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied.  See the License for the specific language governing
+permissions and limitations under the License.
+"""
+
+import json
+import logging
+import sys
+
+from enum import Enum
+
+def qualifiedClassName(obj):
+    """
+    Utility method for returning the fully qualified class name of an instance.
+    Objects must be instances of "new classes."
+    """
+
+    return obj.__module__ + "." + type(obj).__name__
+
+class Status(Enum):
+    """
+    Represents the outcome of a test.
+        HARD_FAILURE: An unrecoverable failure, causing an immediate failure of
+            the probes, i.e. no extra tries to see if the probe will pass.
+        FAILURE: A test failed, but may succeed on a subsequent execution
+        NOT_READY: The functionality being tested is not in a failed state, but
+        is also not ready, e.g. it may still be starting up, rebalancing, etc.
+        READY: The functionality being tested is ready to handle requests.
+    """
+
+    HARD_FAILURE = 1
+    FAILURE = 2
+    NOT_READY = 4
+    READY = 8
+
+    def __str__(self):
+        return self.name
+
+    def __cmp__(self, other):
+        if type(other) is self.__class__:
+            return self.value - other.value
+        return NotImplemented
+
+    def __le__(self, other):
+        if type(other) is self.__class__:
+            return self.value <= other.value
+        return NotImplemented
+
+    def __lt__(self, other):
+        if type(other) is self.__class__:
+            return self.value < other.value
+        return NotImplemented
+
+    def __ge__(self, other):
+        if type(other) is self.__class__:
+            return self.value >= other.value
+        return NotImplemented
+
+    def __gt__(self, other):
+        if type(other) is self.__class__:
+            return self.value > other.value
+        return NotImplemented
+
+class Test(object):
+    """
+    An object which provides a query and evaluates the response.  A Probe may
+    consist of many tests, which determine the liveness or readiness of the
+    server.
+    """
+
+    def __init__(self, query):
+        self.query = query
+
+    def getQuery(self):
+        """
+        Returns the query used by this test.  The return value is Probe
+        specific.  Many Probe implementations use JSON for submitting queries,
+        which means this function would return a dict.
+        """
+        return self.query
+
+    def evaluate(self, results):
+        """
+        Evaluate the response from the server, returning Status and messages.
+        messages should be returned as an object, list or dict. 
+        """
+        raise NotImplementedError("Implement evaluate() for Test: " + qualifiedClassName(self))
+
+class Probe(object):
+    """
+    Runs a series of tests against a server to determine its readiness or
+    liveness.
+    """
+
+    def __init__(self, tests = []):
+        self.tests = tests
+
+    def addTest(self, test):
+        """
+        Adds a test to this Probe.  The Test must provide a query that is
+        compatible with the Probe implementation (e.g. a DMR request formatted
+        as JSON).  The Test must be capable of understanding the results
+        returned by the Probe (e.g. a JSON response from DMR).
+        """
+
+        self.tests.append(test)
+
+    def execute(self):
+        """
+        Executes the queries and evaluates the tests and returns a set of Status
+        and messages collected for each test.
+        """
+
+        raise NotImplementedError("Implement execute() for Probe: " + qualifiedClassName(self))
+
+class BatchingProbe(Probe):
+    """
+    Base class which supports batching queries to be sent to a server and
+    splitting the results to correspond with the individual tests.
+    """
+
+    def __init__(self, tests = []):
+        super(BatchingProbe, self).__init__(tests)
+        self.logger = logging.getLogger(qualifiedClassName(self))
+
+    def execute(self):
+        self.logger.info("Executing the following tests: [%s]", ", ".join(qualifiedClassName(test) for test in self.tests))
+        request = self.createRequest()
+
+        try:
+            results = self.sendRequest(request)
+            status = set()
+            output = {}
+            for index, test in enumerate(self.tests):
+                self.logger.info("Executing test %s", qualifiedClassName(test))
+                try:
+                    testResults = self.getTestInput(results, index)
+                    if self.logger.isEnabledFor(logging.DEBUG):
+                        self.logger.debug("Test input = %s", json.dumps(testResults, indent=4, separators=(',', ': ')))
+                    (state, messages) = test.evaluate(testResults)
+                    self.logger.info("Test %s returned status %s", qualifiedClassName(test), str(state))
+                    status.add(state)
+                    output[qualifiedClassName(test)] = messages
+                except:
+                    self.logger.exception("Unexpected failure running test %s", qualifiedClassName(test))
+                    status.add(Status.FAILURE)
+                    output[qualifiedClassName(test)] = "Exception executing test: %s" % (sys.exc_info()[1])
+            return (status, output)
+        except:
+            self.logger.exception("Unexpected failure sending probe request")
+            return (set([Status.FAILURE]), "Error sending probe request: %s" % (sys.exc_info()[1]))
+
+    def createRequest(self):
+        """
+        Create the request to send to the server.  Subclasses should include the
+        queries from all tests in the request.
+        """
+
+        raise NotImplementedError("Implement createRequest() for BatchingProbe: " + qualifiedClassName(self))
+
+    def sendRequest(self, request):
+        """
+        Send the request to the server.
+        """
+
+        raise NotImplementedError("Implement sendRequest() for BatchingProbe: " + qualifiedClassName(self))
+
+    def getTestInput(self, results, testIndex):
+        """
+        Return the results specific to the indexed test.
+        """
+
+        raise NotImplementedError("Implement getTestInput() for BatchingProbe: " + qualifiedClassName(self))

os-eap-probes/3.0/added/probes/probe/dmr.py

@@ -0,0 +1,121 @@
+"""
+Copyright 2017 Red Hat, Inc.
+
+Red Hat licenses this file to you under the Apache License, version
+2.0 (the "License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied.  See the License for the specific language governing
+permissions and limitations under the License.
+"""
+
+import json
+import logging
+import os
+import requests
+import sys
+
+from collections import OrderedDict
+
+from probe.api import qualifiedClassName, BatchingProbe, Status, Test
+
+class DmrProbe(BatchingProbe):
+    """
+    A Probe implementation that sends a batch of queries to a server using EAP's
+    management interface.  Tests should provide JSON queries specific to EAP's
+    management interface and should be able to handle DMR results.
+    """
+
+    def __init__(self, tests = []):
+        super(DmrProbe, self).__init__(tests)
+        self.logger = logging.getLogger(qualifiedClassName(self))
+        self.__readConfig()
+
+    def __readConfig(self):
+        """
+        Configuration consists of:
+            host: localhost
+            port: 9990 + $PORT_OFFSET
+            user: $ADMIN_USERNAME
+            password: $ADMIN_PASSWORD
+        """
+
+        self.host = "localhost"
+        self.port = 9990 + int(os.getenv('PORT_OFFSET', 0))
+        self.user = os.getenv('ADMIN_USERNAME')
+        self.password = os.getenv('ADMIN_PASSWORD')
+        if self.password != "":
+          if self.user is None or self.user == "":
+            self.user = os.getenv('DEFAULT_ADMIN_USERNAME')
+        self.logger.debug("Configuration set as follows: host=%s, port=%s, user=%s, password=***", self.host, self.port, self.user)
+
+    def getTestInput(self, results, testIndex):
+        return list(results["result"].values())[testIndex]
+
+    def createRequest(self):
+        steps = []
+        for test in self.tests:
+            steps.append(test.getQuery())
+        return {
+                    "operation": "composite",
+                    "address": [],
+                    "json.pretty": 1,
+                    "steps": steps
+                }
+
+    def sendRequest(self, request):
+        url = "http://%s:%s/management" % (self.host, self.port)
+        self.logger.info("Sending probe request to %s", url)
+        if self.logger.isEnabledFor(logging.DEBUG):
+            self.logger.debug("Probe request = %s", json.dumps(request, indent=4, separators=(',', ': ')))
+        response = requests.post(
+            url,
+            json = request,
+            headers = {
+                "Accept": "text/plain"
+            },
+            proxies = {
+                "http": None,
+                "https": None
+            },
+            auth = requests.auth.HTTPDigestAuth(self.user, self.password) if self.user else None,
+            verify = False
+        )
+        self.logger.debug("Probe response: %s", response)
+
+        if response.status_code != 200:
+            """
+            See if this non-200 represents an unusable response, or just a failure
+            response because one of the test steps failed, in which case we pass the
+            response to the tests to let them decide how to handle things
+            """
+            self.failUnusableResponse(response, request, url)
+
+        return response.json(object_pairs_hook = OrderedDict)
+
+    def failUnusableResponse(self, response, request, url):
+        respDict = None
+        try:
+            respDict = response.json(object_pairs_hook = OrderedDict)
+        except ValueError:
+            self.logger.debug("Probe request failed with no parseable json response")
+
+        unusable = not respDict or not respDict["outcome"] or respDict["outcome"] != "failed" or not respDict["result"]
+        if not unusable:
+            """
+            An outcome=failed response is usable if the result node has an element for each test
+            """
+            stepResults = list(respDict["result"].values())
+            for index, test in enumerate(self.tests):
+                if not stepResults[index]:
+                    unusable = True
+                    break;
+
+        if unusable:
+            self.logger.error("Probe request failed.  Status code: %s", response.status_code)
+            raise Exception("Probe request failed, code: " + str(response.status_code) + str(url) + str(request) + str(response.json(object_pairs_hook = OrderedDict)))