RabbitMQ stability test

metrics/check_metrics.py

@@ -86,6 +86,21 @@ def istio_requests_sanity(namespace):
     )
 
 
+def stability_query(source, test):
+    total = 'sum(rate(stability_outgoing_requests_total{source="%s"}[5m]))' % source
+    failure = 'sum(rate(stability_outgoing_requests_total{source="%s", succeeded="False"}[5m]))' % source
+    query = Query(
+        '{}: error rate'.format(test),
+        '{}/{}'.format(failure, total),
+        Alarm(
+            lambda errs: errs > 0,
+            'Error rate too high, expected no errors'
+        ),
+        'sum(stability_test_instances{test="%s"})' % test
+    )
+    return query
+
+
 class TestAlarms(unittest.TestCase):
     def test_pilot(self):
         queries = [
@@ -130,15 +145,13 @@ def test_load_test(self):
 
     def test_redis(self):
         queries = [
-            Query(
-                'Redis: error rate',
-                'sum(rate(stability_outgoing_requests_total{source="redis-client", succeeded="False"}[5m]))/sum(rate(stability_outgoing_requests_total{source="redis-client"}[5m]))',
-                Alarm(
-                    lambda errs: errs > 0,
-                    'Error rate too high, expected no errors'
-                ),
-                'sum(stability_test_instances{test="redis"})'
-            )
+            stability_query(source='redis-client', test='redis')
+        ]
+        self.run_queries(queries)
+
+    def test_rabbitmq(self):
+        queries = [
+            stability_query(source='rabbitmq-client', test='rabbitmq')
         ]
         self.run_queries(queries)
 

perf/docker/Dockerfile.rabbitmq

@@ -0,0 +1,11 @@
+FROM python:alpine
+
+RUN pip3 install -q pika prometheus_client
+
+ENV ADDRESS rabbitmq
+ENV USERNAME istio
+
+ADD rabbitmq/client.py /client.py
+ADD prom_client.py /prom_client.py
+
+CMD ["python3", "-u", "/client.py"]

perf/docker/redis/Dockerfile → perf/docker/Dockerfile.redis

@@ -8,7 +8,7 @@ ENV ADDRESS redis-master
 ENV SLAVE_PORT 6379
 ENV SLAVE_ADDRESS redis-slave
 
-ADD client.py /client.py
+ADD redis/client.py /client.py
 ADD prom_client.py /prom_client.py
 
 CMD ["python3", "-u", "/client.py"]

perf/docker/rabbitmq/client.py

@@ -0,0 +1,93 @@
+import os
+import time
+import logging
+import prom_client
+import pika
+import sys
+
+password = os.environ["PASSWORD"]
+username = os.environ["USERNAME"]
+address = os.environ["ADDRESS"]
+
+queue = 'queue'
+
+
+def with_metrics(f, valid=None):
+    return prom_client.attempt_request(
+        f,
+        source='rabbitmq-client',
+        destination='rabbitmq',
+        valid=valid
+    )
+
+
+def with_metrics_or_fail(f, valid=None):
+    r, success = with_metrics(f, valid)
+    if not success:
+        raise Exception("Function failed")
+    return r, success
+
+
+def setup_client():
+    credentials = pika.PlainCredentials(username, password)
+    connection = pika.BlockingConnection(
+        pika.ConnectionParameters(address, credentials=credentials))
+    channel = connection.channel()
+    channel.queue_declare(queue=queue)
+    return channel
+
+
+def send(channel, message):
+    with_metrics_or_fail(
+        lambda: channel.basic_publish(
+            exchange='',
+            routing_key=queue,
+            body=message
+        ),
+        valid=None
+    )
+
+
+def attempt_decode(s):
+    if s is None:
+        return ""
+    return s.decode('utf-8')
+
+
+def receive(channel, expected):
+    with_metrics_or_fail(
+        lambda: attempt_decode(
+            next(channel.consume(queue, inactivity_timeout=1))[2]),
+        valid=lambda resp: resp == expected
+    )
+
+
+def run_test():
+    pub, succeeded = with_metrics(setup_client)
+    if not succeeded:
+        logging.error("Failed to setup client")
+        sys.exit(1)
+    sub, succeeded = with_metrics(setup_client)
+    if not succeeded:
+        logging.error("Failed to setup client")
+        sys.exit(1)
+
+    while True:
+        message = "a message"
+        send(pub, message)
+        receive(sub, message)
+        time.sleep(.5)
+
+
+if __name__ == "__main__":
+    prom_client.report_metrics()
+    prom_client.report_running('rabbitmq')
+
+    time.sleep(10) # Wait for server
+
+    while True:
+        try:
+            run_test()
+        except Exception:
+            logging.warning("Rerunning test due to exception")
+            time.sleep(.5)

perf/istio-install/setup_istio.sh

@@ -39,6 +39,13 @@ function install_istio() {
 
   local outfile="$(download ${DIRNAME} ${release})"
 
+  if [[ ! -d "${DIRNAME}/${release}" ]];then
+      DN=$(mktemp -d)
+      tar -xzf "${outfile}" -C "${DN}" --strip-components 1
+      mv "${DN}/install/kubernetes/helm" "${DIRNAME}/${release}"
+      rm -Rf ${DN}
+  fi
+
   kubectl create ns istio-system || true
 
   if [[ -z "${DRY_RUN}" ]];then

perf/prometheus-install/.gitignore

@@ -0,0 +1 @@
+prometheus-operator

perf/stability/rabbitmq/Chart.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+name: rabbitmq
+version: 1.0.0
+description: Helm chart for testing rabbitmq on Istio
+keywords:
+  - istio
+  - performance
+sources:
+  - http://github.com/istio/istio
+engine: gotpl

perf/stability/rabbitmq/README.md

@@ -0,0 +1,13 @@
+# RabbitMq
+
+This test runs an instance of RabbitMQ, as well as a client that sends messages and later tries to read them.
+
+## Creating the template
+
+The base template was generated with:
+
+```
+helm template stable/rabbitmq --name rabbitmq --set rabbitmq.password=istio --set rabbitmq.username=istio
+```
+
+Then, the `securityContext` was shifted to the container level rather than the pod level on line 281.

perf/stability/rabbitmq/templates/client.yaml

@@ -0,0 +1,25 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rabbitmq-client
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: rabbitmq-client
+  template:
+    metadata:
+      labels:
+        app: rabbitmq-client
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+        prometheus.io/scheme: "http"
+    spec:
+      containers:
+        - name: rabbitmq
+          image: howardjohn/rabbitmq-perf-client
+          imagePullPolicy: Always
+          env:
+            - name: PASSWORD
+              value: istio