etcd: configuration for reproducing #3517

example/etcd/3517-reproduce has a configuration for reproducing
etcd-io/etcd#3517 .

The commit used for test is c645ac23c0093e2b0a93fa5f07a947344d7ef779,
which doesn't have a bugfix 8ebc9331111395fedbeedb3a346b302b95b147cc
for the above issue.

example/etcd/3517-reproduce/config.toml

@@ -0,0 +1,12 @@
+init = "init.sh"
+run = "run.sh"
+validate = "validate.sh"
+clean = "clean.sh"
+explorePolicy = "etcd"
+
+notCleanIfValidationFail= "true"
+
+[explorePolicyParam]
+ interval = 5
+
+# storageType = "mongodb"

example/etcd/3517-reproduce/materials/clean.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+#set -e # exit on an error
+. ${EQ_MATERIALS_DIR}/lib.sh
+
+# CLEAN_VETHS # old pipework needs CLEAN_VETHS
+
+########## Shutdown ##########
+KILL_DOCKER
+if [ -z $EQ_DISABLE ]; then
+    KILL_SWITCH
+    KILL_INSPECTOR
+fi
+
+INFO "Please run \"docker rmi ${DOCKER_IMAGE_NAME}\" if needed"
+
+exit 0

example/etcd/3517-reproduce/materials/etcd_inspector.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+import os
+from pyearthquake.signal.event import PacketEvent
+import base64
+
+from scapy.all import *
+
+ZMQ_ADDR = os.getenv('EQ_ETHER_ZMQ_ADDR')
+
+import pyearthquake
+from pyearthquake.inspector.ether import EtherInspectorBase
+
+LOG = pyearthquake.LOG.getChild(__name__)
+
+class EtcdPacket(Packet):
+    name = 'EtcdPacket'
+    longname = 'EtcdPacket'
+    # fields_desc=[ StrFixedLenField('type', '[NUL]', 5),
+    #               StrStopField('msg', '(NULL)', '\r\n') ]
+
+    def post_dissect(self, s):
+        try:
+            msg = {'asdf': 'hjkl'}
+            src_entity = 'server'
+            dst_entity =  'client'
+            self.event = PacketEvent.from_message(src_entity, dst_entity, msg)
+        except Exception as e:
+            LOG.exception(e)
+
+    def mysummary(self):
+        """
+        human-readable summary
+        """
+        try:
+            msg = self.event.option['message']
+            src_entity = self.event.option['src_entity']
+            dst_entity = self.event.option['dst_entity']
+            return self.sprintf('%s ==> %s EtcdPacket msg: %s' % \
+                                (src_entity, dst_entity, msg['asdf']))
+        except Exception as e:
+            LOG.exception(e)
+            return self.sprintf('ERROR')
+
+class EtcdInspector(EtherInspectorBase):
+
+    def __init__(self, zmq_addr):
+        super(EtcdInspector, self).__init__(zmq_addr)
+        self.regist_layer_on_tcp(EtcdPacket, 7001)
+
+    def map_packet_to_event(self, pkt):
+        return PacketEvent.from_message(src_entity="dummy", dst_entity="dummy", message=base64.b64encode(str((pkt))))
+        # if pkt.haslayer(EtcdPacket):
+        #     LOG.info('%s packet: %s', self.__class__.__name__, pkt[EtcdPacket].mysummary())            
+        #     event = pkt[EtcdPacket].event
+        #     LOG.info('mapped event=%s', event)
+        #     return event
+        # else:
+        #     LOG.info('%s unknown packet: %s', self.__class__.__name__, pkt.mysummary())
+        #     # hexdump.hexdump(str(pkt))
+        #     return None
+        # return PacketEvent.from_message(src_entity="dummy", dst_entity="dummy", message=base64.b64encode(str(pkt)))
+
+if __name__ == '__main__':
+    print ''
+    d = EtcdInspector(zmq_addr=ZMQ_ADDR)
+    d.start()

example/etcd/3517-reproduce/materials/etcd_switch.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+# NOTE: run with ryu-manager
+import os
+
+ZMQ_ADDR = os.getenv('EQ_ETHER_ZMQ_ADDR')
+
+from pyearthquake.middlebox.switch import RyuOF13Switch
+
+class EtcdSwitch(RyuOF13Switch):
+    def __init__(self, *args, **kwargs):
+        super(EtcdSwitch, self).__init__(tcp_ports=[4001, 7001],
+                                           udp_ports=[],
+                                           zmq_addr=ZMQ_ADDR,
+                                           *args, **kwargs)
+

example/etcd/3517-reproduce/materials/etcd_testbed/Dockerfile

@@ -0,0 +1,10 @@
+# Based on akihirosuda/zookeeper-dynamic
+FROM ubuntu:15.04
+MAINTAINER mitake
+
+ADD etcd /
+ADD init.sh /
+
+CMD ["bash", "init.sh"]
+
+EXPOSE 4001 7001

example/etcd/3517-reproduce/materials/etcd_testbed/init.sh

@@ -0,0 +1,3 @@
+#! /bin/bash
+
+etcd -name etcd$ETCDID -listen-client-urls http://192.168.42.$ETCDID:4001 -advertise-client-urls http://192.168.42.$ETCDID:4001 -listen-peer-urls http://192.168.42.$ETCDID:7001 -initial-advertise-peer-urls http://192.168.42.$ETCDID:7001 -initial-cluster-token etcd-cluster-1 -initial-cluster 'etcd1=http://192.168.42.1:7001,etcd2=http://192.168.42.2:7001,etcd3=http://192.168.42.3:7001' -initial-cluster-state new -heartbeat-interval=100 -election-timeout=500

example/etcd/3517-reproduce/materials/init.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e # exit on an error
+. ${EQ_MATERIALS_DIR}/lib.sh
+
+export ETCD_GIT_COMMIT=c645ac23c0093e2b0a93fa5f07a947344d7ef779
+
+CHECK_PREREQUISITES
+FETCH_ETCD
+BUILD_DOCKER_IMAGE
+
+exit 0

example/etcd/3517-reproduce/materials/lib.sh

@@ -0,0 +1,141 @@
+#!/bin/bash
+
+## CONFIG
+# EQ_DISABLE=1 # set to disable earthquake
+ETCD_GIT_COMMIT=${ETCD_GIT_COMMIT:-master}
+DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME:-etcd_testbed}
+
+ETCD_START_WAIT_SECS=${ETCD_START_WAIT_SECS:-10}
+PAUSE_ON_FAILURE=${PAUSE_ON_FAILURE:-0}
+
+## GENERIC FUNCS
+function INFO(){
+    echo -e "\e[104m\e[97m[INFO]\e[49m\e[39m $@"
+}
+
+function IMPORTANT(){
+    echo -e "\e[105m\e[97m[IMPORTANT]\e[49m\e[39m $@"
+}
+
+function SLEEP(){
+    echo -n $(INFO "Sleeping(${1} secs)..")
+    sleep ${1}
+    echo "Done"
+}
+
+function PAUSE(){
+    TMP=$(mktemp)
+    IMPORTANT "PAUSING. remove ${TMP} to continue"
+    while [ -e $TMP ]; do
+      sleep 3
+    done
+}
+
+## FUNCS (INIT)
+function CHECK_PREREQUISITES() {
+    INFO "Checking whether Docker is installed"
+    hash docker
+    INFO "Checking whether pipework is installed"
+    hash pipework
+    INFO "Checking whether ryu is installed"
+    hash ryu-manager
+    INFO "Checking whether ovsbr0 is configured as 192.168.42.254"
+    ip addr show ovsbr0
+    test X$(ip addr show ovsbr0 | sed -nEe 's/^[ \t]*inet[ \t]*([0-9.]+)\/.*$/\1/p') = X192.168.42.254
+}
+
+function FETCH_ETCD() {
+    ( cd ${EQ_MATERIALS_DIR}/etcd_testbed;
+      INFO "Fetching etcd"
+      git clone https://github.com/coreos/etcd.git
+      INFO "Checking out etcd@${ETCD_GIT_COMMIT}"
+      INFO "You can change the etcd version by setting ETCD_GIT_COMMIT"
+      cd etcd
+      git checkout ${ETCD_GIT_COMMIT}
+      ./build )
+}
+
+function BUILD_DOCKER_IMAGE() {
+    ( cd ${EQ_MATERIALS_DIR}/etcd_testbed;
+      docker_build_log=${EQ_MATERIALS_DIR}/docker-build.log
+      INFO "Building Docker Image ${DOCKER_IMAGE_NAME} (${docker_build_log})";
+      docker build -t ${DOCKER_IMAGE_NAME} . > ${docker_build_log} )
+}
+
+
+## FUNCS (BOOT)
+export EQ_ETHER_ZMQ_ADDR="ipc://${EQ_WORKING_DIR}/ether_inspector"
+
+function CHECK_PYTHONPATH() {
+    INFO "Checking PYTHONPATH(=${PYTHONPATH})"
+    ## used for etcd_switch and etcd_inspector
+    python -c "import pyearthquake"
+}    
+
+function START_SWITCH() {
+    INFO "Starting Earthquake Ethernet Switch"
+    ryu-manager --verbose ${EQ_MATERIALS_DIR}/etcd_switch.py > ${EQ_WORKING_DIR}/switch.log 2>&1 &
+    pid=$!
+    INFO "Switch PID: ${pid}"
+    echo ${pid} > ${EQ_WORKING_DIR}/switch.pid
+}
+
+function START_INSPECTOR() {
+    INFO "Starting Earthquake Ethernet Inspector"
+    python ${EQ_MATERIALS_DIR}/etcd_inspector.py > ${EQ_WORKING_DIR}/inspector.log 2>&1 &
+    pid=$!
+    INFO "Inspector PID: ${pid}"
+    echo ${pid} > ${EQ_WORKING_DIR}/inspector.pid
+}
+
+function START_DOCKER() {
+    for f in $(seq 1 3); do
+	    INFO "Starting Docker container etcd${f} from ${DOCKER_IMAGE_NAME}"
+      docker run -i -t -d -e ETCDID=${f} -h etcd${f} --name etcd${f} ${DOCKER_IMAGE_NAME} /bin/bash;
+    done
+}
+
+function SET_PIPEWORK() {
+    for f in $(seq 1 3); do
+	    INFO "Assigning 192.168.42.${f}/24 (ovsbr0) to etcd${f}"
+	    pipework ovsbr0 etcd${f} 192.168.42.${f}/24;
+    done
+}
+
+function START_ETCD() {
+    for f in $(seq 1 3); do 
+	  INFO "Starting etcd(id: ${f}) in Docker container etcd${f}"
+	  docker exec -d etcd${f} /bin/bash -c 'bash /init.sh > /log 2>&1';
+    done
+}
+
+## FUNCS (VALIDATION)
+
+## FUNCS (SHUTDOWN)
+function KILL_SWITCH() {
+    pid=$(cat ${EQ_WORKING_DIR}/switch.pid)
+    INFO "Killing Switch, PID: ${pid}"
+    kill -9 ${pid}
+}
+
+function KILL_INSPECTOR() {
+    pid=$(cat ${EQ_WORKING_DIR}/inspector.pid)
+    INFO "Killing Inspector, PID: ${pid}"
+    kill -9 ${pid}
+}
+
+function KILL_DOCKER() {
+    docker stop etcd1 etcd2 etcd3
+    for f in $(seq 1 3); do
+	   INFO "Killing Docker container etcd${f} (log:${EQ_WORKING_DIR}/etcd${f})"
+	   docker cp etcd${f}:/log ${EQ_WORKING_DIR}/etcd${f}
+	   docker rm -f etcd${f}
+    done
+}
+
+function CLEAN_VETHS(){
+    INFO "Removing garbage veths"
+    IMPORTANT "CLEAN_VETHS() has not been tested well"
+    garbages=$(ip a | egrep -o 'veth.*:' | sed -e s/://g)
+    for f in $garbages; do ip link delete $f; done
+}

example/etcd/3517-reproduce/materials/run.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+. ${EQ_MATERIALS_DIR}/lib.sh
+
+########## Boot ##########
+if [ -z $EQ_DISABLE ]; then
+    CHECK_PYTHONPATH
+    START_SWITCH
+    START_INSPECTOR
+    # TODO: check boot failure
+fi
+START_DOCKER
+SET_PIPEWORK
+START_ETCD
+#PAUSE
+SLEEP ${ETCD_START_WAIT_SECS} # the user should increase this, if could not reproduce the bug
+
+export ETCDCTL_PEERS="http://192.168.42.1:4001,http://192.168.42.2:4001,http://192.168.42.3:4001"
+$EQ_MATERIALS_DIR/etcd_testbed/etcd/bin/etcdctl --no-sync --timeout 10s set /k1 v1
+
+exit 0

example/etcd/3517-reproduce/materials/validate.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+. ${EQ_MATERIALS_DIR}/lib.sh
+
+export ETCDCTL_PEERS=http://192.168.42.1:4001,http://192.168.42.2:4001,http://192.168.42.3:4001
+echo "getting key"
+$EQ_MATERIALS_DIR/etcd_testbed/etcd/bin/etcdctl --no-sync --timeout 10s get /k
+result=$?
+echo "result: $result"
+
+exit $result