qa/suites/rbd: thrash rbd-mirror daemon during tests

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

qa/suites/rbd/mirror-ha/cluster/2-node.yaml

@@ -0,0 +1,31 @@
+meta:
+- desc: 2 ceph clusters with 1 mon and 3 osds each
+roles:
+- - cluster1.mon.a
+  - cluster1.mgr.x
+  - cluster2.mgr.x
+  - cluster1.osd.0
+  - cluster1.osd.1
+  - cluster1.osd.2
+  - cluster1.client.0
+  - cluster2.client.0
+- - cluster2.mon.a
+  - cluster2.osd.0
+  - cluster2.osd.1
+  - cluster2.osd.2
+  - cluster1.client.mirror
+  - cluster1.client.mirror.0
+  - cluster1.client.mirror.1
+  - cluster1.client.mirror.2
+  - cluster1.client.mirror.3
+  - cluster1.client.mirror.4
+  - cluster1.client.mirror.5
+  - cluster1.client.mirror.6
+  - cluster2.client.mirror
+  - cluster2.client.mirror.0
+  - cluster2.client.mirror.1
+  - cluster2.client.mirror.2
+  - cluster2.client.mirror.3
+  - cluster2.client.mirror.4
+  - cluster2.client.mirror.5
+  - cluster2.client.mirror.6

qa/suites/rbd/mirror-ha/users

@@ -0,0 +1 @@
+../mirror/users

qa/suites/rbd/mirror-ha/workloads/rbd-mirror-ha-workunit.yaml

@@ -1,11 +1,19 @@
 meta:
 - desc: run the rbd_mirror_ha.sh workunit to test the rbd-mirror daemon
+overrides:
+  ceph:
+    conf:
+      # override to make these names predictable
+      client.mirror.4:
+        admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
+        pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid
+      client.mirror.5:
+        admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
+        pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid
+      client.mirror.6:
+        admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
+        pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid
 tasks:
-- exec:
-    cluster1.client.mirror:
-    - ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd' osd 'profile rbd'
-    cluster2.client.mirror:
-    - ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd' osd 'profile rbd'
 - workunit:
     clients:
       cluster1.client.mirror: [rbd/rbd_mirror_ha.sh]

qa/suites/rbd/mirror/cluster/2-node.yaml

@@ -3,15 +3,23 @@ meta:
 roles:
 - - cluster1.mon.a
   - cluster1.mgr.x
+  - cluster2.mgr.x
   - cluster1.osd.0
   - cluster1.osd.1
   - cluster1.osd.2
   - cluster1.client.0
   - cluster2.client.0
 - - cluster2.mon.a
-  - cluster2.mgr.x
   - cluster2.osd.0
   - cluster2.osd.1
   - cluster2.osd.2
   - cluster1.client.mirror
+  - cluster1.client.mirror.0
+  - cluster1.client.mirror.1
+  - cluster1.client.mirror.2
+  - cluster1.client.mirror.3
   - cluster2.client.mirror
+  - cluster2.client.mirror.0
+  - cluster2.client.mirror.1
+  - cluster2.client.mirror.2
+  - cluster2.client.mirror.3

qa/suites/rbd/mirror/rbd-mirror/four-per-cluster.yaml

@@ -0,0 +1,23 @@
+meta:
+- desc: run four rbd-mirror daemons per cluster
+tasks:
+- rbd-mirror:
+    client: cluster1.client.mirror.0
+- rbd-mirror:
+    client: cluster1.client.mirror.1
+- rbd-mirror:
+    client: cluster1.client.mirror.2
+- rbd-mirror:
+    client: cluster1.client.mirror.3
+- rbd-mirror:
+    client: cluster2.client.mirror.0
+- rbd-mirror:
+    client: cluster2.client.mirror.1
+- rbd-mirror:
+    client: cluster2.client.mirror.2
+- rbd-mirror:
+    client: cluster2.client.mirror.3
+- rbd-mirror-thrash:
+    cluster: cluster1
+- rbd-mirror-thrash:
+    cluster: cluster2

qa/suites/rbd/mirror/users/mirror.yaml

@@ -0,0 +1,18 @@
+meta:
+- desc: configure the permissions for client.mirror
+overrides:
+  ceph:
+    conf:
+      # override to make these names predictable
+      client.mirror.0:
+        admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
+        pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid
+      client.mirror.1:
+        admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
+        pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid
+      client.mirror.2:
+        admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
+        pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid
+      client.mirror.3:
+        admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok
+        pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid

qa/suites/rbd/mirror/workloads/rbd-mirror-stress-workunit.yaml

@@ -7,6 +7,7 @@ tasks:
     env:
       # override workunit setting of CEPH_ARGS='--cluster'
       CEPH_ARGS: ''
+      RBD_MIRROR_INSTANCES: '4'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
       RBD_MIRROR_USE_RBD_MIRROR: '1'
     timeout: 6h

qa/suites/rbd/mirror/workloads/rbd-mirror-workunit.yaml

@@ -7,5 +7,6 @@ tasks:
     env:
       # override workunit setting of CEPH_ARGS='--cluster'
       CEPH_ARGS: ''
+      RBD_MIRROR_INSTANCES: '4'
       RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
       RBD_MIRROR_USE_RBD_MIRROR: '1'

qa/workunits/rbd/rbd_mirror.sh

@@ -10,7 +10,7 @@
 . $(dirname $0)/rbd_mirror_helpers.sh
 
 testlog "TEST: add image and test replay"
-start_mirror ${CLUSTER1}
+start_mirrors ${CLUSTER1}
 image=test
 create_image ${CLUSTER2} ${POOL} ${image}
 set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1"
@@ -27,11 +27,11 @@ compare_image_meta ${CLUSTER1} ${POOL} ${image} "key1" "value1"
 compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2"
 
 testlog "TEST: stop mirror, add image, start mirror and test replay"
-stop_mirror ${CLUSTER1}
+stop_mirrors ${CLUSTER1}
 image1=test1
 create_image ${CLUSTER2} ${POOL} ${image1}
 write_image ${CLUSTER2} ${POOL} ${image1} 100
-start_mirror ${CLUSTER1}
+start_mirrors ${CLUSTER1}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'master_position'
@@ -48,87 +48,87 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_
 compare_images ${POOL} ${image}
 
 testlog "TEST: stop/start/restart mirror via admin socket"
-admin_daemon ${CLUSTER1} rbd mirror stop
+admin_daemons ${CLUSTER1} rbd mirror stop
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
 
-admin_daemon ${CLUSTER1} rbd mirror start
+admin_daemons ${CLUSTER1} rbd mirror start
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
 
-admin_daemon ${CLUSTER1} rbd mirror restart
+admin_daemons ${CLUSTER1} rbd mirror restart
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
 
-admin_daemon ${CLUSTER1} rbd mirror stop
+admin_daemons ${CLUSTER1} rbd mirror stop
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
 
-admin_daemon ${CLUSTER1} rbd mirror restart
+admin_daemons ${CLUSTER1} rbd mirror restart
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
 
-admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
+admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
 
-admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
+admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 
-admin_daemon ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}
+admin_daemons ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
 
-admin_daemon ${CLUSTER1} rbd mirror restart ${POOL}/${image}
+admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${image}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 
-admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
+admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 
-admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
+admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
 
-admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
+admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
 
-admin_daemon ${CLUSTER1} rbd mirror flush
-admin_daemon ${CLUSTER1} rbd mirror status
+flush ${CLUSTER1}
+admin_daemons ${CLUSTER1} rbd mirror status
 
 testlog "TEST: test image rename"
 new_name="${image}_RENAMED"
 rename_image ${CLUSTER2} ${POOL} ${image} ${new_name}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
-admin_daemon ${CLUSTER1} rbd mirror status ${POOL}/${new_name}
-admin_daemon ${CLUSTER1} rbd mirror restart ${POOL}/${new_name}
+admin_daemons ${CLUSTER1} rbd mirror status ${POOL}/${new_name}
+admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${new_name}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
 rename_image ${CLUSTER2} ${POOL} ${new_name} ${image}
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 
 testlog "TEST: failover and failback"
-start_mirror ${CLUSTER2}
+start_mirrors ${CLUSTER2}
 
 # demote and promote same cluster
 demote_image ${CLUSTER2} ${POOL} ${image}
@@ -303,14 +303,14 @@ done
 wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}"
 
 testlog "TEST: disable mirror while daemon is stopped"
-stop_mirror ${CLUSTER1}
-stop_mirror ${CLUSTER2}
+stop_mirrors ${CLUSTER1}
+stop_mirrors ${CLUSTER2}
 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 disable_mirror ${CLUSTER2} ${POOL} ${image}
 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
   test_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 fi
-start_mirror ${CLUSTER1}
+start_mirrors ${CLUSTER1}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
@@ -325,21 +325,21 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_
 compare_images ${POOL} ${image}
 
 testlog "TEST: image resync while replayer is stopped"
-admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image}
+admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
-admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
+admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
-admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
+admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 compare_images ${POOL} ${image}
 
 testlog "TEST: request image resync while daemon is offline"
-stop_mirror ${CLUSTER1}
+stop_mirrors ${CLUSTER1}
 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
-start_mirror ${CLUSTER1}
+start_mirrors ${CLUSTER1}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
@@ -370,7 +370,7 @@ test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 compare_images ${POOL} ${image}
 
 testlog " - disconnected after max_concurrent_object_sets reached"
-admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image}
+admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 set_image_meta ${CLUSTER2} ${POOL} ${image} \
@@ -382,7 +382,7 @@ set_image_meta ${CLUSTER2} ${POOL} ${image} \
 	       conf_rbd_journal_max_concurrent_object_sets 0
 
 testlog " - replay is still stopped (disconnected) after restart"
-admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
+admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 
@@ -428,8 +428,11 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain
 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 
-testlog "TEST: no blacklists"
-CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
-CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
+if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+  # teuthology will trash the daemon
+  testlog "TEST: no blacklists"
+  CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
+  CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
+fi
 
 echo OK

qa/workunits/rbd/rbd_mirror_ha.sh

@@ -3,6 +3,8 @@
 # rbd_mirror_ha.sh - test rbd-mirror daemons in HA mode
 #
 
+RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-7}
+
 . $(dirname $0)/rbd_mirror_helpers.sh
 
 is_leader()
@@ -23,7 +25,7 @@ wait_for_leader()
 
     for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64; do
 	sleep $s
-	for instance in `seq 0 9`; do
+	for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
 	    is_leader ${instance} || continue
 	    LEADER=${instance}
 	    return 0