/
test.sh
executable file
·1883 lines (1639 loc) · 60 KB
/
test.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash -x
source $(dirname $0)/../ceph-helpers.sh
set -e
set -o functrace
PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: '
SUDO=${SUDO:-sudo}
function check_no_osd_down()
{
! ceph osd dump | grep ' down '
}
function wait_no_osd_down()
{
for i in $(seq 1 300) ; do
if ! check_no_osd_down ; then
echo "waiting for osd(s) to come back up"
sleep 1
else
break
fi
done
check_no_osd_down
}
function expect_false()
{
set -x
if "$@"; then return 1; else return 0; fi
}
TMPDIR=/tmp/cephtool$$
mkdir $TMPDIR
trap "rm -fr $TMPDIR" 0
TMPFILE=$TMPDIR/test_invalid.$$
#
# retry_eagain max cmd args ...
#
# retry cmd args ... if it exits on error and its output contains the
# string EAGAIN, at most $max times
#
function retry_eagain()
{
local max=$1
shift
local status
local tmpfile=$TMPDIR/retry_eagain.$$
local count
for count in $(seq 1 $max) ; do
status=0
"$@" > $tmpfile 2>&1 || status=$?
if test $status = 0 ||
! grep --quiet EAGAIN $tmpfile ; then
break
fi
sleep 1
done
if test $count = $max ; then
echo retried with non zero exit status, $max times: "$@" >&2
fi
cat $tmpfile
rm $tmpfile
return $status
}
#
# map_enxio_to_eagain cmd arg ...
#
# add EAGAIN to the output of cmd arg ... if the output contains
# ENXIO.
#
function map_enxio_to_eagain()
{
local status=0
local tmpfile=$TMPDIR/map_enxio_to_eagain.$$
"$@" > $tmpfile 2>&1 || status=$?
if test $status != 0 &&
grep --quiet ENXIO $tmpfile ; then
echo "EAGAIN added by $0::map_enxio_to_eagain" >> $tmpfile
fi
cat $tmpfile
rm $tmpfile
return $status
}
function check_response()
{
expected_string=$1
retcode=$2
expected_retcode=$3
if [ "$expected_retcode" -a $retcode != $expected_retcode ] ; then
echo "return code invalid: got $retcode, expected $expected_retcode" >&2
exit 1
fi
if ! grep --quiet -- "$expected_string" $TMPFILE ; then
echo "Didn't find $expected_string in output" >&2
cat $TMPFILE >&2
exit 1
fi
}
function get_config_value_or_die()
{
local target config_opt raw val
target=$1
config_opt=$2
raw="`$SUDO ceph daemon $target config get $config_opt 2>/dev/null`"
if [[ $? -ne 0 ]]; then
echo "error obtaining config opt '$config_opt' from '$target': $raw"
exit 1
fi
raw=`echo $raw | sed -e 's/[{} "]//g'`
val=`echo $raw | cut -f2 -d:`
echo "$val"
return 0
}
function expect_config_value()
{
local target config_opt expected_val val
target=$1
config_opt=$2
expected_val=$3
val=$(get_config_value_or_die $target $config_opt)
if [[ "$val" != "$expected_val" ]]; then
echo "expected '$expected_val', got '$val'"
exit 1
fi
}
function ceph_watch_start()
{
local whatch_opt=--watch
if [ -n "$1" ]; then
whatch_opt=--watch-$1
fi
CEPH_WATCH_FILE=${TMPDIR}/CEPH_WATCH_$$
ceph $whatch_opt > $CEPH_WATCH_FILE &
CEPH_WATCH_PID=$!
}
function ceph_watch_wait()
{
local regexp=$1
local timeout=30
if [ -n "$2" ]; then
timeout=$2
fi
for i in `seq ${timeout}`; do
sleep 1
grep -q "$regexp" $CEPH_WATCH_FILE && break
done
kill $CEPH_WATCH_PID
if ! grep "$regexp" $CEPH_WATCH_FILE; then
echo "pattern ${regexp} not found in watch file. Full watch file content:" >&2
cat $CEPH_WATCH_FILE >&2
return 1
fi
}
function test_mon_injectargs()
{
CEPH_ARGS='--mon_debug_dump_location the.dump' ceph tell osd.0 injectargs --no-osd_debug_op_order >& $TMPFILE || return 1
check_response "osd_debug_op_order = 'false'"
! grep "the.dump" $TMPFILE || return 1
ceph tell osd.0 injectargs '--osd_debug_op_order --osd_failsafe_full_ratio .99' >& $TMPFILE || return 1
check_response "osd_debug_op_order = 'true' osd_failsafe_full_ratio = '0.99'"
ceph tell osd.0 injectargs --no-osd_debug_op_order >& $TMPFILE || return 1
check_response "osd_debug_op_order = 'false'"
ceph tell osd.0 injectargs -- --osd_debug_op_order >& $TMPFILE || return 1
check_response "osd_debug_op_order = 'true'"
ceph tell osd.0 injectargs -- '--osd_debug_op_order --osd_failsafe_full_ratio .98' >& $TMPFILE || return 1
check_response "osd_debug_op_order = 'true' osd_failsafe_full_ratio = '0.98'"
ceph tell osd.0 injectargs -- '--osd_failsafe_full_ratio' >& $TMPFILE || return 1
check_response "Option --osd_failsafe_full_ratio requires an argument"
}
function test_mon_injectargs_SI()
{
# Test SI units during injectargs and 'config set'
# We only aim at testing the units are parsed accordingly
# and don't intend to test whether the options being set
# actually expect SI units to be passed.
# Keep in mind that all integer based options (i.e., INT,
# LONG, U32, U64) will accept SI unit modifiers.
initial_value=$(get_config_value_or_die "mon.a" "mon_pg_warn_min_objects")
$SUDO ceph daemon mon.a config set mon_pg_warn_min_objects 10
expect_config_value "mon.a" "mon_pg_warn_min_objects" 10
$SUDO ceph daemon mon.a config set mon_pg_warn_min_objects 10K
expect_config_value "mon.a" "mon_pg_warn_min_objects" 10240
$SUDO ceph daemon mon.a config set mon_pg_warn_min_objects 1G
expect_config_value "mon.a" "mon_pg_warn_min_objects" 1073741824
$SUDO ceph daemon mon.a config set mon_pg_warn_min_objects 10F > $TMPFILE || true
check_response "'10F': (22) Invalid argument"
# now test with injectargs
ceph tell mon.a injectargs '--mon_pg_warn_min_objects 10'
expect_config_value "mon.a" "mon_pg_warn_min_objects" 10
ceph tell mon.a injectargs '--mon_pg_warn_min_objects 10K'
expect_config_value "mon.a" "mon_pg_warn_min_objects" 10240
ceph tell mon.a injectargs '--mon_pg_warn_min_objects 1G'
expect_config_value "mon.a" "mon_pg_warn_min_objects" 1073741824
expect_false ceph tell mon.a injectargs '--mon_pg_warn_min_objects 10F'
$SUDO ceph daemon mon.a config set mon_pg_warn_min_objects $initial_value
}
function test_tiering_agent()
{
local slow=slow_eviction
local fast=fast_eviction
ceph osd pool create $slow 1 1
ceph osd pool create $fast 1 1
ceph osd tier add $slow $fast
ceph osd tier cache-mode $fast writeback
ceph osd tier set-overlay $slow $fast
ceph osd pool set $fast hit_set_type bloom
rados -p $slow put obj1 /etc/group
ceph osd pool set $fast target_max_objects 1
ceph osd pool set $fast hit_set_count 1
ceph osd pool set $fast hit_set_period 5
# wait for the object to be evicted from the cache
local evicted
evicted=false
for i in 1 2 4 8 16 32 64 128 256 ; do
if ! rados -p $fast ls | grep obj1 ; then
evicted=true
break
fi
sleep $i
done
$evicted # assert
# the object is proxy read and promoted to the cache
rados -p $slow get obj1 /tmp/obj1
# wait for the promoted object to be evicted again
evicted=false
for i in 1 2 4 8 16 32 64 128 256 ; do
if ! rados -p $fast ls | grep obj1 ; then
evicted=true
break
fi
sleep $i
done
$evicted # assert
ceph osd tier remove-overlay $slow
ceph osd tier remove $slow $fast
ceph osd pool delete $fast $fast --yes-i-really-really-mean-it
ceph osd pool delete $slow $slow --yes-i-really-really-mean-it
}
function test_tiering()
{
# tiering
ceph osd pool create slow 2
ceph osd pool create slow2 2
ceph osd pool create cache 2
ceph osd pool create cache2 2
ceph osd tier add slow cache
ceph osd tier add slow cache2
expect_false ceph osd tier add slow2 cache
# test some state transitions
ceph osd tier cache-mode cache writeback
ceph osd tier cache-mode cache forward
ceph osd tier cache-mode cache readonly
ceph osd tier cache-mode cache forward
ceph osd tier cache-mode cache none
ceph osd tier cache-mode cache writeback
expect_false ceph osd tier cache-mode cache none
expect_false ceph osd tier cache-mode cache readonly
# test with dirty objects in the tier pool
# tier pool currently set to 'writeback'
rados -p cache put /etc/passwd /etc/passwd
ceph tell osd.\* flush_pg_stats || true
# 1 dirty object in pool 'cache'
ceph osd tier cache-mode cache forward
expect_false ceph osd tier cache-mode cache none
expect_false ceph osd tier cache-mode cache readonly
ceph osd tier cache-mode cache writeback
# remove object from tier pool
rados -p cache rm /etc/passwd
rados -p cache cache-flush-evict-all
ceph tell osd.\* flush_pg_stats || true
# no dirty objects in pool 'cache'
ceph osd tier cache-mode cache forward
ceph osd tier cache-mode cache none
ceph osd tier cache-mode cache readonly
TRIES=0
while ! ceph osd pool set cache pg_num 3 --yes-i-really-mean-it 2>$TMPFILE
do
grep 'currently creating pgs' $TMPFILE
TRIES=$(( $TRIES + 1 ))
test $TRIES -ne 60
sleep 3
done
expect_false ceph osd pool set cache pg_num 4
ceph osd tier cache-mode cache none
ceph osd tier set-overlay slow cache
expect_false ceph osd tier set-overlay slow cache2
expect_false ceph osd tier remove slow cache
ceph osd tier remove-overlay slow
ceph osd tier set-overlay slow cache2
ceph osd tier remove-overlay slow
ceph osd tier remove slow cache
ceph osd tier add slow2 cache
expect_false ceph osd tier set-overlay slow cache
ceph osd tier set-overlay slow2 cache
ceph osd tier remove-overlay slow2
ceph osd tier remove slow2 cache
ceph osd tier remove slow cache2
# make sure a non-empty pool fails
rados -p cache2 put /etc/passwd /etc/passwd
while ! ceph df | grep cache2 | grep ' 1 ' ; do
echo waiting for pg stats to flush
sleep 2
done
expect_false ceph osd tier add slow cache2
ceph osd tier add slow cache2 --force-nonempty
ceph osd tier remove slow cache2
ceph osd pool ls | grep cache2
ceph osd pool ls -f json-pretty | grep cache2
ceph osd pool ls detail | grep cache2
ceph osd pool ls detail -f json-pretty | grep cache2
ceph osd pool delete cache cache --yes-i-really-really-mean-it
ceph osd pool delete cache2 cache2 --yes-i-really-really-mean-it
# make sure we can't clobber snapshot state
ceph osd pool create snap_base 2
ceph osd pool create snap_cache 2
ceph osd pool mksnap snap_cache snapname
expect_false ceph osd tier add snap_base snap_cache
ceph osd pool delete snap_base snap_base --yes-i-really-really-mean-it
ceph osd pool delete snap_cache snap_cache --yes-i-really-really-mean-it
# make sure we can't create an ec pool tier
ceph osd pool create eccache 2 2 erasure
ceph osd pool create repbase 2
expect_false ceph osd tier add repbase eccache
ceph osd pool delete repbase repbase --yes-i-really-really-mean-it
ceph osd pool delete eccache eccache --yes-i-really-really-mean-it
# convenient add-cache command
ceph osd pool create cache3 2
ceph osd tier add-cache slow cache3 1024000
ceph osd dump | grep cache3 | grep bloom | grep 'false_positive_probability: 0.05' | grep 'target_bytes 1024000' | grep '1200s x4'
ceph osd tier remove slow cache3 2> $TMPFILE || true
check_response "EBUSY: tier pool 'cache3' is the overlay for 'slow'; please remove-overlay first"
ceph osd tier remove-overlay slow
ceph osd tier remove slow cache3
ceph osd pool ls | grep cache3
ceph osd pool delete cache3 cache3 --yes-i-really-really-mean-it
! ceph osd pool ls | grep cache3 || exit 1
ceph osd pool delete slow2 slow2 --yes-i-really-really-mean-it
ceph osd pool delete slow slow --yes-i-really-really-mean-it
# check add-cache whether work
ceph osd pool create datapool 2
ceph osd pool create cachepool 2
ceph osd tier add-cache datapool cachepool 1024000
ceph osd tier cache-mode cachepool writeback
rados -p datapool put object /etc/passwd
rados -p cachepool stat object
rados -p cachepool cache-flush object
rados -p datapool stat object
ceph osd tier remove-overlay datapool
ceph osd tier remove datapool cachepool
ceph osd pool delete cachepool cachepool --yes-i-really-really-mean-it
ceph osd pool delete datapool datapool --yes-i-really-really-mean-it
# protection against pool removal when used as tiers
ceph osd pool create datapool 2
ceph osd pool create cachepool 2
ceph osd tier add-cache datapool cachepool 1024000
ceph osd pool delete cachepool cachepool --yes-i-really-really-mean-it 2> $TMPFILE || true
check_response "EBUSY: pool 'cachepool' is a tier of 'datapool'"
ceph osd pool delete datapool datapool --yes-i-really-really-mean-it 2> $TMPFILE || true
check_response "EBUSY: pool 'datapool' has tiers cachepool"
ceph osd tier remove-overlay datapool
ceph osd tier remove datapool cachepool
ceph osd pool delete cachepool cachepool --yes-i-really-really-mean-it
ceph osd pool delete datapool datapool --yes-i-really-really-mean-it
## check health check
ceph osd set notieragent
ceph osd pool create datapool 2
ceph osd pool create cache4 2
ceph osd tier add-cache datapool cache4 1024000
ceph osd tier cache-mode cache4 writeback
tmpfile=$(mktemp|grep tmp)
dd if=/dev/zero of=$tmpfile bs=4K count=1
ceph osd pool set cache4 target_max_objects 200
ceph osd pool set cache4 target_max_bytes 1000000
rados -p cache4 put foo1 $tmpfile
rados -p cache4 put foo2 $tmpfile
rm -f $tmpfile
ceph tell osd.\* flush_pg_stats || true
ceph df | grep datapool | grep ' 2 '
ceph osd tier remove-overlay datapool
ceph osd tier remove datapool cache4
ceph osd pool delete cache4 cache4 --yes-i-really-really-mean-it
ceph osd pool delete datapool datapool --yes-i-really-really-mean-it
ceph osd unset notieragent
# make sure 'tier remove' behaves as we expect
# i.e., removing a tier from a pool that's not its base pool only
# results in a 'pool foo is now (or already was) not a tier of bar'
#
ceph osd pool create basepoolA 2
ceph osd pool create basepoolB 2
poolA_id=$(ceph osd dump | grep 'pool.*basepoolA' | awk '{print $2;}')
poolB_id=$(ceph osd dump | grep 'pool.*basepoolB' | awk '{print $2;}')
ceph osd pool create cache5 2
ceph osd pool create cache6 2
ceph osd tier add basepoolA cache5
ceph osd tier add basepoolB cache6
ceph osd tier remove basepoolB cache5 2>&1 | grep 'not a tier of'
ceph osd dump | grep "pool.*'cache5'" 2>&1 | grep "tier_of[ \t]\+$poolA_id"
ceph osd tier remove basepoolA cache6 2>&1 | grep 'not a tier of'
ceph osd dump | grep "pool.*'cache6'" 2>&1 | grep "tier_of[ \t]\+$poolB_id"
ceph osd tier remove basepoolA cache5 2>&1 | grep 'not a tier of'
! ceph osd dump | grep "pool.*'cache5'" 2>&1 | grep "tier_of" || exit 1
ceph osd tier remove basepoolB cache6 2>&1 | grep 'not a tier of'
! ceph osd dump | grep "pool.*'cache6'" 2>&1 | grep "tier_of" || exit 1
! ceph osd dump | grep "pool.*'basepoolA'" 2>&1 | grep "tiers" || exit 1
! ceph osd dump | grep "pool.*'basepoolB'" 2>&1 | grep "tiers" || exit 1
ceph osd pool delete cache6 cache6 --yes-i-really-really-mean-it
ceph osd pool delete cache5 cache5 --yes-i-really-really-mean-it
ceph osd pool delete basepoolB basepoolB --yes-i-really-really-mean-it
ceph osd pool delete basepoolA basepoolA --yes-i-really-really-mean-it
}
function test_auth()
{
ceph auth add client.xx mon allow osd "allow *"
ceph auth export client.xx >client.xx.keyring
ceph auth add client.xx -i client.xx.keyring
rm -f client.xx.keyring
ceph auth list | grep client.xx
ceph auth get client.xx | grep caps | grep mon
ceph auth get client.xx | grep caps | grep osd
ceph auth get-key client.xx
ceph auth print-key client.xx
ceph auth print_key client.xx
ceph auth caps client.xx osd "allow rw"
expect_false "ceph auth get client.xx | grep caps | grep mon"
ceph auth get client.xx | grep osd | grep "allow rw"
ceph auth export | grep client.xx
ceph auth export -o authfile
ceph auth import -i authfile
ceph auth export -o authfile2
diff authfile authfile2
rm authfile authfile2
ceph auth del client.xx
expect_false ceph auth get client.xx
# (almost) interactive mode
echo -e 'auth add client.xx mon allow osd "allow *"\n' | ceph
ceph auth get client.xx
# script mode
echo 'auth del client.xx' | ceph
expect_false ceph auth get client.xx
#
# get / set auid
#
local auid=444
ceph-authtool --create-keyring --name client.TEST --gen-key --set-uid $auid TEST-keyring
ceph auth import --in-file TEST-keyring
rm TEST-keyring
ceph auth get client.TEST > $TMPFILE
check_response "auid = $auid"
ceph --format json-pretty auth get client.TEST > $TMPFILE
check_response '"auid": '$auid
ceph auth list > $TMPFILE
check_response "auid: $auid"
ceph --format json-pretty auth list > $TMPFILE
check_response '"auid": '$auid
ceph auth del client.TEST
}
function test_auth_profiles()
{
ceph auth add client.xx-profile-ro mon 'allow profile read-only'
ceph auth add client.xx-profile-rw mon 'allow profile read-write'
ceph auth add client.xx-profile-rd mon 'allow profile role-definer'
ceph auth export > client.xx.keyring
# read-only is allowed all read-only commands (auth excluded)
ceph -n client.xx-profile-ro -k client.xx.keyring status
ceph -n client.xx-profile-ro -k client.xx.keyring osd dump
ceph -n client.xx-profile-ro -k client.xx.keyring pg dump
ceph -n client.xx-profile-ro -k client.xx.keyring mon dump
ceph -n client.xx-profile-ro -k client.xx.keyring mds dump
# read-only gets access denied for rw commands or auth commands
ceph -n client.xx-profile-ro -k client.xx.keyring log foo >& $TMPFILE || true
check_response "EACCES: access denied"
ceph -n client.xx-profile-ro -k client.xx.keyring osd set noout >& $TMPFILE || true
check_response "EACCES: access denied"
ceph -n client.xx-profile-ro -k client.xx.keyring auth list >& $TMPFILE || true
check_response "EACCES: access denied"
# read-write is allowed for all read-write commands (except auth)
ceph -n client.xx-profile-rw -k client.xx.keyring status
ceph -n client.xx-profile-rw -k client.xx.keyring osd dump
ceph -n client.xx-profile-rw -k client.xx.keyring pg dump
ceph -n client.xx-profile-rw -k client.xx.keyring mon dump
ceph -n client.xx-profile-rw -k client.xx.keyring mds dump
ceph -n client.xx-profile-rw -k client.xx.keyring log foo
ceph -n client.xx-profile-rw -k client.xx.keyring osd set noout
ceph -n client.xx-profile-rw -k client.xx.keyring osd unset noout
# read-write gets access denied for auth commands
ceph -n client.xx-profile-rw -k client.xx.keyring auth list >& $TMPFILE || true
check_response "EACCES: access denied"
# role-definer is allowed RWX 'auth' commands and read-only 'mon' commands
ceph -n client.xx-profile-rd -k client.xx.keyring auth list
ceph -n client.xx-profile-rd -k client.xx.keyring auth export
ceph -n client.xx-profile-rd -k client.xx.keyring auth add client.xx-profile-foo
ceph -n client.xx-profile-rd -k client.xx.keyring status
ceph -n client.xx-profile-rd -k client.xx.keyring osd dump >& $TMPFILE || true
check_response "EACCES: access denied"
ceph -n client.xx-profile-rd -k client.xx.keyring pg dump >& $TMPFILE || true
check_response "EACCES: access denied"
# read-only 'mon' subsystem commands are allowed
ceph -n client.xx-profile-rd -k client.xx.keyring mon dump
# but read-write 'mon' commands are not
ceph -n client.xx-profile-rd -k client.xx.keyring mon add foo 1.1.1.1 >& $TMPFILE || true
check_response "EACCES: access denied"
ceph -n client.xx-profile-rd -k client.xx.keyring mds dump >& $TMPFILE || true
check_response "EACCES: access denied"
ceph -n client.xx-profile-rd -k client.xx.keyring log foo >& $TMPFILE || true
check_response "EACCES: access denied"
ceph -n client.xx-profile-rd -k client.xx.keyring osd set noout >& $TMPFILE || true
check_response "EACCES: access denied"
ceph -n client.xx-profile-rd -k client.xx.keyring auth del client.xx-profile-ro
ceph -n client.xx-profile-rd -k client.xx.keyring auth del client.xx-profile-rw
# add a new role-definer with the existing role-definer
ceph -n client.xx-profile-rd -k client.xx.keyring \
auth add client.xx-profile-rd2 mon 'allow profile role-definer'
ceph -n client.xx-profile-rd -k client.xx.keyring \
auth export > client.xx.keyring.2
# remove old role-definer using the new role-definer
ceph -n client.xx-profile-rd2 -k client.xx.keyring.2 \
auth del client.xx-profile-rd
# remove the remaining role-definer with admin
ceph auth del client.xx-profile-rd2
rm -f client.xx.keyring client.xx.keyring.2
}
function test_mon_caps()
{
ceph-authtool --create-keyring $TMPDIR/ceph.client.bug.keyring
chmod +r $TMPDIR/ceph.client.bug.keyring
ceph-authtool $TMPDIR/ceph.client.bug.keyring -n client.bug --gen-key
ceph auth add client.bug -i $TMPDIR/ceph.client.bug.keyring
rados lspools --keyring $TMPDIR/ceph.client.bug.keyring -n client.bug >& $TMPFILE || true
check_response "Permission denied"
rm -rf $TMPDIR/ceph.client.bug.keyring
ceph auth del client.bug
ceph-authtool --create-keyring $TMPDIR/ceph.client.bug.keyring
chmod +r $TMPDIR/ceph.client.bug.keyring
ceph-authtool $TMPDIR/ceph.client.bug.keyring -n client.bug --gen-key
ceph-authtool -n client.bug --cap mon '' $TMPDIR/ceph.client.bug.keyring
ceph auth add client.bug -i $TMPDIR/ceph.client.bug.keyring
rados lspools --keyring $TMPDIR/ceph.client.bug.keyring -n client.bug >& $TMPFILE || true
check_response "Permission denied"
}
function test_mon_misc()
{
# with and without verbosity
ceph osd dump | grep '^epoch'
ceph --concise osd dump | grep '^epoch'
# df
ceph df > $TMPFILE
grep GLOBAL $TMPFILE
grep -v DIRTY $TMPFILE
ceph df detail > $TMPFILE
grep CATEGORY $TMPFILE
grep DIRTY $TMPFILE
ceph df --format json > $TMPFILE
grep 'total_bytes' $TMPFILE
grep -v 'dirty' $TMPFILE
ceph df detail --format json > $TMPFILE
grep 'rd_bytes' $TMPFILE
grep 'dirty' $TMPFILE
ceph df --format xml | grep '<total_bytes>'
ceph df detail --format xml | grep '<rd_bytes>'
ceph fsid
ceph health
ceph health detail
ceph health --format json-pretty
ceph health detail --format xml-pretty
ceph node ls
for t in mon osd mds ; do
ceph node ls $t
done
ceph_watch_start
mymsg="this is a test log message $$.$(date)"
ceph log "$mymsg"
ceph_watch_wait "$mymsg"
ceph mon metadata a
ceph node ls
}
function check_mds_active()
{
ceph mds dump | grep active
}
function wait_mds_active()
{
for i in $(seq 1 300) ; do
if ! check_mds_active ; then
echo "waiting for an active MDS daemon"
sleep 5
else
break
fi
done
check_mds_active
}
function get_mds_gids()
{
ceph mds dump --format=json | python -c "import json; import sys; print ' '.join([m['gid'].__str__() for m in json.load(sys.stdin)['info'].values()])"
}
function fail_all_mds()
{
ceph mds cluster_down
mds_gids=$(get_mds_gids)
for mds_gid in $mds_gids ; do
ceph mds fail $mds_gid
done
if check_mds_active ; then
echo "An active MDS remains, something went wrong"
ceph mds dump
exit -1
fi
}
function remove_all_fs()
{
existing_fs=$(ceph fs ls --format=json | python -c "import json; import sys; print ' '.join([fs['name'] for fs in json.load(sys.stdin)])")
if [ -n "$existing_fs" ] ; then
fail_all_mds
echo "Removing existing filesystem '${existing_fs}'..."
ceph fs rm $existing_fs --yes-i-really-mean-it
echo "Removed '${existing_fs}'."
fi
}
# So that tests requiring MDS can skip if one is not configured
# in the cluster at all
function mds_exists()
{
ceph auth list | grep "^mds"
}
function test_mds_tell()
{
if ! mds_exists ; then
echo "Skipping test, no MDS found"
return
fi
remove_all_fs
ceph osd pool create fs_data 10
ceph osd pool create fs_metadata 10
ceph fs new cephfs fs_metadata fs_data
wait_mds_active
# Test injectargs by GID
old_mds_gids=$(get_mds_gids)
echo Old GIDs: $old_mds_gids
for mds_gid in $old_mds_gids ; do
ceph tell mds.$mds_gid injectargs "--debug-mds 20"
done
# Test respawn by rank
ceph tell mds.0 respawn
new_mds_gids=$old_mds_gids
while [ $new_mds_gids -eq $old_mds_gids ] ; do
sleep 5
new_mds_gids=$(get_mds_gids)
done
echo New GIDs: $new_mds_gids
# Test respawn by ID
ceph tell mds.a respawn
new_mds_gids=$old_mds_gids
while [ $new_mds_gids -eq $old_mds_gids ] ; do
sleep 5
new_mds_gids=$(get_mds_gids)
done
echo New GIDs: $new_mds_gids
remove_all_fs
ceph osd pool delete fs_data fs_data --yes-i-really-really-mean-it
ceph osd pool delete fs_metadata fs_metadata --yes-i-really-really-mean-it
}
function test_mon_mds()
{
remove_all_fs
ceph osd pool create fs_data 10
ceph osd pool create fs_metadata 10
ceph fs new cephfs fs_metadata fs_data
ceph mds cluster_down
ceph mds cluster_up
ceph mds compat rm_incompat 4
ceph mds compat rm_incompat 4
# We don't want any MDSs to be up, their activity can interfere with
# the "current_epoch + 1" checking below if they're generating updates
fail_all_mds
# Check for default crash_replay_interval set automatically in 'fs new'
#This may vary based on ceph.conf (e.g., it's 5 in teuthology runs)
#ceph osd dump | grep fs_data > $TMPFILE
#check_response "crash_replay_interval 45 "
ceph mds compat show
expect_false ceph mds deactivate 2
ceph mds dump
for mds_gid in $(get_mds_gids) ; do
ceph mds metadata $mds_id
done
# XXX mds fail, but how do you undo it?
mdsmapfile=$TMPDIR/mdsmap.$$
current_epoch=$(ceph mds getmap -o $mdsmapfile --no-log-to-stderr 2>&1 | grep epoch | sed 's/.*epoch //')
[ -s $mdsmapfile ]
# make several attempts in case we race with another mdsmap update
((epoch = current_epoch + 1))
((epoch2 = current_epoch + 2))
((epoch3 = current_epoch + 3))
((epoch4 = current_epoch + 4))
ceph mds setmap -i $mdsmapfile $epoch || \
ceph mds setmap -i $mdsmapfile $epoch2 || \
ceph mds setmap -i $mdsmapfile $epoch3 || \
ceph mds setmap -i $mdsmapfile $epoch4
rm $mdsmapfile
ceph osd pool create data2 10
ceph osd pool create data3 10
data2_pool=$(ceph osd dump | grep 'pool.*data2' | awk '{print $2;}')
data3_pool=$(ceph osd dump | grep 'pool.*data3' | awk '{print $2;}')
ceph mds add_data_pool $data2_pool
ceph mds add_data_pool $data3_pool
ceph mds add_data_pool 100 >& $TMPFILE || true
check_response "Error ENOENT"
ceph mds add_data_pool foobarbaz >& $TMPFILE || true
check_response "Error ENOENT"
ceph mds remove_data_pool $data2_pool
ceph mds remove_data_pool $data3_pool
ceph osd pool delete data2 data2 --yes-i-really-really-mean-it
ceph osd pool delete data3 data3 --yes-i-really-really-mean-it
ceph mds set_max_mds 4
ceph mds set_max_mds 3
ceph mds set_max_mds 256
expect_false ceph mds set_max_mds 257
ceph mds set max_mds 4
ceph mds set max_mds 256
expect_false ceph mds set max_mds 257
expect_false ceph mds set max_mds asdf
expect_false ceph mds set inline_data true
ceph mds set inline_data true --yes-i-really-mean-it
ceph mds set inline_data yes --yes-i-really-mean-it
ceph mds set inline_data 1 --yes-i-really-mean-it
expect_false ceph mds set inline_data --yes-i-really-mean-it
ceph mds set inline_data false
ceph mds set inline_data no
ceph mds set inline_data 0
expect_false ceph mds set inline_data asdf
ceph mds set max_file_size 1048576
expect_false ceph mds set max_file_size 123asdf
expect_false ceph mds set allow_new_snaps
expect_false ceph mds set allow_new_snaps true
ceph mds set allow_new_snaps true --yes-i-really-mean-it
ceph mds set allow_new_snaps 0
ceph mds set allow_new_snaps false
ceph mds set allow_new_snaps no
expect_false ceph mds set allow_new_snaps taco
# we should never be able to add EC pools as data or metadata pools
# create an ec-pool...
ceph osd pool create mds-ec-pool 10 10 erasure
set +e
ceph mds add_data_pool mds-ec-pool 2>$TMPFILE
check_response 'erasure-code' $? 22
set -e
ec_poolnum=$(ceph osd dump | grep "pool.* 'mds-ec-pool" | awk '{print $2;}')
data_poolnum=$(ceph osd dump | grep "pool.* 'fs_data" | awk '{print $2;}')
metadata_poolnum=$(ceph osd dump | grep "pool.* 'fs_metadata" | awk '{print $2;}')
fail_all_mds
# Check that 'fs reset' runs
ceph fs reset cephfs --yes-i-really-mean-it
fail_all_mds
# Clean up to enable subsequent newfs tests
ceph fs rm cephfs --yes-i-really-mean-it
set +e
ceph mds newfs $metadata_poolnum $ec_poolnum --yes-i-really-mean-it 2>$TMPFILE
check_response 'erasure-code' $? 22
ceph mds newfs $ec_poolnum $data_poolnum --yes-i-really-mean-it 2>$TMPFILE
check_response 'erasure-code' $? 22
ceph mds newfs $ec_poolnum $ec_poolnum --yes-i-really-mean-it 2>$TMPFILE
check_response 'erasure-code' $? 22
ceph fs new cephfs fs_metadata mds-ec-pool 2>$TMPFILE
check_response 'erasure-code' $? 22
ceph fs new cephfs mds-ec-pool fs_data 2>$TMPFILE
check_response 'erasure-code' $? 22
ceph fs new cephfs mds-ec-pool mds-ec-pool 2>$TMPFILE
check_response 'erasure-code' $? 22
set -e
# ... new create a cache tier in front of the EC pool...
ceph osd pool create mds-tier 2
ceph osd tier add mds-ec-pool mds-tier
ceph osd tier set-overlay mds-ec-pool mds-tier
tier_poolnum=$(ceph osd dump | grep "pool.* 'mds-tier" | awk '{print $2;}')
# Use of a readonly tier should be forbidden
ceph osd tier cache-mode mds-tier readonly
set +e
ceph fs new cephfs fs_metadata mds-ec-pool 2>$TMPFILE
check_response 'has a write tier (mds-tier) that is configured to forward' $? 22
set -e
# Use of a writeback tier should enable FS creation
ceph osd tier cache-mode mds-tier writeback
ceph fs new cephfs fs_metadata mds-ec-pool
# While a FS exists using the tiered pools, I should not be allowed
# to remove the tier
set +e
ceph osd tier remove-overlay mds-ec-pool 2>$TMPFILE
check_response 'in use by CephFS' $? 16
ceph osd tier remove mds-ec-pool mds-tier 2>$TMPFILE
check_response 'in use by CephFS' $? 16
set -e
fail_all_mds
ceph fs rm cephfs --yes-i-really-mean-it
# ... but we should be forbidden from using the cache pool in the FS directly.
set +e
ceph mds newfs $metadata_poolnum $tier_poolnum --yes-i-really-mean-it 2>$TMPFILE
check_response 'in use as a cache tier' $? 22
ceph mds newfs $tier_poolnum $data_poolnum --yes-i-really-mean-it 2>$TMPFILE
check_response 'in use as a cache tier' $? 22
ceph mds newfs $tier_poolnum $tier_poolnum --yes-i-really-mean-it 2>$TMPFILE
check_response 'in use as a cache tier' $? 22
ceph fs new cephfs fs_metadata mds-tier 2>$TMPFILE
check_response 'in use as a cache tier' $? 22
ceph fs new cephfs mds-tier fs_data 2>$TMPFILE
check_response 'in use as a cache tier' $? 22
ceph fs new cephfs mds-tier mds-tier 2>$TMPFILE
check_response 'in use as a cache tier' $? 22
set -e
# Clean up tier + EC pools
ceph osd tier remove-overlay mds-ec-pool
ceph osd tier remove mds-ec-pool mds-tier
# Create a FS using the 'cache' pool now that it's no longer a tier
ceph fs new cephfs fs_metadata mds-tier
# We should be forbidden from using this pool as a tier now that
# it's in use for CephFS
set +e
ceph osd tier add mds-ec-pool mds-tier 2>$TMPFILE
check_response 'in use by CephFS' $? 16
set -e
fail_all_mds
ceph fs rm cephfs --yes-i-really-mean-it
ceph osd pool delete mds-ec-pool mds-ec-pool --yes-i-really-really-mean-it
# Create a FS and check that we can subsequently add a cache tier to it
ceph fs new cephfs fs_metadata fs_data
# Adding overlay to FS pool should be permitted, RADOS clients handle this.
ceph osd tier add fs_metadata mds-tier
ceph osd tier cache-mode mds-tier writeback
ceph osd tier set-overlay fs_metadata mds-tier
# Removing tier should be permitted because the underlying pool is
# replicated (#11504 case)
ceph osd tier cache-mode mds-tier forward
ceph osd tier remove-overlay fs_metadata
ceph osd tier remove fs_metadata mds-tier
ceph osd pool delete mds-tier mds-tier --yes-i-really-really-mean-it
# Clean up FS
fail_all_mds
ceph fs rm cephfs --yes-i-really-mean-it
ceph mds stat
# ceph mds tell mds.a getmap
# ceph mds rm
# ceph mds rmfailed
# ceph mds set_state
# ceph mds stop
ceph osd pool delete fs_data fs_data --yes-i-really-really-mean-it
ceph osd pool delete fs_metadata fs_metadata --yes-i-really-really-mean-it
}
function test_mon_mds_metadata()
{
local nmons=$(ceph tell 'mon.*' version | grep -c 'version')
test "$nmons" -gt 0
ceph mds dump |
sed -nEe "s/^([0-9]+):.*'([a-z])' mds\\.([0-9]+)\\..*/\\1 \\2 \\3/p" |
while read gid id rank; do
ceph mds metadata ${gid} | grep '"hostname":'
ceph mds metadata ${id} | grep '"hostname":'
ceph mds metadata ${rank} | grep '"hostname":'
local n=$(ceph tell 'mon.*' mds metadata ${id} | grep -c '"hostname":')
test "$n" -eq "$nmons"
done
expect_false ceph mds metadata UNKNOWN
}
function test_mon_mon()
{
# print help message
ceph mon
# no mon add/remove
ceph mon dump
ceph mon getmap -o $TMPDIR/monmap.$$
[ -s $TMPDIR/monmap.$$ ]
# ceph mon tell
ceph mon_status
}
function test_mon_osd()
{
#
# osd blacklist
#
bl=192.168.0.1:0/1000
# Escaped form which may appear in JSON output
bl_json=192.168.0.1:0\\\\/1000
ceph osd blacklist add $bl
ceph osd blacklist ls | grep $bl
ceph osd blacklist ls --format=json-pretty | grep $bl_json
ceph osd dump --format=json-pretty | grep $bl
ceph osd dump | grep "^blacklist $bl"
ceph osd blacklist rm $bl
expect_false "ceph osd blacklist ls | grep $bl"