-
Notifications
You must be signed in to change notification settings - Fork 5.9k
/
test.sh
executable file
·2976 lines (2595 loc) · 103 KB
/
test.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env bash
# -*- mode:shell-script; tab-width:8; sh-basic-offset:2; indent-tabs-mode:t -*-
# vim: ts=8 sw=8 ft=bash smarttab
set -x
source $(dirname $0)/../../standalone/ceph-helpers.sh
set -e
set -o functrace
PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: '
SUDO=${SUDO:-sudo}
export CEPH_DEV=1
function check_no_osd_down()
{
! ceph osd dump | grep ' down '
}
function wait_no_osd_down()
{
max_run=300
for i in $(seq 1 $max_run) ; do
if ! check_no_osd_down ; then
echo "waiting for osd(s) to come back up ($i/$max_run)"
sleep 1
else
break
fi
done
check_no_osd_down
}
function expect_false()
{
set -x
if "$@"; then return 1; else return 0; fi
}
function expect_true()
{
set -x
if ! "$@"; then return 1; else return 0; fi
}
TEMP_DIR=$(mktemp -d ${TMPDIR-/tmp}/cephtool.XXX)
trap "rm -fr $TEMP_DIR" 0
TMPFILE=$(mktemp $TEMP_DIR/test_invalid.XXX)
#
# retry_eagain max cmd args ...
#
# retry cmd args ... if it exits on error and its output contains the
# string EAGAIN, at most $max times
#
function retry_eagain()
{
local max=$1
shift
local status
local tmpfile=$TEMP_DIR/retry_eagain.$$
local count
for count in $(seq 1 $max) ; do
status=0
"$@" > $tmpfile 2>&1 || status=$?
if test $status = 0 ||
! grep --quiet EAGAIN $tmpfile ; then
break
fi
sleep 1
done
if test $count = $max ; then
echo retried with non zero exit status, $max times: "$@" >&2
fi
cat $tmpfile
rm $tmpfile
return $status
}
#
# map_enxio_to_eagain cmd arg ...
#
# add EAGAIN to the output of cmd arg ... if the output contains
# ENXIO.
#
function map_enxio_to_eagain()
{
local status=0
local tmpfile=$TEMP_DIR/map_enxio_to_eagain.$$
"$@" > $tmpfile 2>&1 || status=$?
if test $status != 0 &&
grep --quiet ENXIO $tmpfile ; then
echo "EAGAIN added by $0::map_enxio_to_eagain" >> $tmpfile
fi
cat $tmpfile
rm $tmpfile
return $status
}
function check_response()
{
expected_string=$1
retcode=$2
expected_retcode=$3
if [ "$expected_retcode" -a $retcode != $expected_retcode ] ; then
echo "return code invalid: got $retcode, expected $expected_retcode" >&2
exit 1
fi
if ! grep --quiet -- "$expected_string" $TMPFILE ; then
echo "Didn't find $expected_string in output" >&2
cat $TMPFILE >&2
exit 1
fi
}
function get_config_value_or_die()
{
local target config_opt raw val
target=$1
config_opt=$2
raw="`$SUDO ceph daemon $target config get $config_opt 2>/dev/null`"
if [[ $? -ne 0 ]]; then
echo "error obtaining config opt '$config_opt' from '$target': $raw"
exit 1
fi
raw=`echo $raw | sed -e 's/[{} "]//g'`
val=`echo $raw | cut -f2 -d:`
echo "$val"
return 0
}
function expect_config_value()
{
local target config_opt expected_val val
target=$1
config_opt=$2
expected_val=$3
val=$(get_config_value_or_die $target $config_opt)
if [[ "$val" != "$expected_val" ]]; then
echo "expected '$expected_val', got '$val'"
exit 1
fi
}
function ceph_watch_start()
{
local whatch_opt=--watch
if [ -n "$1" ]; then
whatch_opt=--watch-$1
if [ -n "$2" ]; then
whatch_opt+=" --watch-channel $2"
fi
fi
CEPH_WATCH_FILE=${TEMP_DIR}/CEPH_WATCH_$$
ceph $whatch_opt > $CEPH_WATCH_FILE &
CEPH_WATCH_PID=$!
# wait until the "ceph" client is connected and receiving
# log messages from monitor
for i in `seq 3`; do
grep -q "cluster" $CEPH_WATCH_FILE && break
sleep 1
done
}
function ceph_watch_wait()
{
local regexp=$1
local timeout=30
if [ -n "$2" ]; then
timeout=$2
fi
for i in `seq ${timeout}`; do
grep -q "$regexp" $CEPH_WATCH_FILE && break
sleep 1
done
kill $CEPH_WATCH_PID
if ! grep "$regexp" $CEPH_WATCH_FILE; then
echo "pattern ${regexp} not found in watch file. Full watch file content:" >&2
cat $CEPH_WATCH_FILE >&2
return 1
fi
}
function test_mon_injectargs()
{
ceph tell osd.0 injectargs --no-osd_enable_op_tracker
ceph tell osd.0 config get osd_enable_op_tracker | grep false
ceph tell osd.0 injectargs '--osd_enable_op_tracker --osd_op_history_duration 500'
ceph tell osd.0 config get osd_enable_op_tracker | grep true
ceph tell osd.0 config get osd_op_history_duration | grep 500
ceph tell osd.0 injectargs --no-osd_enable_op_tracker
ceph tell osd.0 config get osd_enable_op_tracker | grep false
ceph tell osd.0 injectargs -- --osd_enable_op_tracker
ceph tell osd.0 config get osd_enable_op_tracker | grep true
ceph tell osd.0 injectargs -- '--osd_enable_op_tracker --osd_op_history_duration 600'
ceph tell osd.0 config get osd_enable_op_tracker | grep true
ceph tell osd.0 config get osd_op_history_duration | grep 600
ceph tell osd.0 injectargs -- '--osd_deep_scrub_interval 2419200'
ceph tell osd.0 config get osd_deep_scrub_interval | grep 2419200
ceph tell osd.0 injectargs -- '--mon_probe_timeout 2'
ceph tell osd.0 config get mon_probe_timeout | grep 2
ceph tell osd.0 injectargs -- '--mon-lease 6'
ceph tell osd.0 config get mon_lease | grep 6
# osd-scrub-auto-repair-num-errors is an OPT_U32, so -1 is not a valid setting
expect_false ceph tell osd.0 injectargs --osd-scrub-auto-repair-num-errors -1 2> $TMPFILE || return 1
check_response "Error EINVAL: Parse error setting osd_scrub_auto_repair_num_errors to '-1' using injectargs"
expect_failure $TEMP_DIR "Option --osd_op_history_duration requires an argument" \
ceph tell osd.0 injectargs -- '--osd_op_history_duration'
}
function test_mon_injectargs_SI()
{
# Test SI units during injectargs and 'config set'
# We only aim at testing the units are parsed accordingly
# and don't intend to test whether the options being set
# actually expect SI units to be passed.
# Keep in mind that all integer based options that are not based on bytes
# (i.e., INT, LONG, U32, U64) will accept SI unit modifiers and be parsed to
# base 10.
initial_value=$(get_config_value_or_die "mon.a" "mon_pg_warn_min_objects")
$SUDO ceph daemon mon.a config set mon_pg_warn_min_objects 10
expect_config_value "mon.a" "mon_pg_warn_min_objects" 10
$SUDO ceph daemon mon.a config set mon_pg_warn_min_objects 10K
expect_config_value "mon.a" "mon_pg_warn_min_objects" 10000
$SUDO ceph daemon mon.a config set mon_pg_warn_min_objects 1G
expect_config_value "mon.a" "mon_pg_warn_min_objects" 1000000000
$SUDO ceph daemon mon.a config set mon_pg_warn_min_objects 10F > $TMPFILE || true
check_response "(22) Invalid argument"
# now test with injectargs
ceph tell mon.a injectargs '--mon_pg_warn_min_objects 10'
expect_config_value "mon.a" "mon_pg_warn_min_objects" 10
ceph tell mon.a injectargs '--mon_pg_warn_min_objects 10K'
expect_config_value "mon.a" "mon_pg_warn_min_objects" 10000
ceph tell mon.a injectargs '--mon_pg_warn_min_objects 1G'
expect_config_value "mon.a" "mon_pg_warn_min_objects" 1000000000
expect_false ceph tell mon.a injectargs '--mon_pg_warn_min_objects 10F'
expect_false ceph tell mon.a injectargs '--mon_globalid_prealloc -1'
$SUDO ceph daemon mon.a config set mon_pg_warn_min_objects $initial_value
}
function test_mon_injectargs_IEC()
{
# Test IEC units during injectargs and 'config set'
# We only aim at testing the units are parsed accordingly
# and don't intend to test whether the options being set
# actually expect IEC units to be passed.
# Keep in mind that all integer based options that are based on bytes
# (i.e., INT, LONG, U32, U64) will accept IEC unit modifiers, as well as SI
# unit modifiers (for backwards compatibility and convenience) and be parsed
# to base 2.
initial_value=$(get_config_value_or_die "mon.a" "mon_data_size_warn")
$SUDO ceph daemon mon.a config set mon_data_size_warn 15000000000
expect_config_value "mon.a" "mon_data_size_warn" 15000000000
$SUDO ceph daemon mon.a config set mon_data_size_warn 15G
expect_config_value "mon.a" "mon_data_size_warn" 16106127360
$SUDO ceph daemon mon.a config set mon_data_size_warn 16Gi
expect_config_value "mon.a" "mon_data_size_warn" 17179869184
$SUDO ceph daemon mon.a config set mon_data_size_warn 10F > $TMPFILE || true
check_response "(22) Invalid argument"
# now test with injectargs
ceph tell mon.a injectargs '--mon_data_size_warn 15000000000'
expect_config_value "mon.a" "mon_data_size_warn" 15000000000
ceph tell mon.a injectargs '--mon_data_size_warn 15G'
expect_config_value "mon.a" "mon_data_size_warn" 16106127360
ceph tell mon.a injectargs '--mon_data_size_warn 16Gi'
expect_config_value "mon.a" "mon_data_size_warn" 17179869184
expect_false ceph tell mon.a injectargs '--mon_data_size_warn 10F'
$SUDO ceph daemon mon.a config set mon_data_size_warn $initial_value
}
function test_tiering_agent()
{
local slow=slow_eviction
local fast=fast_eviction
ceph osd pool create $slow 1 1
ceph osd pool application enable $slow rados
ceph osd pool create $fast 1 1
ceph osd tier add $slow $fast
ceph osd tier cache-mode $fast writeback
ceph osd tier set-overlay $slow $fast
ceph osd pool set $fast hit_set_type bloom
rados -p $slow put obj1 /etc/group
ceph osd pool set $fast target_max_objects 1
ceph osd pool set $fast hit_set_count 1
ceph osd pool set $fast hit_set_period 5
# wait for the object to be evicted from the cache
local evicted
evicted=false
for i in `seq 1 300` ; do
if ! rados -p $fast ls | grep obj1 ; then
evicted=true
break
fi
sleep 1
done
$evicted # assert
# the object is proxy read and promoted to the cache
rados -p $slow get obj1 - >/dev/null
# wait for the promoted object to be evicted again
evicted=false
for i in `seq 1 300` ; do
if ! rados -p $fast ls | grep obj1 ; then
evicted=true
break
fi
sleep 1
done
$evicted # assert
ceph osd tier remove-overlay $slow
ceph osd tier remove $slow $fast
ceph osd pool delete $fast $fast --yes-i-really-really-mean-it
ceph osd pool delete $slow $slow --yes-i-really-really-mean-it
}
function test_tiering_1()
{
# tiering
ceph osd pool create slow 2
ceph osd pool application enable slow rados
ceph osd pool create slow2 2
ceph osd pool application enable slow2 rados
ceph osd pool create cache 2
ceph osd pool create cache2 2
ceph osd tier add slow cache
ceph osd tier add slow cache2
expect_false ceph osd tier add slow2 cache
# application metadata should propagate to the tiers
ceph osd pool ls detail -f json | jq '.[] | select(.pool_name == "slow") | .application_metadata["rados"]' | grep '{}'
ceph osd pool ls detail -f json | jq '.[] | select(.pool_name == "slow2") | .application_metadata["rados"]' | grep '{}'
ceph osd pool ls detail -f json | jq '.[] | select(.pool_name == "cache") | .application_metadata["rados"]' | grep '{}'
ceph osd pool ls detail -f json | jq '.[] | select(.pool_name == "cache2") | .application_metadata["rados"]' | grep '{}'
# forward and proxy are removed/deprecated
expect_false ceph osd tier cache-mode cache forward
expect_false ceph osd tier cache-mode cache forward --yes-i-really-mean-it
expect_false ceph osd tier cache-mode cache proxy
expect_false ceph osd tier cache-mode cache proxy --yes-i-really-mean-it
# test some state transitions
ceph osd tier cache-mode cache writeback
expect_false ceph osd tier cache-mode cache readonly
expect_false ceph osd tier cache-mode cache readonly --yes-i-really-mean-it
ceph osd tier cache-mode cache readproxy
ceph osd tier cache-mode cache none
ceph osd tier cache-mode cache readonly --yes-i-really-mean-it
ceph osd tier cache-mode cache none
ceph osd tier cache-mode cache writeback
expect_false ceph osd tier cache-mode cache none
expect_false ceph osd tier cache-mode cache readonly --yes-i-really-mean-it
# test with dirty objects in the tier pool
# tier pool currently set to 'writeback'
rados -p cache put /etc/passwd /etc/passwd
flush_pg_stats
# 1 dirty object in pool 'cache'
ceph osd tier cache-mode cache readproxy
expect_false ceph osd tier cache-mode cache none
expect_false ceph osd tier cache-mode cache readonly --yes-i-really-mean-it
ceph osd tier cache-mode cache writeback
# remove object from tier pool
rados -p cache rm /etc/passwd
rados -p cache cache-flush-evict-all
flush_pg_stats
# no dirty objects in pool 'cache'
ceph osd tier cache-mode cache readproxy
ceph osd tier cache-mode cache none
ceph osd tier cache-mode cache readonly --yes-i-really-mean-it
TRIES=0
while ! ceph osd pool set cache pg_num 3 --yes-i-really-mean-it 2>$TMPFILE
do
grep 'currently creating pgs' $TMPFILE
TRIES=$(( $TRIES + 1 ))
test $TRIES -ne 60
sleep 3
done
expect_false ceph osd pool set cache pg_num 4
ceph osd tier cache-mode cache none
ceph osd tier set-overlay slow cache
expect_false ceph osd tier set-overlay slow cache2
expect_false ceph osd tier remove slow cache
ceph osd tier remove-overlay slow
ceph osd tier set-overlay slow cache2
ceph osd tier remove-overlay slow
ceph osd tier remove slow cache
ceph osd tier add slow2 cache
expect_false ceph osd tier set-overlay slow cache
ceph osd tier set-overlay slow2 cache
ceph osd tier remove-overlay slow2
ceph osd tier remove slow2 cache
ceph osd tier remove slow cache2
# make sure a non-empty pool fails
rados -p cache2 put /etc/passwd /etc/passwd
while ! ceph df | grep cache2 | grep ' 1 ' ; do
echo waiting for pg stats to flush
sleep 2
done
expect_false ceph osd tier add slow cache2
ceph osd tier add slow cache2 --force-nonempty
ceph osd tier remove slow cache2
ceph osd pool ls | grep cache2
ceph osd pool ls -f json-pretty | grep cache2
ceph osd pool ls detail | grep cache2
ceph osd pool ls detail -f json-pretty | grep cache2
ceph osd pool delete slow slow --yes-i-really-really-mean-it
ceph osd pool delete slow2 slow2 --yes-i-really-really-mean-it
ceph osd pool delete cache cache --yes-i-really-really-mean-it
ceph osd pool delete cache2 cache2 --yes-i-really-really-mean-it
}
function test_tiering_2()
{
# make sure we can't clobber snapshot state
ceph osd pool create snap_base 2
ceph osd pool application enable snap_base rados
ceph osd pool create snap_cache 2
ceph osd pool mksnap snap_cache snapname
expect_false ceph osd tier add snap_base snap_cache
ceph osd pool delete snap_base snap_base --yes-i-really-really-mean-it
ceph osd pool delete snap_cache snap_cache --yes-i-really-really-mean-it
}
function test_tiering_3()
{
# make sure we can't create snapshot on tier
ceph osd pool create basex 2
ceph osd pool application enable basex rados
ceph osd pool create cachex 2
ceph osd tier add basex cachex
expect_false ceph osd pool mksnap cache snapname
ceph osd tier remove basex cachex
ceph osd pool delete basex basex --yes-i-really-really-mean-it
ceph osd pool delete cachex cachex --yes-i-really-really-mean-it
}
function test_tiering_4()
{
# make sure we can't create an ec pool tier
ceph osd pool create eccache 2 2 erasure
expect_false ceph osd set-require-min-compat-client bobtail
ceph osd pool create repbase 2
ceph osd pool application enable repbase rados
expect_false ceph osd tier add repbase eccache
ceph osd pool delete repbase repbase --yes-i-really-really-mean-it
ceph osd pool delete eccache eccache --yes-i-really-really-mean-it
}
function test_tiering_5()
{
# convenient add-cache command
ceph osd pool create slow 2
ceph osd pool application enable slow rados
ceph osd pool create cache3 2
ceph osd tier add-cache slow cache3 1024000
ceph osd dump | grep cache3 | grep bloom | grep 'false_positive_probability: 0.05' | grep 'target_bytes 1024000' | grep '1200s x4'
ceph osd tier remove slow cache3 2> $TMPFILE || true
check_response "EBUSY: tier pool 'cache3' is the overlay for 'slow'; please remove-overlay first"
ceph osd tier remove-overlay slow
ceph osd tier remove slow cache3
ceph osd pool ls | grep cache3
ceph osd pool delete cache3 cache3 --yes-i-really-really-mean-it
! ceph osd pool ls | grep cache3 || exit 1
ceph osd pool delete slow slow --yes-i-really-really-mean-it
}
function test_tiering_6()
{
# check add-cache whether work
ceph osd pool create datapool 2
ceph osd pool application enable datapool rados
ceph osd pool create cachepool 2
ceph osd tier add-cache datapool cachepool 1024000
ceph osd tier cache-mode cachepool writeback
rados -p datapool put object /etc/passwd
rados -p cachepool stat object
rados -p cachepool cache-flush object
rados -p datapool stat object
ceph osd tier remove-overlay datapool
ceph osd tier remove datapool cachepool
ceph osd pool delete cachepool cachepool --yes-i-really-really-mean-it
ceph osd pool delete datapool datapool --yes-i-really-really-mean-it
}
function test_tiering_7()
{
# protection against pool removal when used as tiers
ceph osd pool create datapool 2
ceph osd pool application enable datapool rados
ceph osd pool create cachepool 2
ceph osd tier add-cache datapool cachepool 1024000
ceph osd pool delete cachepool cachepool --yes-i-really-really-mean-it 2> $TMPFILE || true
check_response "EBUSY: pool 'cachepool' is a tier of 'datapool'"
ceph osd pool delete datapool datapool --yes-i-really-really-mean-it 2> $TMPFILE || true
check_response "EBUSY: pool 'datapool' has tiers cachepool"
ceph osd tier remove-overlay datapool
ceph osd tier remove datapool cachepool
ceph osd pool delete cachepool cachepool --yes-i-really-really-mean-it
ceph osd pool delete datapool datapool --yes-i-really-really-mean-it
}
function test_tiering_8()
{
## check health check
ceph osd set notieragent
ceph osd pool create datapool 2
ceph osd pool application enable datapool rados
ceph osd pool create cache4 2
ceph osd tier add-cache datapool cache4 1024000
ceph osd tier cache-mode cache4 writeback
tmpfile=$(mktemp|grep tmp)
dd if=/dev/zero of=$tmpfile bs=4K count=1
ceph osd pool set cache4 target_max_objects 200
ceph osd pool set cache4 target_max_bytes 1000000
rados -p cache4 put foo1 $tmpfile
rados -p cache4 put foo2 $tmpfile
rm -f $tmpfile
flush_pg_stats
ceph df | grep datapool | grep ' 2 '
ceph osd tier remove-overlay datapool
ceph osd tier remove datapool cache4
ceph osd pool delete cache4 cache4 --yes-i-really-really-mean-it
ceph osd pool delete datapool datapool --yes-i-really-really-mean-it
ceph osd unset notieragent
}
function test_tiering_9()
{
# make sure 'tier remove' behaves as we expect
# i.e., removing a tier from a pool that's not its base pool only
# results in a 'pool foo is now (or already was) not a tier of bar'
#
ceph osd pool create basepoolA 2
ceph osd pool application enable basepoolA rados
ceph osd pool create basepoolB 2
ceph osd pool application enable basepoolB rados
poolA_id=$(ceph osd dump | grep 'pool.*basepoolA' | awk '{print $2;}')
poolB_id=$(ceph osd dump | grep 'pool.*basepoolB' | awk '{print $2;}')
ceph osd pool create cache5 2
ceph osd pool create cache6 2
ceph osd tier add basepoolA cache5
ceph osd tier add basepoolB cache6
ceph osd tier remove basepoolB cache5 2>&1 | grep 'not a tier of'
ceph osd dump | grep "pool.*'cache5'" 2>&1 | grep "tier_of[ \t]\+$poolA_id"
ceph osd tier remove basepoolA cache6 2>&1 | grep 'not a tier of'
ceph osd dump | grep "pool.*'cache6'" 2>&1 | grep "tier_of[ \t]\+$poolB_id"
ceph osd tier remove basepoolA cache5 2>&1 | grep 'not a tier of'
! ceph osd dump | grep "pool.*'cache5'" 2>&1 | grep "tier_of" || exit 1
ceph osd tier remove basepoolB cache6 2>&1 | grep 'not a tier of'
! ceph osd dump | grep "pool.*'cache6'" 2>&1 | grep "tier_of" || exit 1
! ceph osd dump | grep "pool.*'basepoolA'" 2>&1 | grep "tiers" || exit 1
! ceph osd dump | grep "pool.*'basepoolB'" 2>&1 | grep "tiers" || exit 1
ceph osd pool delete cache6 cache6 --yes-i-really-really-mean-it
ceph osd pool delete cache5 cache5 --yes-i-really-really-mean-it
ceph osd pool delete basepoolB basepoolB --yes-i-really-really-mean-it
ceph osd pool delete basepoolA basepoolA --yes-i-really-really-mean-it
}
function test_auth()
{
expect_false ceph auth add client.xx mon 'invalid' osd "allow *"
expect_false ceph auth add client.xx mon 'allow *' osd "allow *" invalid "allow *"
ceph auth add client.xx mon 'allow *' osd "allow *"
ceph auth export client.xx >client.xx.keyring
ceph auth add client.xx -i client.xx.keyring
rm -f client.xx.keyring
ceph auth list | grep client.xx
ceph auth ls | grep client.xx
ceph auth get client.xx | grep caps | grep mon
ceph auth get client.xx | grep caps | grep osd
ceph auth get-key client.xx
ceph auth print-key client.xx
ceph auth print_key client.xx
ceph auth caps client.xx osd "allow rw"
expect_false sh <<< "ceph auth get client.xx | grep caps | grep mon"
ceph auth get client.xx | grep osd | grep "allow rw"
ceph auth caps client.xx mon 'allow command "osd tree"'
ceph auth export | grep client.xx
ceph auth export -o authfile
ceph auth import -i authfile 2>$TMPFILE
check_response "imported keyring"
ceph auth export -o authfile2
diff authfile authfile2
rm authfile authfile2
ceph auth del client.xx
expect_false ceph auth get client.xx
# (almost) interactive mode
echo -e 'auth add client.xx mon "allow *" osd "allow *"\n' | ceph
ceph auth get client.xx
# script mode
echo 'auth del client.xx' | ceph
expect_false ceph auth get client.xx
}
function test_auth_profiles()
{
ceph auth add client.xx-profile-ro mon 'allow profile read-only' \
mgr 'allow profile read-only'
ceph auth add client.xx-profile-rw mon 'allow profile read-write' \
mgr 'allow profile read-write'
ceph auth add client.xx-profile-rd mon 'allow profile role-definer'
ceph auth export > client.xx.keyring
# read-only is allowed all read-only commands (auth excluded)
ceph -n client.xx-profile-ro -k client.xx.keyring status
ceph -n client.xx-profile-ro -k client.xx.keyring osd dump
ceph -n client.xx-profile-ro -k client.xx.keyring pg dump
ceph -n client.xx-profile-ro -k client.xx.keyring mon dump
# read-only gets access denied for rw commands or auth commands
ceph -n client.xx-profile-ro -k client.xx.keyring log foo >& $TMPFILE || true
check_response "EACCES: access denied"
ceph -n client.xx-profile-ro -k client.xx.keyring osd set noout >& $TMPFILE || true
check_response "EACCES: access denied"
ceph -n client.xx-profile-ro -k client.xx.keyring auth ls >& $TMPFILE || true
check_response "EACCES: access denied"
# read-write is allowed for all read-write commands (except auth)
ceph -n client.xx-profile-rw -k client.xx.keyring status
ceph -n client.xx-profile-rw -k client.xx.keyring osd dump
ceph -n client.xx-profile-rw -k client.xx.keyring pg dump
ceph -n client.xx-profile-rw -k client.xx.keyring mon dump
ceph -n client.xx-profile-rw -k client.xx.keyring fs dump
ceph -n client.xx-profile-rw -k client.xx.keyring log foo
ceph -n client.xx-profile-rw -k client.xx.keyring osd set noout
ceph -n client.xx-profile-rw -k client.xx.keyring osd unset noout
# read-write gets access denied for auth commands
ceph -n client.xx-profile-rw -k client.xx.keyring auth ls >& $TMPFILE || true
check_response "EACCES: access denied"
# role-definer is allowed RWX 'auth' commands and read-only 'mon' commands
ceph -n client.xx-profile-rd -k client.xx.keyring auth ls
ceph -n client.xx-profile-rd -k client.xx.keyring auth export
ceph -n client.xx-profile-rd -k client.xx.keyring auth add client.xx-profile-foo
ceph -n client.xx-profile-rd -k client.xx.keyring status
ceph -n client.xx-profile-rd -k client.xx.keyring osd dump >& $TMPFILE || true
check_response "EACCES: access denied"
ceph -n client.xx-profile-rd -k client.xx.keyring pg dump >& $TMPFILE || true
check_response "EACCES: access denied"
# read-only 'mon' subsystem commands are allowed
ceph -n client.xx-profile-rd -k client.xx.keyring mon dump
# but read-write 'mon' commands are not
ceph -n client.xx-profile-rd -k client.xx.keyring mon add foo 1.1.1.1 >& $TMPFILE || true
check_response "EACCES: access denied"
ceph -n client.xx-profile-rd -k client.xx.keyring fs dump >& $TMPFILE || true
check_response "EACCES: access denied"
ceph -n client.xx-profile-rd -k client.xx.keyring log foo >& $TMPFILE || true
check_response "EACCES: access denied"
ceph -n client.xx-profile-rd -k client.xx.keyring osd set noout >& $TMPFILE || true
check_response "EACCES: access denied"
ceph -n client.xx-profile-rd -k client.xx.keyring auth del client.xx-profile-ro
ceph -n client.xx-profile-rd -k client.xx.keyring auth del client.xx-profile-rw
# add a new role-definer with the existing role-definer
ceph -n client.xx-profile-rd -k client.xx.keyring \
auth add client.xx-profile-rd2 mon 'allow profile role-definer'
ceph -n client.xx-profile-rd -k client.xx.keyring \
auth export > client.xx.keyring.2
# remove old role-definer using the new role-definer
ceph -n client.xx-profile-rd2 -k client.xx.keyring.2 \
auth del client.xx-profile-rd
# remove the remaining role-definer with admin
ceph auth del client.xx-profile-rd2
rm -f client.xx.keyring client.xx.keyring.2
}
function test_mon_caps()
{
ceph-authtool --create-keyring $TEMP_DIR/ceph.client.bug.keyring
chmod +r $TEMP_DIR/ceph.client.bug.keyring
ceph-authtool $TEMP_DIR/ceph.client.bug.keyring -n client.bug --gen-key
ceph auth add client.bug -i $TEMP_DIR/ceph.client.bug.keyring
# pass --no-mon-config since we are looking for the permission denied error
rados lspools --no-mon-config --keyring $TEMP_DIR/ceph.client.bug.keyring -n client.bug >& $TMPFILE || true
cat $TMPFILE
check_response "Permission denied"
rm -rf $TEMP_DIR/ceph.client.bug.keyring
ceph auth del client.bug
ceph-authtool --create-keyring $TEMP_DIR/ceph.client.bug.keyring
chmod +r $TEMP_DIR/ceph.client.bug.keyring
ceph-authtool $TEMP_DIR/ceph.client.bug.keyring -n client.bug --gen-key
ceph-authtool -n client.bug --cap mon '' $TEMP_DIR/ceph.client.bug.keyring
ceph auth add client.bug -i $TEMP_DIR/ceph.client.bug.keyring
rados lspools --no-mon-config --keyring $TEMP_DIR/ceph.client.bug.keyring -n client.bug >& $TMPFILE || true
check_response "Permission denied"
}
function test_mon_misc()
{
# with and without verbosity
ceph osd dump | grep '^epoch'
ceph --concise osd dump | grep '^epoch'
ceph osd df | grep 'MIN/MAX VAR'
# df
ceph df > $TMPFILE
grep RAW $TMPFILE
grep -v DIRTY $TMPFILE
ceph df detail > $TMPFILE
grep DIRTY $TMPFILE
ceph df --format json > $TMPFILE
grep 'total_bytes' $TMPFILE
grep -v 'dirty' $TMPFILE
ceph df detail --format json > $TMPFILE
grep 'rd_bytes' $TMPFILE
grep 'dirty' $TMPFILE
ceph df --format xml | grep '<total_bytes>'
ceph df detail --format xml | grep '<rd_bytes>'
ceph fsid
ceph health
ceph health detail
ceph health --format json-pretty
ceph health detail --format xml-pretty
ceph time-sync-status
ceph node ls
for t in mon osd mds mgr ; do
ceph node ls $t
done
ceph_watch_start
mymsg="this is a test log message $$.$(date)"
ceph log "$mymsg"
ceph log last | grep "$mymsg"
ceph log last 100 | grep "$mymsg"
ceph_watch_wait "$mymsg"
ceph mgr stat
ceph mgr dump
ceph mgr dump | jq -e '.active_clients[0].name'
ceph mgr module ls
ceph mgr module enable restful
expect_false ceph mgr module enable foodne
ceph mgr module enable foodne --force
ceph mgr module disable foodne
ceph mgr module disable foodnebizbangbash
ceph mon metadata a
ceph mon metadata
ceph mon count-metadata ceph_version
ceph mon versions
ceph mgr metadata
ceph mgr versions
ceph mgr count-metadata ceph_version
ceph versions
ceph node ls
}
function check_mds_active()
{
fs_name=$1
ceph fs get $fs_name | grep active
}
function wait_mds_active()
{
fs_name=$1
max_run=300
for i in $(seq 1 $max_run) ; do
if ! check_mds_active $fs_name ; then
echo "waiting for an active MDS daemon ($i/$max_run)"
sleep 5
else
break
fi
done
check_mds_active $fs_name
}
function get_mds_gids()
{
fs_name=$1
ceph fs get $fs_name --format=json | python3 -c "import json; import sys; print(' '.join([m['gid'].__str__() for m in json.load(sys.stdin)['mdsmap']['info'].values()]))"
}
function fail_all_mds()
{
fs_name=$1
ceph fs set $fs_name cluster_down true
mds_gids=$(get_mds_gids $fs_name)
for mds_gid in $mds_gids ; do
ceph mds fail $mds_gid
done
if check_mds_active $fs_name ; then
echo "An active MDS remains, something went wrong"
ceph fs get $fs_name
exit -1
fi
}
function remove_all_fs()
{
existing_fs=$(ceph fs ls --format=json | python3 -c "import json; import sys; print(' '.join([fs['name'] for fs in json.load(sys.stdin)]))")
for fs_name in $existing_fs ; do
echo "Removing fs ${fs_name}..."
fail_all_mds $fs_name
echo "Removing existing filesystem '${fs_name}'..."
ceph fs rm $fs_name --yes-i-really-mean-it
echo "Removed '${fs_name}'."
done
}
# So that tests requiring MDS can skip if one is not configured
# in the cluster at all
function mds_exists()
{
ceph auth ls | grep "^mds"
}
# some of the commands are just not idempotent.
function without_test_dup_command()
{
if [ -z ${CEPH_CLI_TEST_DUP_COMMAND+x} ]; then
$@
else
local saved=${CEPH_CLI_TEST_DUP_COMMAND}
unset CEPH_CLI_TEST_DUP_COMMAND
$@
CEPH_CLI_TEST_DUP_COMMAND=saved
fi
}
function test_mds_tell()
{
local FS_NAME=cephfs
if ! mds_exists ; then
echo "Skipping test, no MDS found"
return
fi
remove_all_fs
ceph osd pool create fs_data 16
ceph osd pool create fs_metadata 16
ceph fs new $FS_NAME fs_metadata fs_data
wait_mds_active $FS_NAME
# Test injectargs by GID
old_mds_gids=$(get_mds_gids $FS_NAME)
echo Old GIDs: $old_mds_gids
for mds_gid in $old_mds_gids ; do
ceph tell mds.$mds_gid injectargs "--debug-mds 20"
done
expect_false ceph tell mds.a injectargs mds_max_file_recover -1
# Test respawn by rank
without_test_dup_command ceph tell mds.0 respawn
new_mds_gids=$old_mds_gids
while [ $new_mds_gids -eq $old_mds_gids ] ; do
sleep 5
new_mds_gids=$(get_mds_gids $FS_NAME)
done
echo New GIDs: $new_mds_gids
# Test respawn by ID
without_test_dup_command ceph tell mds.a respawn
new_mds_gids=$old_mds_gids
while [ $new_mds_gids -eq $old_mds_gids ] ; do
sleep 5
new_mds_gids=$(get_mds_gids $FS_NAME)
done
echo New GIDs: $new_mds_gids
remove_all_fs
ceph osd pool delete fs_data fs_data --yes-i-really-really-mean-it
ceph osd pool delete fs_metadata fs_metadata --yes-i-really-really-mean-it
}
function test_mon_mds()
{
local FS_NAME=cephfs
remove_all_fs
ceph osd pool create fs_data 16
ceph osd pool create fs_metadata 16
ceph fs new $FS_NAME fs_metadata fs_data
ceph fs set $FS_NAME cluster_down true
ceph fs set $FS_NAME cluster_down false
ceph mds compat rm_incompat 4
ceph mds compat rm_incompat 4
# We don't want any MDSs to be up, their activity can interfere with
# the "current_epoch + 1" checking below if they're generating updates
fail_all_mds $FS_NAME
ceph mds compat show
ceph fs dump
ceph fs get $FS_NAME
for mds_gid in $(get_mds_gids $FS_NAME) ; do
ceph mds metadata $mds_id
done
ceph mds metadata
ceph mds versions
ceph mds count-metadata os
# XXX mds fail, but how do you undo it?
mdsmapfile=$TEMP_DIR/mdsmap.$$
current_epoch=$(ceph fs dump -o $mdsmapfile --no-log-to-stderr 2>&1 | grep epoch | sed 's/.*epoch //')
[ -s $mdsmapfile ]
rm $mdsmapfile
ceph osd pool create data2 16
ceph osd pool create data3 16
data2_pool=$(ceph osd dump | grep "pool.*'data2'" | awk '{print $2;}')
data3_pool=$(ceph osd dump | grep "pool.*'data3'" | awk '{print $2;}')
ceph fs add_data_pool cephfs $data2_pool
ceph fs add_data_pool cephfs $data3_pool
ceph fs add_data_pool cephfs 100 >& $TMPFILE || true
check_response "Error ENOENT"
ceph fs add_data_pool cephfs foobarbaz >& $TMPFILE || true
check_response "Error ENOENT"
ceph fs rm_data_pool cephfs $data2_pool
ceph fs rm_data_pool cephfs $data3_pool
ceph osd pool delete data2 data2 --yes-i-really-really-mean-it
ceph osd pool delete data3 data3 --yes-i-really-really-mean-it
ceph fs set cephfs max_mds 4
ceph fs set cephfs max_mds 3
ceph fs set cephfs max_mds 256
expect_false ceph fs set cephfs max_mds 257
ceph fs set cephfs max_mds 4
ceph fs set cephfs max_mds 256
expect_false ceph fs set cephfs max_mds 257
expect_false ceph fs set cephfs max_mds asdf
expect_false ceph fs set cephfs inline_data true
ceph fs set cephfs inline_data true --yes-i-really-really-mean-it
ceph fs set cephfs inline_data yes --yes-i-really-really-mean-it
ceph fs set cephfs inline_data 1 --yes-i-really-really-mean-it
expect_false ceph fs set cephfs inline_data --yes-i-really-really-mean-it
ceph fs set cephfs inline_data false
ceph fs set cephfs inline_data no
ceph fs set cephfs inline_data 0
expect_false ceph fs set cephfs inline_data asdf
ceph fs set cephfs max_file_size 1048576
expect_false ceph fs set cephfs max_file_size 123asdf
expect_false ceph fs set cephfs allow_new_snaps
ceph fs set cephfs allow_new_snaps true
ceph fs set cephfs allow_new_snaps 0
ceph fs set cephfs allow_new_snaps false
ceph fs set cephfs allow_new_snaps no
expect_false ceph fs set cephfs allow_new_snaps taco
# we should never be able to add EC pools as data or metadata pools
# create an ec-pool...
ceph osd pool create mds-ec-pool 16 16 erasure
set +e
ceph fs add_data_pool cephfs mds-ec-pool 2>$TMPFILE
check_response 'erasure-code' $? 22
set -e
ec_poolnum=$(ceph osd dump | grep "pool.* 'mds-ec-pool" | awk '{print $2;}')
data_poolnum=$(ceph osd dump | grep "pool.* 'fs_data" | awk '{print $2;}')
metadata_poolnum=$(ceph osd dump | grep "pool.* 'fs_metadata" | awk '{print $2;}')
fail_all_mds $FS_NAME
set +e
# Check that rmfailed requires confirmation
expect_false ceph mds rmfailed 0
ceph mds rmfailed 0 --yes-i-really-mean-it
set -e
# Check that `fs new` is no longer permitted
expect_false ceph fs new cephfs $metadata_poolnum $data_poolnum --yes-i-really-mean-it 2>$TMPFILE