-
Notifications
You must be signed in to change notification settings - Fork 2k
/
playTests.sh
executable file
·1841 lines (1606 loc) · 66 KB
/
playTests.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/sh
set -e
unset ZSTD_CLEVEL
unset ZSTD_NBTHREADS
die() {
println "$@" 1>&2
exit 1
}
datagen() {
"$DATAGEN_BIN" "$@"
}
zstd() {
if [ -z "$EXEC_PREFIX" ]; then
"$ZSTD_BIN" "$@"
else
"$EXEC_PREFIX" "$ZSTD_BIN" "$@"
fi
}
sudoZstd() {
if [ -z "$EXEC_PREFIX" ]; then
sudo "$ZSTD_BIN" "$@"
else
sudo "$EXEC_PREFIX" "$ZSTD_BIN" "$@"
fi
}
roundTripTest() {
if [ -n "$3" ]; then
cLevel="$3"
proba="$2"
else
cLevel="$2"
proba=""
fi
if [ -n "$4" ]; then
dLevel="$4"
else
dLevel="$cLevel"
fi
rm -f tmp1 tmp2
println "roundTripTest: datagen $1 $proba | zstd -v$cLevel | zstd -d$dLevel"
datagen $1 $proba | $MD5SUM > tmp1
datagen $1 $proba | zstd --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2
$DIFF -q tmp1 tmp2
}
fileRoundTripTest() {
if [ -n "$3" ]; then
local_c="$3"
local_p="$2"
else
local_c="$2"
local_p=""
fi
if [ -n "$4" ]; then
local_d="$4"
else
local_d="$local_c"
fi
rm -f tmp.zst tmp.md5.1 tmp.md5.2
println "fileRoundTripTest: datagen $1 $local_p > tmp && zstd -v$local_c -c tmp | zstd -d$local_d"
datagen $1 $local_p > tmp
< tmp $MD5SUM > tmp.md5.1
zstd --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2
$DIFF -q tmp.md5.1 tmp.md5.2
}
truncateLastByte() {
dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1"
}
println() {
printf '%b\n' "${*}"
}
if [ -z "${size}" ]; then
size=
else
size=${size}
fi
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
PRGDIR="$SCRIPT_DIR/../programs"
TESTDIR="$SCRIPT_DIR/../tests"
UNAME=$(uname)
detectedTerminal=false
if [ -t 0 ] && [ -t 1 ]
then
detectedTerminal=true
fi
isTerminal=${isTerminal:-$detectedTerminal}
isWindows=false
INTOVOID="/dev/null"
case "$UNAME" in
GNU) DEVDEVICE="/dev/random" ;;
*) DEVDEVICE="/dev/zero" ;;
esac
case "$OS" in
Windows*)
isWindows=true
INTOVOID="NUL"
DEVDEVICE="NUL"
;;
esac
case "$UNAME" in
Darwin) MD5SUM="md5 -r" ;;
FreeBSD) MD5SUM="gmd5sum" ;;
NetBSD) MD5SUM="md5 -n" ;;
OpenBSD) MD5SUM="md5" ;;
*) MD5SUM="md5sum" ;;
esac
MTIME="stat -c %Y"
case "$UNAME" in
Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;;
esac
assertSameMTime() {
MT1=$($MTIME "$1")
MT2=$($MTIME "$2")
echo MTIME $MT1 $MT2
[ "$MT1" = "$MT2" ] || die "mtime on $1 doesn't match mtime on $2 ($MT1 != $MT2)"
}
GET_PERMS="stat -c %a"
case "$UNAME" in
Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;;
esac
assertFilePermissions() {
STAT1=$($GET_PERMS "$1")
STAT2=$2
[ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match expected ($STAT1 != $STAT2)"
}
assertSamePermissions() {
STAT1=$($GET_PERMS "$1")
STAT2=$($GET_PERMS "$2")
[ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)"
}
DIFF="diff"
case "$UNAME" in
SunOS) DIFF="gdiff" ;;
esac
# check if ZSTD_BIN is defined. if not, use the default value
if [ -z "${ZSTD_BIN}" ]; then
println "\nZSTD_BIN is not set. Using the default value..."
ZSTD_BIN="$PRGDIR/zstd"
fi
# check if DATAGEN_BIN is defined. if not, use the default value
if [ -z "${DATAGEN_BIN}" ]; then
println "\nDATAGEN_BIN is not set. Using the default value..."
DATAGEN_BIN="$TESTDIR/datagen"
fi
# Why was this line here ? Generates a strange ZSTD_BIN when EXE_PREFIX is non empty
# ZSTD_BIN="$EXE_PREFIX$ZSTD_BIN"
# assertions
[ -n "$ZSTD_BIN" ] || die "zstd not found at $ZSTD_BIN! \n Please define ZSTD_BIN pointing to the zstd binary. You might also consider rebuilding zstd following the instructions in README.md"
[ -n "$DATAGEN_BIN" ] || die "datagen not found at $DATAGEN_BIN! \n Please define DATAGEN_BIN pointing to the datagen binary. You might also consider rebuilding zstd tests following the instructions in README.md. "
println "\nStarting playTests.sh isWindows=$isWindows EXE_PREFIX='$EXE_PREFIX' ZSTD_BIN='$ZSTD_BIN' DATAGEN_BIN='$DATAGEN_BIN'"
if echo hello | zstd -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled'
then
hasMT=""
else
hasMT="true"
fi
zstd -vvV
println "\n===> simple tests "
datagen > tmp
zstd -h
zstd -H
zstd -V
println "test : basic compression "
zstd -f tmp # trivial compression case, creates tmp.zst
zstd -f -z tmp
zstd -f -k tmp
zstd -f -C tmp
println "test : basic decompression"
zstd -df tmp.zst # trivial decompression case (overwrites tmp)
println "test : too large compression level => auto-fix"
zstd -99 -f tmp # too large compression level, automatic sized down
zstd -5000000000 -f tmp && die "too large numeric value : must fail"
println "test : --fast aka negative compression levels"
zstd --fast -f tmp # == -1
zstd --fast=3 -f tmp # == -3
zstd --fast=200000 -f tmp # too low compression level, automatic fixed
zstd --fast=5000000000 -f tmp && die "too large numeric value : must fail"
zstd -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0"
println "test : too large numeric argument"
zstd --fast=9999999999 -f tmp && die "should have refused numeric value"
println "test : set compression level with environment variable ZSTD_CLEVEL"
ZSTD_CLEVEL=12 zstd -f tmp # positive compression level
ZSTD_CLEVEL=-12 zstd -f tmp # negative compression level
ZSTD_CLEVEL=+12 zstd -f tmp # valid: verbose '+' sign
ZSTD_CLEVEL='' zstd -f tmp # empty env var, warn and revert to default setting
ZSTD_CLEVEL=- zstd -f tmp # malformed env var, warn and revert to default setting
ZSTD_CLEVEL=a zstd -f tmp # malformed env var, warn and revert to default setting
ZSTD_CLEVEL=+a zstd -f tmp # malformed env var, warn and revert to default setting
ZSTD_CLEVEL=3a7 zstd -f tmp # malformed env var, warn and revert to default setting
ZSTD_CLEVEL=50000000000 zstd -f tmp # numeric value too large, warn and revert to default setting
println "test : override ZSTD_CLEVEL with command line option"
ZSTD_CLEVEL=12 zstd --fast=3 -f tmp # overridden by command line option
# temporary envvar changes in the above tests would actually persist in macos /bin/sh
unset ZSTD_CLEVEL
println "test : compress to stdout"
zstd tmp -c > tmpCompressed
zstd tmp --stdout > tmpCompressed # long command format
println "test : compress to named file"
rm -f tmpCompressed
zstd tmp -o tmpCompressed
test -f tmpCompressed # file must be created
println "test : force write, correct order"
zstd tmp -fo tmpCompressed
println "test : forgotten argument"
cp tmp tmp2
zstd tmp2 -fo && die "-o must be followed by filename "
println "test : implied stdout when input is stdin"
println bob | zstd | zstd -d
if [ "$isTerminal" = true ]; then
println "test : compressed data to terminal"
println bob | zstd && die "should have refused : compressed data to terminal"
println "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting on data from terminal)"
zstd -d > $INTOVOID && die "should have refused : compressed data from terminal"
fi
println "test : null-length file roundtrip"
println -n '' | zstd - --stdout | zstd -d --stdout
println "test : ensure small file doesn't add 3-bytes null block"
datagen -g1 > tmp1
zstd tmp1 -c | wc -c | grep "14"
zstd < tmp1 | wc -c | grep "14"
println "test : decompress file with wrong suffix (must fail)"
zstd -d tmpCompressed && die "wrong suffix error not detected!"
zstd -df tmp && die "should have refused : wrong extension"
println "test : decompress into stdout"
zstd -d tmpCompressed -c > tmpResult # decompression using stdout
zstd --decompress tmpCompressed -c > tmpResult
zstd --decompress tmpCompressed --stdout > tmpResult
println "test : decompress from stdin into stdout"
zstd -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout
zstd -dc - < tmp.zst > $INTOVOID
zstd -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used
zstd -d - < tmp.zst > $INTOVOID
println "test : impose memory limitation (must fail)"
datagen -g500K > tmplimit
zstd -f tmplimit
zstd -d -f tmplimit.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed"
zstd -d -f tmplimit.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command
zstd -d -f tmplimit.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command
zstd -d -f tmplimit.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command
rm -f tmplimit tmplimit.zst
println "test : overwrite protection"
zstd -q tmp && die "overwrite check failed!"
println "test : force overwrite"
zstd -q -f tmp
zstd -q --force tmp
println "test : overwrite readonly file"
rm -f tmpro tmpro.zst
println foo > tmpro.zst
println foo > tmpro
chmod 400 tmpro.zst
zstd -q tmpro && die "should have refused to overwrite read-only file"
zstd -q -f tmpro
println "test: --no-progress flag"
zstd tmpro -c --no-progress | zstd -d -f -o "$INTOVOID" --no-progress
zstd tmpro -cv --no-progress | zstd -dv -f -o "$INTOVOID" --no-progress
println "test: --progress flag"
zstd tmpro -c | zstd -d -f -o "$INTOVOID" --progress 2>&1 | grep -E "[A-Za-z0-9._ ]+: [0-9]+ bytes"
zstd tmpro -c | zstd -d -f -q -o "$INTOVOID" --progress 2>&1 | grep -E "[A-Za-z0-9._ ]+: [0-9]+ bytes"
zstd tmpro -c | zstd -d -f -v -o "$INTOVOID" 2>&1 | grep -E "[A-Za-z0-9._ ]+: [0-9]+ bytes"
rm -f tmpro tmpro.zst
println "test: overwrite input file (must fail)"
zstd tmp -fo tmp && die "zstd compression overwrote the input file"
zstd tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file"
println "test: detect that input file does not exist"
zstd nothere && die "zstd hasn't detected that input file does not exist"
println "test: --[no-]compress-literals"
zstd tmp -c --no-compress-literals -1 | zstd -t
zstd tmp -c --no-compress-literals --fast=1 | zstd -t
zstd tmp -c --no-compress-literals -19 | zstd -t
zstd tmp -c --compress-literals -1 | zstd -t
zstd tmp -c --compress-literals --fast=1 | zstd -t
zstd tmp -c --compress-literals -19 | zstd -t
zstd -b --fast=1 -i0e1 tmp --compress-literals
zstd -b --fast=1 -i0e1 tmp --no-compress-literals
println "test: --no-check for decompression"
zstd -f tmp -o tmp_corrupt.zst --check
zstd -f tmp -o tmp.zst --no-check
printf '\xDE\xAD\xBE\xEF' | dd of=tmp_corrupt.zst bs=1 seek=$(($(wc -c < "tmp_corrupt.zst") - 4)) count=4 conv=notrunc # corrupt checksum in tmp
zstd -d -f tmp_corrupt.zst --no-check
zstd -d -f tmp_corrupt.zst --check --no-check # final flag overrides
zstd -d -f tmp.zst --no-check
if [ "$isWindows" = false ] && [ "$UNAME" != "AIX" ]; then
if [ -n "$(which readelf)" ]; then
println "test: check if binary has executable stack (#2963)"
readelf -lW "$ZSTD_BIN" | grep 'GNU_STACK .* RW ' || die "zstd binary has executable stack!"
fi
fi
println "\n===> --exclude-compressed flag"
rm -rf precompressedFilterTestDir
mkdir -p precompressedFilterTestDir
datagen $size > precompressedFilterTestDir/input.5
datagen $size > precompressedFilterTestDir/input.6
zstd --exclude-compressed --long --rm -r precompressedFilterTestDir
datagen $size > precompressedFilterTestDir/input.7
datagen $size > precompressedFilterTestDir/input.8
zstd --exclude-compressed --long --rm -r precompressedFilterTestDir
test ! -f precompressedFilterTestDir/input.5.zst.zst
test ! -f precompressedFilterTestDir/input.6.zst.zst
file1timestamp=`$MTIME precompressedFilterTestDir/input.5.zst`
file2timestamp=`$MTIME precompressedFilterTestDir/input.7.zst`
if [ $file2timestamp -ge $file1timestamp ]; then
println "Test is successful. input.5.zst is precompressed and therefore not compressed/modified again."
else
println "Test is not successful"
fi
# File Extension check.
datagen $size > precompressedFilterTestDir/input.zstbar
zstd --exclude-compressed --long --rm -r precompressedFilterTestDir
# zstd should compress input.zstbar
test -f precompressedFilterTestDir/input.zstbar.zst
# Check without the --exclude-compressed flag
zstd --long --rm -r precompressedFilterTestDir
# Files should get compressed again without the --exclude-compressed flag.
test -f precompressedFilterTestDir/input.5.zst.zst
test -f precompressedFilterTestDir/input.6.zst.zst
rm -rf precompressedFilterTestDir
println "Test completed"
println "\n===> warning prompts should not occur if stdin is an input"
println "y" > tmpPrompt
println "hello world" >> tmpPrompt
zstd tmpPrompt -f
zstd < tmpPrompt -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite"
zstd < tmpPrompt -o tmpPrompt.zst -f # should successfully overwrite with -f
zstd -q -d -f tmpPrompt.zst -o tmpPromptRegenerated
$DIFF tmpPromptRegenerated tmpPrompt # the first 'y' character should not be swallowed
echo 'yes' | zstd tmpPrompt -v -o tmpPrompt.zst # accept piped "y" input to force overwrite when using files
echo 'yes' | zstd < tmpPrompt -v -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite"
zstd tmpPrompt - < tmpPrompt -o tmpPromp.zst --rm && die "should have aborted immediately and failed to remove"
println "Test completed"
println "\n===> recursive mode test "
# combination of -r with empty list of input file
zstd -c -r < tmp > tmp.zst
# combination of -r with empty folder
mkdir -p tmpEmptyDir
zstd -r tmpEmptyDir
rm -rf tmpEmptyDir
println "\n===> file removal"
zstd -f --rm tmp
test ! -f tmp # tmp should no longer be present
zstd -f -d --rm tmp.zst
test ! -f tmp.zst # tmp.zst should no longer be present
println "test: --rm is disabled when output is stdout"
test -f tmp
zstd --rm tmp -c > $INTOVOID
test -f tmp # tmp shall still be there
zstd -f --rm tmp -c > $INTOVOID
test -f tmp # tmp shall still be there
zstd -f tmp -c > $INTOVOID --rm
test -f tmp # tmp shall still be there
println "test: --rm is disabled when multiple inputs are concatenated into a single output"
cp tmp tmp2
zstd --rm tmp tmp2 -c > $INTOVOID
test -f tmp
test -f tmp2
rm -f tmp3.zst
echo 'y' | zstd -v tmp tmp2 -o tmp3.zst --rm # prompt for confirmation
test -f tmp
test -f tmp2
zstd -f tmp tmp2 -o tmp3.zst --rm # just warns, no prompt
test -f tmp
test -f tmp2
zstd -q tmp tmp2 -o tmp3.zst --rm && die "should refuse to concatenate"
println "test : should quietly not remove non-regular file"
println hello > tmp
zstd tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID"
grep "Refusing to remove non-regular file" tmplog && die
rm -f tmplog
zstd tmp -f -o "$INTOVOID" 2>&1 | grep "Refusing to remove non-regular file" && die
println "test : --rm on stdin"
println a | zstd --rm > $INTOVOID # --rm should remain silent
rm -f tmp
zstd -f tmp && die "tmp not present : should have failed"
test ! -f tmp.zst # tmp.zst should not be created
println "test : -d -f do not delete destination when source is not present"
touch tmp # create destination file
zstd -d -f tmp.zst && die "attempt to decompress a non existing file"
test -f tmp # destination file should still be present
println "test : -f do not delete destination when source is not present"
rm -f tmp # erase source file
touch tmp.zst # create destination file
zstd -f tmp && die "attempt to compress a non existing file"
test -f tmp.zst # destination file should still be present
rm -rf tmp* # may also erase tmp* directory from previous failed run
println "\n===> decompression only tests "
# the following test verifies that the decoder is compatible with RLE as first block
# older versions of zstd cli are not able to decode such corner case.
# As a consequence, the zstd cli do not generate them, to maintain compatibility with older versions.
dd bs=1048576 count=1 if=/dev/zero of=tmp
zstd -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst"
$DIFF -s tmp1 tmp
touch tmp_empty
zstd -d -o tmp2 "$TESTDIR/golden-decompression/empty-block.zst"
$DIFF -s tmp2 tmp_empty
zstd -t "$TESTDIR/golden-decompression/zeroSeq_2B.zst"
zstd -t "$TESTDIR/golden-decompression-errors/zeroSeq_extraneous.zst" && die "invalid Sequences section should have been detected"
rm -f tmp*
println "\n===> compress multiple files"
println hello > tmp1
println world > tmp2
zstd tmp1 tmp2 -o "$INTOVOID" -f
zstd tmp1 tmp2 -c | zstd -t
echo 'y' | zstd -v tmp1 tmp2 -o tmp.zst
test ! -f tmp1.zst
test ! -f tmp2.zst
zstd tmp1 tmp2
zstd -t tmp1.zst tmp2.zst
zstd -dc tmp1.zst tmp2.zst
zstd tmp1.zst tmp2.zst -o "$INTOVOID" -f
echo 'y' | zstd -v -d tmp1.zst tmp2.zst -o tmp
touch tmpexists
zstd tmp1 tmp2 -f -o tmpexists
zstd tmp1 tmp2 -q -o tmpexists && die "should have refused to overwrite"
println gooder > tmp_rm1
println boi > tmp_rm2
println worldly > tmp_rm3
echo 'y' | zstd -v tmp_rm1 tmp_rm2 -v -o tmp_rm3.zst
test -f tmp_rm1
test -f tmp_rm2
cp tmp_rm3.zst tmp_rm4.zst
echo 'Y' | zstd -v -d tmp_rm3.zst tmp_rm4.zst -v -o tmp_rm_out --rm
test -f tmp_rm3.zst
test -f tmp_rm4.zst
println gooder > tmpexists1
zstd tmpexists1 tmpexists -c --rm -f > $INTOVOID
# Bug: PR #972
if [ "$?" -eq 139 ]; then
die "should not have segfaulted"
fi
test -f tmpexists1
test -f tmpexists
println "\n===> multiple files and shell completion "
datagen -s1 > tmp1 2> $INTOVOID
datagen -s2 -g100K > tmp2 2> $INTOVOID
datagen -s3 -g1M > tmp3 2> $INTOVOID
println "compress tmp* : "
zstd -f tmp*
test -f tmp1.zst
test -f tmp2.zst
test -f tmp3.zst
rm -f tmp1 tmp2 tmp3
println "decompress tmp* : "
zstd -df ./*.zst
test -f tmp1
test -f tmp2
test -f tmp3
println "compress tmp* into stdout > tmpall : "
zstd -c tmp1 tmp2 tmp3 > tmpall
test -f tmpall # should check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst)
println "decompress tmpall* into stdout > tmpdec : "
cp tmpall tmpall2
zstd -dc tmpall* > tmpdec
test -f tmpdec # should check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3))
println "compress multiple files including a missing one (notHere) : "
zstd -f tmp1 notHere tmp2 && die "missing file not detected!"
rm -f tmp*
if [ "$isWindows" = false ] ; then
println "\n===> zstd fifo named pipe test "
echo "Hello World!" > tmp_original
mkfifo tmp_named_pipe
# note : fifo test doesn't work in combination with `dd` or `cat`
echo "Hello World!" > tmp_named_pipe &
zstd tmp_named_pipe -o tmp_compressed
zstd -d -o tmp_decompressed tmp_compressed
$DIFF -s tmp_original tmp_decompressed
rm -rf tmp*
fi
println "\n===> zstd created file permissions tests"
if [ "$isWindows" = false ] ; then
rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out # todo: remove
ORIGINAL_UMASK=$(umask)
umask 0000
datagen > tmp1
datagen > tmp2
assertFilePermissions tmp1 666
assertFilePermissions tmp2 666
println "test : copy 666 permissions in file -> file compression "
zstd -f tmp1 -o tmp1.zst
assertSamePermissions tmp1 tmp1.zst
println "test : copy 666 permissions in file -> file decompression "
zstd -f -d tmp1.zst -o tmp1.out
assertSamePermissions tmp1.zst tmp1.out
rm -f tmp1.zst tmp1.out
println "test : copy 400 permissions in file -> file compression (write to a read-only file) "
chmod 0400 tmp1
assertFilePermissions tmp1 400
zstd -f tmp1 -o tmp1.zst
assertSamePermissions tmp1 tmp1.zst
println "test : copy 400 permissions in file -> file decompression (write to a read-only file) "
zstd -f -d tmp1.zst -o tmp1
assertSamePermissions tmp1.zst tmp1
rm -f tmp1.zst tmp1.out
println "test : check created permissions from stdin input in compression "
zstd -f -o tmp1.zst < tmp1
assertFilePermissions tmp1.zst 666
println "test : check created permissions from stdin input in decompression "
zstd -f -d -o tmp1.out < tmp1.zst
assertFilePermissions tmp1.out 666
rm -f tmp1.zst tmp1.out
println "test : check created permissions from multiple inputs in compression "
zstd -f tmp1 tmp2 -o tmp1.zst
assertFilePermissions tmp1.zst 666
println "test : check created permissions from multiple inputs in decompression "
cp tmp1.zst tmp2.zst
zstd -f -d tmp1.zst tmp2.zst -o tmp1.out
assertFilePermissions tmp1.out 666
rm -f tmp1.zst tmp2.zst tmp1.out tmp2.out
println "test : check permissions on pre-existing output file in compression "
chmod 0600 tmp1
touch tmp1.zst
chmod 0400 tmp1.zst
zstd -f tmp1 -o tmp1.zst
assertFilePermissions tmp1.zst 600
println "test : check permissions on pre-existing output file in decompression "
chmod 0400 tmp1.zst
touch tmp1.out
chmod 0200 tmp1.out
zstd -f -d tmp1.zst -o tmp1.out
assertFilePermissions tmp1.out 400
umask 0666
chmod 0666 tmp1 tmp2
rm -f tmp1.zst tmp1.out
println "test : respect umask when compressing from stdin input "
zstd -f -o tmp1.zst < tmp1
assertFilePermissions tmp1.zst 0
println "test : respect umask when decompressing from stdin input "
chmod 0666 tmp1.zst
zstd -f -d -o tmp1.out < tmp1.zst
assertFilePermissions tmp1.out 0
rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out
umask $ORIGINAL_UMASK
fi
if [ -n "$DEVNULLRIGHTS" ] ; then
# these tests requires sudo rights, which is uncommon.
# they are only triggered if DEVNULLRIGHTS macro is defined.
println "\n===> checking /dev/null permissions are unaltered "
datagen > tmp
sudoZstd tmp -o $INTOVOID # sudo rights could modify /dev/null permissions
sudoZstd tmp -c > $INTOVOID
zstd tmp -f -o tmp.zst
sudoZstd -d tmp.zst -c > $INTOVOID
sudoZstd -d tmp.zst -o $INTOVOID
ls -las $INTOVOID | grep "rw-rw-rw-"
fi
if [ -n "$READFROMBLOCKDEVICE" ] ; then
# This creates a temporary block device, which is only possible on unix-y
# systems, is somewhat invasive, and requires sudo. For these reasons, you
# have to specifically ask for this test.
println "\n===> checking that zstd can read from a block device"
datagen -g65536 > tmp.img
sudo losetup -fP tmp.img
LOOP_DEV=$(losetup -a | grep 'tmp\.img' | cut -f1 -d:)
[ -z "$LOOP_DEV" ] && die "failed to get loopback device"
sudoZstd $LOOP_DEV -c > tmp.img.zst && die "should fail without -f"
sudoZstd -f $LOOP_DEV -c > tmp.img.zst
zstd -d tmp.img.zst -o tmp.img.copy
sudo losetup -d $LOOP_DEV
$DIFF -s tmp.img tmp.img.copy || die "round trip failed"
rm -f tmp.img tmp.img.zst tmp.img.copy
fi
println "\n===> zstd created file timestamp tests"
datagen > tmp
touch -m -t 200001010000.00 tmp
println "test : copy mtime in file -> file compression "
zstd -f tmp -o tmp.zst
assertSameMTime tmp tmp.zst
println "test : copy mtime in file -> file decompression "
zstd -f -d tmp.zst -o tmp.out
assertSameMTime tmp.zst tmp.out
rm -f tmp
println "\n===> compress multiple files into an output directory, --output-dir-flat"
println henlo > tmp1
mkdir tmpInputTestDir
mkdir tmpInputTestDir/we
mkdir tmpInputTestDir/we/must
mkdir tmpInputTestDir/we/must/go
mkdir tmpInputTestDir/we/must/go/deeper
println cool > tmpInputTestDir/we/must/go/deeper/tmp2
mkdir tmpOutDir
zstd tmp1 tmpInputTestDir/we/must/go/deeper/tmp2 --output-dir-flat tmpOutDir
test -f tmpOutDir/tmp1.zst
test -f tmpOutDir/tmp2.zst
println "test : decompress multiple files into an output directory, --output-dir-flat"
mkdir tmpOutDirDecomp
zstd tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp
test -f tmpOutDirDecomp/tmp2
test -f tmpOutDirDecomp/tmp1
rm -f tmpOutDirDecomp/*
zstd tmpOutDir -r -d --output-dir-flat=tmpOutDirDecomp
test -f tmpOutDirDecomp/tmp2
test -f tmpOutDirDecomp/tmp1
rm -rf tmp*
if [ "$isWindows" = false ] ; then
println "\n===> compress multiple files into an output directory and mirror input folder, --output-dir-mirror"
println "test --output-dir-mirror" > tmp1
mkdir -p tmpInputTestDir/we/.../..must/go/deeper..
println cool > tmpInputTestDir/we/.../..must/go/deeper../tmp2
zstd tmp1 -r tmpInputTestDir --output-dir-mirror tmpOutDir
test -f tmpOutDir/tmp1.zst
test -f tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2.zst
println "test: compress input dir will be ignored if it has '..'"
zstd -r tmpInputTestDir/we/.../..must/../..mustgo/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'"
zstd -r tmpInputTestDir/we/.../..must/deeper../.. --output-dir-mirror non-exist && die "input cannot contain '..'"
zstd -r ../tests/tmpInputTestDir/we/.../..must/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'"
test ! -d non-exist
println "test: compress input dir should succeed with benign uses of '..'"
zstd -r tmpInputTestDir/we/.../..must/go/deeper.. --output-dir-mirror tmpout
test -d tmpout
println "test : decompress multiple files into an output directory, --output-dir-mirror"
zstd tmpOutDir -r -d --output-dir-mirror tmpOutDirDecomp
test -f tmpOutDirDecomp/tmpOutDir/tmp1
test -f tmpOutDirDecomp/tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2
println "test: decompress input dir will be ignored if it has '..'"
zstd -r tmpOutDir/tmpInputTestDir/we/.../..must/../..must --output-dir-mirror non-exist && die "input cannot contain '..'"
test ! -d non-exist
rm -rf tmp*
fi
println "test : compress multiple files reading them from a file, --filelist=FILE"
println "Hello world!, file1" > tmp1
println "Hello world!, file2" > tmp2
println tmp1 > tmp_fileList
println tmp2 >> tmp_fileList
zstd -f --filelist=tmp_fileList
test -f tmp2.zst
test -f tmp1.zst
println "test : alternate syntax: --filelist FILE"
zstd -f --filelist tmp_fileList
test -f tmp2.zst
test -f tmp1.zst
println "test : reading file list from a symlink, --filelist=FILE"
rm -f *.zst
ln -s tmp_fileList tmp_symLink
zstd -f --filelist=tmp_symLink
test -f tmp2.zst
test -f tmp1.zst
println "test : compress multiple files reading them from multiple files, --filelist=FILE"
rm -f *.zst
println "Hello world!, file3" > tmp3
println "Hello world!, file4" > tmp4
println tmp3 > tmp_fileList2
println tmp4 >> tmp_fileList2
zstd -f --filelist=tmp_fileList --filelist=tmp_fileList2
test -f tmp1.zst
test -f tmp2.zst
test -f tmp3.zst
test -f tmp4.zst
println "test : decompress multiple files reading them from a file, --filelist=FILE"
rm -f tmp1 tmp2
println tmp1.zst > tmpZst
println tmp2.zst >> tmpZst
zstd -d -f --filelist=tmpZst
test -f tmp1
test -f tmp2
println "test : decompress multiple files reading them from multiple files, --filelist=FILE"
rm -f tmp1 tmp2 tmp3 tmp4
println tmp3.zst > tmpZst2
println tmp4.zst >> tmpZst2
zstd -d -f --filelist=tmpZst --filelist=tmpZst2
test -f tmp1
test -f tmp2
test -f tmp3
test -f tmp4
println "test : survive the list of files with too long filenames (--filelist=FILE)"
datagen -g5M > tmp_badList
zstd -qq -f --filelist=tmp_badList && die "should have failed : file name length is too long" # printing very long text garbage on console will cause CI failure
println "test : survive a list of files which is text garbage (--filelist=FILE)"
datagen > tmp_badList
zstd -qq -f --filelist=tmp_badList && die "should have failed : list is text garbage" # printing very long text garbage on console will cause CI failure
println "test : survive a list of files which is binary garbage (--filelist=FILE)"
datagen -P0 -g1M > tmp_badList
zstd -qq -f --filelist=tmp_badList && die "should have failed : list is binary garbage" # let's avoid printing binary garbage on console
println "test : try to overflow internal list of files (--filelist=FILE)"
touch tmp1 tmp2 tmp3 tmp4 tmp5 tmp6
ls tmp* > tmpList
zstd -f tmp1 --filelist=tmpList --filelist=tmpList tmp2 tmp3 # can trigger an overflow of internal file list
rm -rf tmp*
println "\n===> --[no-]content-size tests"
datagen > tmp_contentsize
zstd -f tmp_contentsize
zstd -lv tmp_contentsize.zst | grep "Decompressed Size:"
zstd -f --no-content-size tmp_contentsize
zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" && die
zstd -f --content-size tmp_contentsize
zstd -lv tmp_contentsize.zst | grep "Decompressed Size:"
zstd -f --content-size --no-content-size tmp_contentsize
zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" && die
rm -rf tmp*
println "test : show-default-cparams regular"
datagen > tmp
zstd --show-default-cparams -f tmp
zstd --show-default-cparams -d tmp.zst && die "error: can't use --show-default-cparams in decompression mode"
rm -rf tmp*
println "test : show-default-cparams recursive"
mkdir tmp_files
datagen -g15000 > tmp_files/tmp1
datagen -g129000 > tmp_files/tmp2
datagen -g257000 > tmp_files/tmp3
zstd --show-default-cparams -f -r tmp_files
rm -rf tmp*
println "test : show compression parameters in verbose mode"
datagen > tmp
zstd -vv tmp 2>&1 | \
grep -q -E -- "--zstd=wlog=[[:digit:]]+,clog=[[:digit:]]+,hlog=[[:digit:]]+,\
slog=[[:digit:]]+,mml=[[:digit:]]+,tlen=[[:digit:]]+,strat=[[:digit:]]+"
rm -rf tmp*
println "\n===> Advanced compression parameters "
println "Hello world!" | zstd --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
println "Hello world!" | zstd --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
println "Hello world!" | zstd --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
println "Hello world!" | zstd --zstd=strategy=10 - -o tmp.zst && die "parameter out of bound not detected!" # > btultra2 : does not exist
test ! -f tmp.zst # tmp.zst should not be created
roundTripTest -g512K
roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6"
roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6"
roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7"
roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7"
roundTripTest -g64K "19 --zstd=strat=9" # btultra2
println "\n===> Pass-Through mode "
println "Hello world 1!" | zstd -df
println "Hello world 2!" | zstd -dcf
println "Hello world 3!" > tmp1
zstd -dcf tmp1
println "" | zstd -df > tmp1
println "" > tmp2
$DIFF -q tmp1 tmp2
println "1" | zstd -df > tmp1
println "1" > tmp2
$DIFF -q tmp1 tmp2
println "12" | zstd -df > tmp1
println "12" > tmp2
$DIFF -q tmp1 tmp2
rm -rf tmp*
println "\n===> frame concatenation "
println "hello " > hello.tmp
println "world!" > world.tmp
cat hello.tmp world.tmp > helloworld.tmp
zstd -c hello.tmp > hello.zst
zstd -c world.tmp > world.zst
zstd -c hello.tmp world.tmp > helloworld.zst
zstd -dc helloworld.zst > result.tmp
$DIFF helloworld.tmp result.tmp
cat hello.zst world.zst > helloworld.zst
zstd -dc helloworld.zst > result.tmp
cat result.tmp
$DIFF helloworld.tmp result.tmp
println "frame concatenation without checksum"
zstd -c hello.tmp > hello.zst --no-check
zstd -c world.tmp > world.zst --no-check
cat hello.zst world.zst > helloworld.zstd
zstd -dc helloworld.zst > result.tmp
$DIFF helloworld.tmp result.tmp
println "testing zstdcat symlink"
ln -sf "$ZSTD_BIN" zstdcat
$EXE_PREFIX ./zstdcat helloworld.zst > result.tmp
$DIFF helloworld.tmp result.tmp
ln -s helloworld.zst helloworld.link.zst
$EXE_PREFIX ./zstdcat helloworld.link.zst > result.tmp
$DIFF helloworld.tmp result.tmp
rm -f zstdcat
rm -f result.tmp
println "testing zcat symlink"
ln -sf "$ZSTD_BIN" zcat
$EXE_PREFIX ./zcat helloworld.zst > result.tmp
$DIFF helloworld.tmp result.tmp
$EXE_PREFIX ./zcat helloworld.link.zst > result.tmp
$DIFF helloworld.tmp result.tmp
rm -f zcat
rm -f ./*.tmp ./*.zstd
println "frame concatenation tests completed"
if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] && [ "$UNAME" != "OpenBSD" ] && [ "$UNAME" != "AIX" ]; then
println "\n**** flush write error test **** "
println "println foo | zstd > /dev/full"
println foo | zstd > /dev/full && die "write error not detected!"
println "println foo | zstd | zstd -d > /dev/full"
println foo | zstd | zstd -d > /dev/full && die "write error not detected!"
fi
if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] ; then
println "\n===> symbolic link test "
rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst
println "hello world" > hello.tmp
ln -s hello.tmp world.tmp
ln -s hello.tmp world2.tmp
zstd world.tmp hello.tmp || true
test -f hello.tmp.zst # regular file should have been compressed!
test ! -f world.tmp.zst # symbolic link should not have been compressed!
zstd world.tmp || true
test ! -f world.tmp.zst # symbolic link should not have been compressed!
zstd world.tmp world2.tmp || true
test ! -f world.tmp.zst # symbolic link should not have been compressed!
test ! -f world2.tmp.zst # symbolic link should not have been compressed!
zstd world.tmp hello.tmp -f
test -f world.tmp.zst # symbolic link should have been compressed with --force
rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst
fi
println "\n===> test sparse file support "
datagen -g5M -P100 > tmpSparse
zstd tmpSparse -c | zstd -dv -o tmpSparseRegen
$DIFF -s tmpSparse tmpSparseRegen
zstd tmpSparse -c | zstd -dv --sparse -c > tmpOutSparse
$DIFF -s tmpSparse tmpOutSparse
zstd tmpSparse -c | zstd -dv --no-sparse -c > tmpOutNoSparse
$DIFF -s tmpSparse tmpOutNoSparse
ls -ls tmpSparse* # look at file size and block size on disk
datagen -s1 -g1200007 -P100 | zstd | zstd -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks)
datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd
ls -ls tmpSparseOdd # look at file size and block size on disk
println "\n Sparse Compatibility with Console :"
println "Hello World 1 !" | zstd | zstd -d -c
println "Hello World 2 !" | zstd | zstd -d | cat
println "\n Sparse Compatibility with Append :"
datagen -P100 -g1M > tmpSparse1M
cat tmpSparse1M tmpSparse1M > tmpSparse2M
zstd -v -f tmpSparse1M -o tmpSparseCompressed
zstd -d -v -f tmpSparseCompressed -o tmpSparseRegenerated
zstd -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated
ls -ls tmpSparse* # look at file size and block size on disk
$DIFF tmpSparse2M tmpSparseRegenerated
rm -f tmpSparse*
println "\n===> stream-size mode"
datagen -g11000 > tmp
println "test : basic file compression vs sized streaming compression"
file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst)
stream_size=$(cat tmp | zstd -14 --stream-size=11000 | wc -c)
if [ "$stream_size" -gt "$file_size" ]; then
die "hinted compression larger than expected"
fi
println "test : sized streaming compression and decompression"
cat tmp | zstd -14 -f tmp -o tmp.zst --stream-size=11000
zstd -df tmp.zst -o tmp_decompress
cmp tmp tmp_decompress || die "difference between original and decompressed file"
println "test : incorrect stream size"
cat tmp | zstd -14 -f -o tmp.zst --stream-size=11001 && die "should fail with incorrect stream size"
println "\n===> zstd zero weight dict test "
rm -f tmp*
cp "$TESTDIR/dict-files/zero-weight-dict" tmp_input
zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input
zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp
$DIFF tmp_decomp tmp_input
rm -rf tmp*
println "\n===> zstd (valid) zero weight dict test "
rm -f tmp*
# 0 has a non-zero weight in the dictionary
echo "0000000000000000000000000" > tmp_input
zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input
zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp
$DIFF tmp_decomp tmp_input
rm -rf tmp*
println "\n===> size-hint mode"
datagen -g11000 > tmp
datagen -g11000 > tmp2
datagen > tmpDict
println "test : basic file compression vs hinted streaming compression"
file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst)
stream_size=$(cat tmp | zstd -14 --size-hint=11000 | wc -c)
if [ "$stream_size" -ge "$file_size" ]; then
die "hinted compression larger than expected"
fi
println "test : hinted streaming compression and decompression"
cat tmp | zstd -14 -f -o tmp.zst --size-hint=11000
zstd -df tmp.zst -o tmp_decompress
cmp tmp tmp_decompress || die "difference between original and decompressed file"
println "test : hinted streaming compression with dictionary"
cat tmp | zstd -14 -f -D tmpDict --size-hint=11000 | zstd -t -D tmpDict
println "test : multiple file compression with hints and dictionary"
zstd -14 -f -D tmpDict --size-hint=11000 tmp tmp2
zstd -14 -f -o tmp1_.zst -D tmpDict --size-hint=11000 tmp
zstd -14 -f -o tmp2_.zst -D tmpDict --size-hint=11000 tmp2
cmp tmp.zst tmp1_.zst || die "first file's output differs"
cmp tmp2.zst tmp2_.zst || die "second file's output differs"
println "test : incorrect hinted stream sizes"
cat tmp | zstd -14 -f --size-hint=11050 | zstd -t # slightly too high
cat tmp | zstd -14 -f --size-hint=10950 | zstd -t # slightly too low
cat tmp | zstd -14 -f --size-hint=22000 | zstd -t # considerably too high
cat tmp | zstd -14 -f --size-hint=5500 | zstd -t # considerably too low
println "test : allows and interprets K,KB,KiB,M,MB and MiB suffix"