many small files benchmark

sharkdp · Feb 7, 2022 · 386c898 · 386c898
1 parent 8d45cbe
commit 386c898
Show file tree

Hide file tree

Showing 103 changed files with 112 additions and 1 deletion.
diff --git a/Cargo.toml b/Cargo.toml
@@ -65,7 +65,7 @@ dirs-next = { version = "2.0.0", optional = true }
 grep-cli = { version = "0.1.6", optional = true }
 regex = { version = "1.0", optional = true }
 walkdir = { version = "2.0", optional = true }
-bytesize = {version = "1.1.0" }
+bytesize = { version = "1.1.0" }
 
 [dependencies.git2]
 version = "0.13"

diff --git a/tests/benchmarks/many-small-files/small-file-0.txt b/tests/benchmarks/many-small-files/small-file-0.txt
@@ -0,0 +1 @@
+I am small file 0
diff --git a/tests/benchmarks/many-small-files/small-file-1.txt b/tests/benchmarks/many-small-files/small-file-1.txt
@@ -0,0 +1 @@
+I am small file 1
diff --git a/tests/benchmarks/many-small-files/small-file-10.txt b/tests/benchmarks/many-small-files/small-file-10.txt
@@ -0,0 +1 @@
+I am small file 10
diff --git a/tests/benchmarks/many-small-files/small-file-100.txt b/tests/benchmarks/many-small-files/small-file-100.txt
@@ -0,0 +1 @@
+I am small file 100
diff --git a/tests/benchmarks/many-small-files/small-file-11.txt b/tests/benchmarks/many-small-files/small-file-11.txt
@@ -0,0 +1 @@
+I am small file 11
diff --git a/tests/benchmarks/many-small-files/small-file-12.txt b/tests/benchmarks/many-small-files/small-file-12.txt
@@ -0,0 +1 @@
+I am small file 12
diff --git a/tests/benchmarks/many-small-files/small-file-13.txt b/tests/benchmarks/many-small-files/small-file-13.txt
@@ -0,0 +1 @@
+I am small file 13
diff --git a/tests/benchmarks/many-small-files/small-file-14.txt b/tests/benchmarks/many-small-files/small-file-14.txt
@@ -0,0 +1 @@
+I am small file 14
diff --git a/tests/benchmarks/many-small-files/small-file-15.txt b/tests/benchmarks/many-small-files/small-file-15.txt
@@ -0,0 +1 @@
+I am small file 15
diff --git a/tests/benchmarks/many-small-files/small-file-16.txt b/tests/benchmarks/many-small-files/small-file-16.txt
@@ -0,0 +1 @@
+I am small file 16
diff --git a/tests/benchmarks/many-small-files/small-file-17.txt b/tests/benchmarks/many-small-files/small-file-17.txt
@@ -0,0 +1 @@
+I am small file 17
diff --git a/tests/benchmarks/many-small-files/small-file-18.txt b/tests/benchmarks/many-small-files/small-file-18.txt
@@ -0,0 +1 @@
+I am small file 18
diff --git a/tests/benchmarks/many-small-files/small-file-19.txt b/tests/benchmarks/many-small-files/small-file-19.txt
@@ -0,0 +1 @@
+I am small file 19
diff --git a/tests/benchmarks/many-small-files/small-file-2.txt b/tests/benchmarks/many-small-files/small-file-2.txt
@@ -0,0 +1 @@
+I am small file 2
diff --git a/tests/benchmarks/many-small-files/small-file-20.txt b/tests/benchmarks/many-small-files/small-file-20.txt
@@ -0,0 +1 @@
+I am small file 20
diff --git a/tests/benchmarks/many-small-files/small-file-21.txt b/tests/benchmarks/many-small-files/small-file-21.txt
@@ -0,0 +1 @@
+I am small file 21
diff --git a/tests/benchmarks/many-small-files/small-file-22.txt b/tests/benchmarks/many-small-files/small-file-22.txt
@@ -0,0 +1 @@
+I am small file 22
diff --git a/tests/benchmarks/many-small-files/small-file-23.txt b/tests/benchmarks/many-small-files/small-file-23.txt
@@ -0,0 +1 @@
+I am small file 23
diff --git a/tests/benchmarks/many-small-files/small-file-24.txt b/tests/benchmarks/many-small-files/small-file-24.txt
@@ -0,0 +1 @@
+I am small file 24
diff --git a/tests/benchmarks/many-small-files/small-file-25.txt b/tests/benchmarks/many-small-files/small-file-25.txt
@@ -0,0 +1 @@
+I am small file 25
diff --git a/tests/benchmarks/many-small-files/small-file-26.txt b/tests/benchmarks/many-small-files/small-file-26.txt
@@ -0,0 +1 @@
+I am small file 26
diff --git a/tests/benchmarks/many-small-files/small-file-27.txt b/tests/benchmarks/many-small-files/small-file-27.txt
@@ -0,0 +1 @@
+I am small file 27
diff --git a/tests/benchmarks/many-small-files/small-file-28.txt b/tests/benchmarks/many-small-files/small-file-28.txt
@@ -0,0 +1 @@
+I am small file 28
diff --git a/tests/benchmarks/many-small-files/small-file-29.txt b/tests/benchmarks/many-small-files/small-file-29.txt
@@ -0,0 +1 @@
+I am small file 29
diff --git a/tests/benchmarks/many-small-files/small-file-3.txt b/tests/benchmarks/many-small-files/small-file-3.txt
@@ -0,0 +1 @@
+I am small file 3
diff --git a/tests/benchmarks/many-small-files/small-file-30.txt b/tests/benchmarks/many-small-files/small-file-30.txt
@@ -0,0 +1 @@
+I am small file 30
diff --git a/tests/benchmarks/many-small-files/small-file-31.txt b/tests/benchmarks/many-small-files/small-file-31.txt
@@ -0,0 +1 @@
+I am small file 31
diff --git a/tests/benchmarks/many-small-files/small-file-32.txt b/tests/benchmarks/many-small-files/small-file-32.txt
@@ -0,0 +1 @@
+I am small file 32
diff --git a/tests/benchmarks/many-small-files/small-file-33.txt b/tests/benchmarks/many-small-files/small-file-33.txt
@@ -0,0 +1 @@
+I am small file 33
diff --git a/tests/benchmarks/many-small-files/small-file-34.txt b/tests/benchmarks/many-small-files/small-file-34.txt
@@ -0,0 +1 @@
+I am small file 34
diff --git a/tests/benchmarks/many-small-files/small-file-35.txt b/tests/benchmarks/many-small-files/small-file-35.txt
@@ -0,0 +1 @@
+I am small file 35
diff --git a/tests/benchmarks/many-small-files/small-file-36.txt b/tests/benchmarks/many-small-files/small-file-36.txt
@@ -0,0 +1 @@
+I am small file 36
diff --git a/tests/benchmarks/many-small-files/small-file-37.txt b/tests/benchmarks/many-small-files/small-file-37.txt
@@ -0,0 +1 @@
+I am small file 37
diff --git a/tests/benchmarks/many-small-files/small-file-38.txt b/tests/benchmarks/many-small-files/small-file-38.txt
@@ -0,0 +1 @@
+I am small file 38
diff --git a/tests/benchmarks/many-small-files/small-file-39.txt b/tests/benchmarks/many-small-files/small-file-39.txt
@@ -0,0 +1 @@
+I am small file 39
diff --git a/tests/benchmarks/many-small-files/small-file-4.txt b/tests/benchmarks/many-small-files/small-file-4.txt
@@ -0,0 +1 @@
+I am small file 4
diff --git a/tests/benchmarks/many-small-files/small-file-40.txt b/tests/benchmarks/many-small-files/small-file-40.txt
@@ -0,0 +1 @@
+I am small file 40
diff --git a/tests/benchmarks/many-small-files/small-file-41.txt b/tests/benchmarks/many-small-files/small-file-41.txt
@@ -0,0 +1 @@
+I am small file 41
diff --git a/tests/benchmarks/many-small-files/small-file-42.txt b/tests/benchmarks/many-small-files/small-file-42.txt
@@ -0,0 +1 @@
+I am small file 42
diff --git a/tests/benchmarks/many-small-files/small-file-43.txt b/tests/benchmarks/many-small-files/small-file-43.txt
@@ -0,0 +1 @@
+I am small file 43
diff --git a/tests/benchmarks/many-small-files/small-file-44.txt b/tests/benchmarks/many-small-files/small-file-44.txt
@@ -0,0 +1 @@
+I am small file 44
diff --git a/tests/benchmarks/many-small-files/small-file-45.txt b/tests/benchmarks/many-small-files/small-file-45.txt
@@ -0,0 +1 @@
+I am small file 45
diff --git a/tests/benchmarks/many-small-files/small-file-46.txt b/tests/benchmarks/many-small-files/small-file-46.txt
@@ -0,0 +1 @@
+I am small file 46
diff --git a/tests/benchmarks/many-small-files/small-file-47.txt b/tests/benchmarks/many-small-files/small-file-47.txt
@@ -0,0 +1 @@
+I am small file 47
diff --git a/tests/benchmarks/many-small-files/small-file-48.txt b/tests/benchmarks/many-small-files/small-file-48.txt
@@ -0,0 +1 @@
+I am small file 48
diff --git a/tests/benchmarks/many-small-files/small-file-49.txt b/tests/benchmarks/many-small-files/small-file-49.txt
@@ -0,0 +1 @@
+I am small file 49
diff --git a/tests/benchmarks/many-small-files/small-file-5.txt b/tests/benchmarks/many-small-files/small-file-5.txt
@@ -0,0 +1 @@
+I am small file 5
diff --git a/tests/benchmarks/many-small-files/small-file-50.txt b/tests/benchmarks/many-small-files/small-file-50.txt
@@ -0,0 +1 @@
+I am small file 50
diff --git a/tests/benchmarks/many-small-files/small-file-51.txt b/tests/benchmarks/many-small-files/small-file-51.txt
@@ -0,0 +1 @@
+I am small file 51
diff --git a/tests/benchmarks/many-small-files/small-file-52.txt b/tests/benchmarks/many-small-files/small-file-52.txt
@@ -0,0 +1 @@
+I am small file 52
diff --git a/tests/benchmarks/many-small-files/small-file-53.txt b/tests/benchmarks/many-small-files/small-file-53.txt
@@ -0,0 +1 @@
+I am small file 53
diff --git a/tests/benchmarks/many-small-files/small-file-54.txt b/tests/benchmarks/many-small-files/small-file-54.txt
@@ -0,0 +1 @@
+I am small file 54
diff --git a/tests/benchmarks/many-small-files/small-file-55.txt b/tests/benchmarks/many-small-files/small-file-55.txt
@@ -0,0 +1 @@
+I am small file 55
diff --git a/tests/benchmarks/many-small-files/small-file-56.txt b/tests/benchmarks/many-small-files/small-file-56.txt
@@ -0,0 +1 @@
+I am small file 56
diff --git a/tests/benchmarks/many-small-files/small-file-57.txt b/tests/benchmarks/many-small-files/small-file-57.txt
@@ -0,0 +1 @@
+I am small file 57
diff --git a/tests/benchmarks/many-small-files/small-file-58.txt b/tests/benchmarks/many-small-files/small-file-58.txt
@@ -0,0 +1 @@
+I am small file 58
diff --git a/tests/benchmarks/many-small-files/small-file-59.txt b/tests/benchmarks/many-small-files/small-file-59.txt
@@ -0,0 +1 @@
+I am small file 59
diff --git a/tests/benchmarks/many-small-files/small-file-6.txt b/tests/benchmarks/many-small-files/small-file-6.txt
@@ -0,0 +1 @@
+I am small file 6
diff --git a/tests/benchmarks/many-small-files/small-file-60.txt b/tests/benchmarks/many-small-files/small-file-60.txt
@@ -0,0 +1 @@
+I am small file 60
diff --git a/tests/benchmarks/many-small-files/small-file-61.txt b/tests/benchmarks/many-small-files/small-file-61.txt
@@ -0,0 +1 @@
+I am small file 61
diff --git a/tests/benchmarks/many-small-files/small-file-62.txt b/tests/benchmarks/many-small-files/small-file-62.txt
@@ -0,0 +1 @@
+I am small file 62
diff --git a/tests/benchmarks/many-small-files/small-file-63.txt b/tests/benchmarks/many-small-files/small-file-63.txt
@@ -0,0 +1 @@
+I am small file 63
diff --git a/tests/benchmarks/many-small-files/small-file-64.txt b/tests/benchmarks/many-small-files/small-file-64.txt
@@ -0,0 +1 @@
+I am small file 64
diff --git a/tests/benchmarks/many-small-files/small-file-65.txt b/tests/benchmarks/many-small-files/small-file-65.txt
@@ -0,0 +1 @@
+I am small file 65
diff --git a/tests/benchmarks/many-small-files/small-file-66.txt b/tests/benchmarks/many-small-files/small-file-66.txt
@@ -0,0 +1 @@
+I am small file 66
diff --git a/tests/benchmarks/many-small-files/small-file-67.txt b/tests/benchmarks/many-small-files/small-file-67.txt
@@ -0,0 +1 @@
+I am small file 67
diff --git a/tests/benchmarks/many-small-files/small-file-68.txt b/tests/benchmarks/many-small-files/small-file-68.txt
@@ -0,0 +1 @@
+I am small file 68
diff --git a/tests/benchmarks/many-small-files/small-file-69.txt b/tests/benchmarks/many-small-files/small-file-69.txt
@@ -0,0 +1 @@
+I am small file 69
diff --git a/tests/benchmarks/many-small-files/small-file-7.txt b/tests/benchmarks/many-small-files/small-file-7.txt
@@ -0,0 +1 @@
+I am small file 7
diff --git a/tests/benchmarks/many-small-files/small-file-70.txt b/tests/benchmarks/many-small-files/small-file-70.txt
@@ -0,0 +1 @@
+I am small file 70
diff --git a/tests/benchmarks/many-small-files/small-file-71.txt b/tests/benchmarks/many-small-files/small-file-71.txt
@@ -0,0 +1 @@
+I am small file 71
diff --git a/tests/benchmarks/many-small-files/small-file-72.txt b/tests/benchmarks/many-small-files/small-file-72.txt
@@ -0,0 +1 @@
+I am small file 72
diff --git a/tests/benchmarks/many-small-files/small-file-73.txt b/tests/benchmarks/many-small-files/small-file-73.txt
@@ -0,0 +1 @@
+I am small file 73
diff --git a/tests/benchmarks/many-small-files/small-file-74.txt b/tests/benchmarks/many-small-files/small-file-74.txt
@@ -0,0 +1 @@
+I am small file 74
diff --git a/tests/benchmarks/many-small-files/small-file-75.txt b/tests/benchmarks/many-small-files/small-file-75.txt
@@ -0,0 +1 @@
+I am small file 75
diff --git a/tests/benchmarks/many-small-files/small-file-76.txt b/tests/benchmarks/many-small-files/small-file-76.txt
@@ -0,0 +1 @@
+I am small file 76
diff --git a/tests/benchmarks/many-small-files/small-file-77.txt b/tests/benchmarks/many-small-files/small-file-77.txt
@@ -0,0 +1 @@
+I am small file 77
diff --git a/tests/benchmarks/many-small-files/small-file-78.txt b/tests/benchmarks/many-small-files/small-file-78.txt
@@ -0,0 +1 @@
+I am small file 78
diff --git a/tests/benchmarks/many-small-files/small-file-79.txt b/tests/benchmarks/many-small-files/small-file-79.txt
@@ -0,0 +1 @@
+I am small file 79
diff --git a/tests/benchmarks/many-small-files/small-file-8.txt b/tests/benchmarks/many-small-files/small-file-8.txt
@@ -0,0 +1 @@
+I am small file 8
diff --git a/tests/benchmarks/many-small-files/small-file-80.txt b/tests/benchmarks/many-small-files/small-file-80.txt
@@ -0,0 +1 @@
+I am small file 80
diff --git a/tests/benchmarks/many-small-files/small-file-81.txt b/tests/benchmarks/many-small-files/small-file-81.txt
@@ -0,0 +1 @@
+I am small file 81
diff --git a/tests/benchmarks/many-small-files/small-file-82.txt b/tests/benchmarks/many-small-files/small-file-82.txt
@@ -0,0 +1 @@
+I am small file 82
diff --git a/tests/benchmarks/many-small-files/small-file-83.txt b/tests/benchmarks/many-small-files/small-file-83.txt
@@ -0,0 +1 @@
+I am small file 83
diff --git a/tests/benchmarks/many-small-files/small-file-84.txt b/tests/benchmarks/many-small-files/small-file-84.txt
@@ -0,0 +1 @@
+I am small file 84
diff --git a/tests/benchmarks/many-small-files/small-file-85.txt b/tests/benchmarks/many-small-files/small-file-85.txt
@@ -0,0 +1 @@
+I am small file 85
diff --git a/tests/benchmarks/many-small-files/small-file-86.txt b/tests/benchmarks/many-small-files/small-file-86.txt
@@ -0,0 +1 @@
+I am small file 86
diff --git a/tests/benchmarks/many-small-files/small-file-87.txt b/tests/benchmarks/many-small-files/small-file-87.txt
@@ -0,0 +1 @@
+I am small file 87
diff --git a/tests/benchmarks/many-small-files/small-file-88.txt b/tests/benchmarks/many-small-files/small-file-88.txt
@@ -0,0 +1 @@
+I am small file 88
diff --git a/tests/benchmarks/many-small-files/small-file-89.txt b/tests/benchmarks/many-small-files/small-file-89.txt
@@ -0,0 +1 @@
+I am small file 89
diff --git a/tests/benchmarks/many-small-files/small-file-9.txt b/tests/benchmarks/many-small-files/small-file-9.txt
@@ -0,0 +1 @@
+I am small file 9
diff --git a/tests/benchmarks/many-small-files/small-file-90.txt b/tests/benchmarks/many-small-files/small-file-90.txt
@@ -0,0 +1 @@
+I am small file 90
diff --git a/tests/benchmarks/many-small-files/small-file-91.txt b/tests/benchmarks/many-small-files/small-file-91.txt
@@ -0,0 +1 @@
+I am small file 91
diff --git a/tests/benchmarks/many-small-files/small-file-92.txt b/tests/benchmarks/many-small-files/small-file-92.txt
@@ -0,0 +1 @@
+I am small file 92
diff --git a/tests/benchmarks/many-small-files/small-file-93.txt b/tests/benchmarks/many-small-files/small-file-93.txt
@@ -0,0 +1 @@
+I am small file 93
diff --git a/tests/benchmarks/many-small-files/small-file-94.txt b/tests/benchmarks/many-small-files/small-file-94.txt
@@ -0,0 +1 @@
+I am small file 94
diff --git a/tests/benchmarks/many-small-files/small-file-95.txt b/tests/benchmarks/many-small-files/small-file-95.txt
@@ -0,0 +1 @@
+I am small file 95
diff --git a/tests/benchmarks/many-small-files/small-file-96.txt b/tests/benchmarks/many-small-files/small-file-96.txt
@@ -0,0 +1 @@
+I am small file 96
diff --git a/tests/benchmarks/many-small-files/small-file-97.txt b/tests/benchmarks/many-small-files/small-file-97.txt
@@ -0,0 +1 @@
+I am small file 97
diff --git a/tests/benchmarks/many-small-files/small-file-98.txt b/tests/benchmarks/many-small-files/small-file-98.txt
@@ -0,0 +1 @@
+I am small file 98
diff --git a/tests/benchmarks/many-small-files/small-file-99.txt b/tests/benchmarks/many-small-files/small-file-99.txt
@@ -0,0 +1 @@
+I am small file 99
diff --git a/tests/benchmarks/run-benchmarks.sh b/tests/benchmarks/run-benchmarks.sh
@@ -136,3 +136,13 @@ for wrap in character never; do
 		cat "$RESULT_DIR/syntax-highlighting-speed-${filename}.md" >> "$REPORT"
 	done
 done
+
+
+heading "Many small files speed (overhead of metadata)"
+hyperfine \
+	"$(printf "%q" "$BAT") --no-config --language=txt --style=plain many-small-files/*.txt" \
+	--command-name 'bat … --language=txt *.txt' \
+	--warmup "$WARMUP_COUNT" \
+    --export-markdown "$RESULT_DIR/many-small-files-speed.md" \
+    --export-json "$RESULT_DIR/many-small-files-speed.json"
+cat "$RESULT_DIR/many-small-files-speed.md" >> "$REPORT"