-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.go
370 lines (350 loc) · 10.7 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
package main
import (
"flag"
"fmt"
"math/rand"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
)
var (
version [3]int = [3]int{0, 4, 7}
optNumSet uint
optNumReader uint
optNumWriter uint
optNumRepeat int
optTimeMinute uint
optTimeSecond uint
optMonitorIntMinute uint
optMonitorIntSecond uint
optStatOnly bool
optIgnoreDot bool
optFollowSymlink bool
optReadBufferSize uint
optReadSize int
optWriteBufferSize uint
optWriteSize int
optRandomWriteData bool
optNumWritePaths int
optTruncateWritePaths bool
optFsyncWritePaths bool
optDirsyncWritePaths bool
optKeepWritePaths bool
optCleanWritePaths bool
optWritePathsBase string
optWritePathsType []fileType
optPathIter uint
optFlistFile string
optFlistFileCreate bool
optForce bool
optVerbose bool
optDebug bool
)
func getVersionString() string {
return fmt.Sprintf("%d.%d.%d", version[0], version[1], version[2])
}
func printVersion() {
fmt.Println(getVersionString())
}
func usage(progname string) {
fmt.Fprintln(os.Stderr, "usage: "+progname+": [<options>] <paths>")
flag.PrintDefaults()
}
func main() {
progname := path.Base(os.Args[0])
opt_num_set := flag.Int("num_set", 1, "Number of sets to run")
opt_num_reader := flag.Int("num_reader", 0, "Number of reader Goroutines")
opt_num_writer := flag.Int("num_writer", 0, "Number of writer Goroutines")
opt_num_repeat := flag.Int("num_repeat", -1, "Exit Goroutines after specified iterations if > 0")
opt_time_minute := flag.Int("time_minute", 0, "Exit Goroutines after sum of this and -time_second option if > 0")
opt_time_second := flag.Int("time_second", 0, "Exit Goroutines after sum of this and -time_minute option if > 0")
opt_monitor_int_minute := flag.Int("monitor_interval_minute", 0, "Monitor Goroutines every sum of this and -monitor_interval_second option if > 0")
opt_monitor_int_second := flag.Int("monitor_interval_second", 0, "Monitor Goroutines every sum of this and -monitor_interval_minute option if > 0")
opt_stat_only := flag.Bool("stat_only", false, "Do not read file data")
opt_ignore_dot := flag.Bool("ignore_dot", false, "Ignore entries start with .")
opt_follow_symlink := flag.Bool("follow_symlink", false, "Follow symbolic links for read unless directory")
opt_read_buffer_size := flag.Int("read_buffer_size", 1<<16, "Read buffer size")
opt_read_size := flag.Int("read_size", -1, "Read residual size per file read, use < read_buffer_size random size if 0")
opt_write_buffer_size := flag.Int("write_buffer_size", 1<<16, "Write buffer size")
opt_write_size := flag.Int("write_size", -1, "Write residual size per file write, use < write_buffer_size random size if 0")
opt_random_write_data := flag.Bool("random_write_data", false, "Use pseudo random write data")
opt_num_write_paths := flag.Int("num_write_paths", 1<<10, "Exit writer Goroutines after creating specified files or directories if > 0")
opt_truncate_write_paths := flag.Bool("truncate_write_paths", false, "ftruncate(2) write paths for regular files instead of write(2)")
opt_fsync_write_paths := flag.Bool("fsync_write_paths", false, "fsync(2) write paths")
opt_dirsync_write_paths := flag.Bool("dirsync_write_paths", false, "fsync(2) parent directories of write paths")
opt_keep_write_paths := flag.Bool("keep_write_paths", false, "Do not unlink write paths after writer Goroutines exit")
opt_clean_write_paths := flag.Bool("clean_write_paths", false, "Unlink existing write paths and exit")
opt_write_paths_base := flag.String("write_paths_base", "x", "Base name for write paths")
opt_write_paths_type := flag.String("write_paths_type", "dr", "File types for write paths [d|r|s|l]")
opt_path_iter := flag.String("path_iter", "ordered", "<paths> iteration type [walk|ordered|reverse|random]")
opt_flist_file := flag.String("flist_file", "", "Path to flist file")
opt_flist_file_create := flag.Bool("flist_file_create", false, "Create flist file and exit")
opt_force := flag.Bool("force", false, "Enable force mode")
opt_verbose := flag.Bool("verbose", false, "Enable verbose print")
opt_debug := flag.Bool("debug", false, "Create debug log file under home directory")
opt_version := flag.Bool("v", false, "Print version and exit")
opt_help_h := flag.Bool("h", false, "Print usage and exit")
flag.Parse()
args := flag.Args()
optNumSet = uint(*opt_num_set)
optNumReader = uint(*opt_num_reader)
optNumWriter = uint(*opt_num_writer)
optNumRepeat = *opt_num_repeat
if optNumRepeat == 0 || optNumRepeat < -1 {
optNumRepeat = -1
}
optTimeMinute = uint(*opt_time_minute)
optTimeSecond = uint(*opt_time_second)
optTimeSecond += optTimeMinute * 60
optTimeMinute = 0
optMonitorIntMinute = uint(*opt_monitor_int_minute)
optMonitorIntSecond = uint(*opt_monitor_int_second)
optMonitorIntSecond += optMonitorIntMinute * 60
optMonitorIntMinute = 0
optStatOnly = *opt_stat_only
optIgnoreDot = *opt_ignore_dot
optFollowSymlink = *opt_follow_symlink
optReadBufferSize = uint(*opt_read_buffer_size)
if optReadBufferSize > maxBufferSize {
fmt.Println("Invalid read buffer size", optReadBufferSize)
os.Exit(1)
}
optReadSize = *opt_read_size
if optReadSize < -1 {
optReadSize = -1
} else if optReadSize > int(maxBufferSize) {
fmt.Println("Invalid read size", optReadSize)
os.Exit(1)
}
optWriteBufferSize = uint(*opt_write_buffer_size)
if optWriteBufferSize > maxBufferSize {
fmt.Println("Invalid write buffer size", optWriteBufferSize)
os.Exit(1)
}
optWriteSize = *opt_write_size
if optWriteSize < -1 {
optWriteSize = -1
} else if optWriteSize > int(maxBufferSize) {
fmt.Println("Invalid write size", optWriteSize)
os.Exit(1)
}
optRandomWriteData = *opt_random_write_data
optNumWritePaths = *opt_num_write_paths
if optNumWritePaths < -1 {
optNumWritePaths = -1
}
optTruncateWritePaths = *opt_truncate_write_paths
optFsyncWritePaths = *opt_fsync_write_paths
optDirsyncWritePaths = *opt_dirsync_write_paths
optKeepWritePaths = *opt_keep_write_paths
optCleanWritePaths = *opt_clean_write_paths
optWritePathsBase = *opt_write_paths_base
if len(optWritePathsBase) == 0 {
fmt.Println("Empty write paths base")
os.Exit(1)
}
if n, err := strconv.Atoi(optWritePathsBase); err == nil {
optWritePathsBase = strings.Repeat("x", n)
fmt.Println("Using base name", optWritePathsBase, "for write paths")
}
if s := *opt_write_paths_type; len(s) == 0 {
fmt.Println("Empty write paths type")
os.Exit(1)
} else {
optWritePathsType = make([]fileType, len(s))
for i, x := range s {
var t fileType
switch x {
case 'd':
t = DIR
case 'r':
t = REG
case 's':
t = SYMLINK
case 'l':
t = LINK
default:
fmt.Println("Invalid write paths type", string(x))
os.Exit(1)
}
optWritePathsType[i] = t
}
}
switch *opt_path_iter {
case "walk":
optPathIter = PATH_ITER_WALK
case "ordered":
optPathIter = PATH_ITER_ORDERED
case "reverse":
optPathIter = PATH_ITER_REVERSE
case "random":
optPathIter = PATH_ITER_RANDOM
default:
fmt.Println("Invalid path iteration type", *opt_path_iter)
os.Exit(1)
}
optFlistFile = *opt_flist_file
// using flist file means not walking input directories
if len(optFlistFile) != 0 && optPathIter == PATH_ITER_WALK {
optPathIter = PATH_ITER_ORDERED
fmt.Println("Using flist, force -path_iter=ordered")
}
optFlistFileCreate = *opt_flist_file_create
optForce = *opt_force
optVerbose = *opt_verbose
optDebug = *opt_debug
if *opt_version {
printVersion()
os.Exit(1)
}
if *opt_help_h {
usage(progname)
os.Exit(1)
}
if len(args) < 1 {
usage(progname)
os.Exit(1)
}
if isWindows() {
fmt.Println("Windows unsupported")
os.Exit(1)
}
if s := getPathSeparator(); s != "/" {
fmt.Println("Invalid path separator", s)
os.Exit(1)
}
defer cleanupLock()
initLock()
defer cleanupLog()
if err := initLog(progname); err != nil {
fmt.Println(err)
os.Exit(1)
}
dbg(os.Args)
flag.VisitAll(func(f *flag.Flag) {
dbgf("option \"%s\" -> %s\n", f.Name, f.Value)
})
// only allow directories since now that write is supported
var input []string
for _, f := range args {
absf, err := filepath.Abs(f)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
assert(!strings.HasSuffix(absf, "/"))
if t, err := getRawFileType(absf); err != nil {
fmt.Println(err)
os.Exit(1)
} else if t != DIR {
fmt.Println(absf, "not directory")
os.Exit(1)
}
if !optForce {
count := 0
for _, x := range absf {
if x == '/' {
count++
}
}
// /path/to/dir is allowed, but /path/to is not
if count < 3 {
fmt.Println(absf, "not allowed, use -force option to proceed")
os.Exit(1)
}
}
input = append(input, absf)
}
dbg("input", input)
// and the directories should be writable
if optDebug && optNumWriter > 0 {
for _, f := range input {
if writable, err := isDirWritable(f); err != nil {
fmt.Println(err)
os.Exit(1)
} else {
dbgf("%s writable %t", f, writable)
}
}
}
// create flist and exit
if optFlistFileCreate {
if len(optFlistFile) == 0 {
fmt.Println("Empty flist file path")
os.Exit(1)
}
if err := createFlistFile(input, optFlistFile, optIgnoreDot, optForce); err != nil {
fmt.Println(err)
os.Exit(1)
}
if info, err := os.Stat(optFlistFile); err != nil {
fmt.Println(err)
os.Exit(1)
} else {
fmt.Printf("%+v\n", info)
}
os.Exit(0)
}
// clean write paths and exit
if optCleanWritePaths {
if l, err := collectWritePaths(input); err != nil {
fmt.Println(err)
os.Exit(1)
} else if rl, err := unlinkWritePaths(l, -1); err != nil {
fmt.Println(err)
os.Exit(1)
} else {
fmt.Println("Unlinked", len(l)-len(rl), "/", len(l), "write paths")
if len(rl) != 0 {
fmt.Println(len(rl), "/", len(l), "write paths remaining")
os.Exit(1)
}
}
os.Exit(0)
}
// ready to dispatch workers
for i := uint(0); i < optNumSet; i++ {
if optNumSet != 1 {
fmt.Println(strings.Repeat("=", 80))
s := fmt.Sprintf("Set %d/%d", i+1, optNumSet)
fmt.Println(s)
dbg(s)
}
rand.Seed(time.Now().UnixNano())
_, num_interrupted, num_error, num_remain, tsv, err := dispatchWorker(input)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
if num_interrupted > 0 {
var s string
if num_interrupted > 1 {
s = "s"
}
fmt.Printf("%d worker%s interrupted\n", num_interrupted, s)
}
if num_error > 0 {
var s string
if num_error > 1 {
s = "s"
}
fmt.Printf("%d worker%s failed\n", num_error, s)
}
if num_remain > 0 {
var s string
if num_remain > 1 {
s = "s"
}
fmt.Printf("%d write path%s remaining\n", num_remain, s)
}
printStat(tsv)
if num_interrupted > 0 {
break
} else if optNumSet != 1 && i != optNumSet-1 {
fmt.Println()
}
}
}