-
Notifications
You must be signed in to change notification settings - Fork 67
/
flags.go
79 lines (71 loc) · 2.13 KB
/
flags.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
package inputflags
import (
"context"
"errors"
"flag"
"fmt"
"os"
"github.com/brimdata/zed"
"github.com/brimdata/zed/cli/auto"
"github.com/brimdata/zed/pkg/storage"
"github.com/brimdata/zed/zio"
"github.com/brimdata/zed/zio/anyio"
"github.com/brimdata/zed/zio/zngio"
)
type Flags struct {
anyio.ReaderOpts
ReadMax auto.Bytes
ReadSize auto.Bytes
Threads int
}
func (f *Flags) Options() anyio.ReaderOpts {
return f.ReaderOpts
}
func (f *Flags) SetFlags(fs *flag.FlagSet, validate bool) {
fs.StringVar(&f.Format, "i", "auto", "format of input data [auto,arrows,csv,json,line,parquet,vng,zeek,zjson,zng,zson]")
f.CSV.Delim = ','
fs.Func("csv.delim", `CSV field delimiter (default ",")`, func(s string) error {
if len(s) != 1 {
return errors.New("CSV field delimiter must be exactly one character")
}
f.CSV.Delim = rune(s[0])
return nil
})
fs.BoolVar(&f.ZNG.Validate, "zng.validate", validate, "validate format when reading ZNG")
fs.IntVar(&f.ZNG.Threads, "zng.threads", 0, "number of ZNG read threads (0=GOMAXPROCS)")
f.ReadMax = auto.NewBytes(zngio.MaxSize)
fs.Var(&f.ReadMax, "zng.readmax", "maximum ZNG read buffer size in MiB, MB, etc.")
f.ReadSize = auto.NewBytes(zngio.ReadSize)
fs.Var(&f.ReadSize, "zng.readsize", "target ZNG read buffer size in MiB, MB, etc.")
}
// Init is called after flags have been parsed.
func (f *Flags) Init() error {
f.ZNG.Max = int(f.ReadMax.Bytes)
if f.ZNG.Max < 0 {
return errors.New("max read buffer size must be greater than zero")
}
f.ZNG.Size = int(f.ReadSize.Bytes)
if f.ZNG.Size < 0 {
return errors.New("target read buffer size must be greater than zero")
}
return nil
}
func (f *Flags) Open(ctx context.Context, zctx *zed.Context, engine storage.Engine, paths []string, stopOnErr bool) ([]zio.Reader, error) {
var readers []zio.Reader
for _, path := range paths {
if path == "-" {
path = "stdio:stdin"
}
file, err := anyio.Open(ctx, zctx, engine, path, f.ReaderOpts)
if err != nil {
err = fmt.Errorf("%s: %w", path, err)
if stopOnErr {
return nil, err
}
fmt.Fprintln(os.Stderr, err)
continue
}
readers = append(readers, file)
}
return readers, nil
}