/
scroll.go
70 lines (63 loc) · 1.38 KB
/
scroll.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
package main
import (
"context"
"encoding/json"
"github.com/olivere/elastic"
"io"
"os"
)
func doScroll(client *elastic.Client) error {
ctx := context.Background()
enc := json.NewEncoder(os.Stdout)
scroll := client.Scroll(*esIndex).Size(*scrollSize)
if *queryStr != "" {
scroll = scroll.Query(elastic.NewQueryStringQuery(*queryStr))
} else if *queryFile != nil {
defer (*queryFile).Close()
dec := json.NewDecoder(*queryFile)
var body interface{}
err := dec.Decode(&body)
if err != nil {
return err
}
scroll = scroll.Body(body)
}
if ProgressBar.IsEnabled() {
countCtx, countCancel := context.WithTimeout(ctx, *esTimeout)
defer countCancel()
total, err := client.Count(*esIndex).Do(countCtx)
if err != nil {
return err
}
ProgressBar.SetTotal(int(total))
}
for {
scrollCtx, scrollCancel := context.WithTimeout(ctx, *scrollTimeout)
results, err := scroll.Do(scrollCtx)
if err == io.EOF {
scrollCancel()
break
}
if err != nil {
scrollCancel()
return err
}
for _, hit := range results.Hits.Hits {
var source map[string]interface{}
err := json.Unmarshal(*hit.Source, &source)
if err != nil {
return err
}
source["_index"] = hit.Index
source["_type"] = hit.Type
source["_id"] = hit.Id
err = enc.Encode(source)
if err != nil {
return err
}
ProgressBar.Increment()
}
scrollCancel()
}
return nil
}