-
Notifications
You must be signed in to change notification settings - Fork 23
/
export.go
134 lines (115 loc) · 3.21 KB
/
export.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
package client
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
)
type ExportError struct {
Details string
}
func (e ExportError) Error() string {
return fmt.Sprintf("failed to complete export: %s", e.Details)
}
type exportType string
const exportType_Event exportType = "TYPE_EVENT"
type exportFormat string
const exportFormat_Json exportFormat = "FORMAT_JSON"
type timeRange struct {
Start string `json:"start"`
End string `json:"end"`
}
type createParams struct {
SegmentId string `json:"segmentId"`
Type exportType `json:"type"`
Format exportFormat `json:"format"`
SegmentTimeRange timeRange `json:"segmentTimeRange"`
TimeRange timeRange `json:"timeRange"`
Fields []string `json:"fields"`
}
type createSegmentResponse struct {
OperationId string `json:"operationId"`
}
type getExportResultsResponse struct {
Location string `json:"location"`
}
func (c *Client) CreateExport(start time.Time, end time.Time, fields []string) (string, error) {
params := createParams{
SegmentId: c.Config.SegmentId,
Type: exportType_Event,
Format: exportFormat_Json,
// Specify a segment time range with empty values to indicate "All Time"
SegmentTimeRange: timeRange{Start: "", End: ""},
// Limit the exported data to the requested time range.
TimeRange: timeRange{
Start: start.UTC().Format(time.RFC3339),
End: end.UTC().Format(time.RFC3339),
},
Fields: fields,
}
reqBody, err := json.Marshal(params)
if err != nil {
return "", err
}
url := fmt.Sprintf("%s/segments/v1/exports", c.Config.ApiURL)
req, err := http.NewRequest("POST", url, bytes.NewBuffer(reqBody))
if err != nil {
return "", err
}
if c.createRequestModifier != nil {
c.createRequestModifier(req)
}
resBody, err := c.doReq(req)
if err != nil {
// Status 429, 499, 500 -- retry
// Status else fail
return "", err
}
defer resBody.Close()
resp := createSegmentResponse{}
err = json.NewDecoder(resBody).Decode(&resp)
return resp.OperationId, err
}
func (c *Client) GetExportProgress(operationId string) (int, string, error) {
resp, err := c.getExportOperation(operationId)
if err != nil {
return 0, "", err
}
if resp.State == operationComplete {
return resp.EstimatedPctComplete, resp.Results.SearchExportId, nil
}
return resp.EstimatedPctComplete, "", nil
}
func (c *Client) GetExport(exportId string) (io.ReadCloser, error) {
url := fmt.Sprintf("%s/search/v1/exports/%s/results", c.Config.ApiURL, exportId)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
body, err := c.doReq(req)
if err != nil {
return nil, err
}
defer body.Close()
rsp := &getExportResultsResponse{}
if err := json.NewDecoder(body).Decode(&rsp); err != nil {
return nil, err
}
// Use a vanilla http client for downloading the stream since auth
// is built into the URL itself.
streamRsp, err := http.Get(rsp.Location)
if err != nil {
return nil, err
}
if streamRsp.StatusCode != http.StatusOK {
return nil, StatusError{
Status: streamRsp.Status,
StatusCode: streamRsp.StatusCode,
RetryAfter: time.Duration(getRetryAfter(streamRsp)) * time.Second,
Body: nil,
}
}
return streamRsp.Body, nil
}