/
scheduledsearch.go
369 lines (323 loc) · 9.76 KB
/
scheduledsearch.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
/*************************************************************************
* Copyright 2021 Gravwell, Inc. All rights reserved.
* Contact: <legal@gravwell.io>
*
* This software may be modified and distributed under the terms of the
* BSD 2-clause license. See the LICENSE file for details.
**************************************************************************/
package types
import (
"bytes"
"errors"
"fmt"
"strings"
"time"
"github.com/google/uuid"
)
const (
ScriptAnko ScriptLang = 0 // default is anko
ScriptGo ScriptLang = 1 // new hotness is go
ScheduledTypeSearch string = "search"
ScheduledTypeScript string = "script"
ScheduledTypeFlow string = "flow"
SEQ_NODE_NOT_EXECUTED = 9999999
)
type ScriptLang uint
type ScriptDeployConfig struct {
Disabled bool
RunImmediately bool
}
var (
ErrUnknownScriptLanguage = errors.New("Unknown script language")
)
// ScheduledSearch represents a scheduled search, including rules, description,
// etc.
type ScheduledSearch struct {
Synced bool
ID int32
GUID uuid.UUID
Groups []int32
Global bool
WriteAccess Access
Name string // the name of this scheduled search
Description string // freeform description
Labels []string
Owner int32 // uid of owner
Schedule string // when to run: a cron spec
Timezone string // a location to use for the timezone, e.g. "America/New_York"
Updated time.Time
Disabled bool
// These values are used for debug/testing runs
OneShot bool // Set this flag to 'true' to make the search fire ONCE
DebugMode bool // set this to true to enable debug mode
DebugEvent *Event // If provided, this will be inserted as `event` into the flow payload.
// if true, search agent will attempt to "backfill" missed runs since
// the more recent of Updated or LastRun.
BackfillEnabled bool
// This sets what kind of scheduled "thing" it is: search, script, or flow
ScheduledType string
// Fields for scheduled searches
SearchReference uuid.UUID // A reference to a saved query item by UUID. If SearchString is populated on a GET, it represents the query referenced by SearchReference.
SearchString string // The actual search to run. If SearchReference is populated on a GET, SearchString represents the query referenced by SearchReference.
Duration int64 // How many seconds back to search, MUST BE NEGATIVE
SearchSinceLastRun bool // If set, ignore Duration and run from last run time to now.
TimeframeOffset int64 // How many seconds to offset the search timeframe, MUST BE NEGATIVE.
// For scheduled scripts
Script string // If set, execute the contents rather than running SearchString
ScriptLanguage ScriptLang // what script type is this: anko, go
// For scheduled flows
Flow string // The flow specification itself
FlowNodeResults map[int]FlowNodeResult // results for each node in the flow
// These fields are updated by the search agent after it runs a search
PersistentMaps map[string]map[string]interface{}
LastRun time.Time
LastRunDuration time.Duration // how many nanoseconds did it take
LastSearchIDs []string // the IDs of the most recently performed searches
LastError string // any error from the last run of the scheduled search
ErrorHistory []ScheduledError // a list of previously-occurring errors
DebugOutput []byte // output of the script if debugmode was enabled
}
type ScheduledError struct {
Error string
Timestamp time.Time
}
type FlowNodeResult struct {
Payload map[string]interface{}
ID int // the node ID
Type string // the type of node, e.g. RunQuery
Log string
Error string
Start int64 // unix nanoseconds
End int64 // unix nanoseconds
// The first node executed has sequence 0, the next is sequence 1, etc.
// Nodes which were not executed have Sequence = SEQ_NODE_NOT_EXECUTED
Sequence int
}
type ScheduledSearchParseRequest struct {
Version int
Script string
}
type ScheduledSearchParseResponse struct {
OK bool
Error string `json:",omitempty"`
ErrorLine int
ErrorColumn int
}
type FlowParseRequest struct {
Flow string
DebugEvent *Event // If provided, this will be set as `event` in the flow payload for parsing.
}
type FlowParseResponse struct {
OK bool
// Error and ErrorNode are now deprecated; look at the Failures map
// to see if there were parse problems. They are retained for compatibility.
Error string `json:",omitempty"`
ErrorNode int // the node which failed to parse (ignore if Error is empty)
OutputPayloads map[int]map[string]interface{}
InitialPayload map[string]interface{} // the payload which gets passed to nodes with no dependencies
Failures map[int]NodeParseFailure
}
// NodeParseFailure represents all problems encountered during a node's Parse phase
type NodeParseFailure struct {
Errors []NodeParseError
}
// Error returns an error string for the NodeParseFailure. It just returns the first error
// if there are multiple errors; to handle it better, walk the Errors array yourself.
func (f NodeParseFailure) Error() string {
if len(f.Errors) > 0 {
// just print the first error
return f.Errors[0].String()
}
return ""
}
// AddError registers a new error. It can take regular errors, NodeParseError, or NodeParseFailure.
func (f *NodeParseFailure) AddError(e error) {
if e == nil {
return
}
switch t := e.(type) {
case NodeParseError:
f.Errors = append(f.Errors, t)
case *NodeParseError:
f.Errors = append(f.Errors, *t)
case NodeParseFailure:
f.Errors = append(f.Errors, t.Errors...)
case *NodeParseFailure:
f.Errors = append(f.Errors, t.Errors...)
default:
f.Errors = append(f.Errors, NodeParseError{Err: e.Error()})
}
}
// ErrCount returns the number of errors registered.
func (f *NodeParseFailure) ErrCount() int {
return len(f.Errors)
}
// NodeParseError represents a single problem encountered during the parse phase,
// e.g. an un-set config field. Field represents which config field, if any, was
// the source of the problem; if unset, the error was of a more general nature.
type NodeParseError struct {
Err string
Field string `json:",omitempty"`
}
func (f NodeParseError) Error() string {
return f.String()
}
func (f NodeParseError) String() string {
if f.Field != "" {
return fmt.Sprintf("%v: %v", f.Field, f.Err)
}
return f.Err
}
func (ss *ScheduledSearch) TypeName() string {
// if the type is set, use that
if ss.ScheduledType != "" {
return ss.ScheduledType
}
if len(ss.Script) > 0 {
return "script"
}
return "search"
}
func (ss *ScheduledSearch) Dedup() {
gidmap := make(map[int32]bool)
for _, gid := range ss.Groups {
gidmap[gid] = true
}
var newgids []int32
for gid, val := range gidmap {
if val {
newgids = append(newgids, gid)
}
}
ss.Groups = newgids
}
type UserMailConfig struct {
Server string
Port int
Username string
Password string
UseTLS bool
InsecureSkipVerify bool
}
type UserMail struct {
From string
To []string
Cc []string
Bcc []string
Subject string
Body string
Attachments []UserMailAttachment
}
func (um UserMail) Validate() error {
if um.From == `` {
return errors.New("Missing from")
}
if len(um.To) == 0 {
return errors.New("no recepients")
}
for _, v := range um.To {
if v == `` {
return errors.New("Invalid recepient")
}
}
for _, v := range um.Attachments {
if err := v.Validate(); err != nil {
return err
}
}
return nil
}
type UserMailAttachment struct {
Name string
Content []byte
}
func (uma UserMailAttachment) Validate() error {
if uma.Name == `` {
return errors.New("Invalid attachment name")
}
return nil
}
func (s ScheduledSearch) Equal(v ScheduledSearch) bool {
if s.ID != v.ID || s.GUID != v.GUID || s.Name != v.Name ||
s.Description != v.Description || s.Owner != v.Owner ||
s.Schedule != v.Schedule || s.Timezone != v.Timezone ||
s.Disabled != v.Disabled || s.OneShot != v.OneShot ||
s.Global != v.Global || s.DebugMode != v.DebugMode {
return false
}
if s.SearchString != v.SearchString ||
s.Duration != v.Duration ||
s.SearchSinceLastRun != v.SearchSinceLastRun ||
s.Script != v.Script {
return false
}
if len(s.Groups) != len(v.Groups) || len(s.Labels) != len(v.Labels) || len(s.LastSearchIDs) != len(v.LastSearchIDs) {
return false
}
for i, g := range s.Groups {
if v.Groups[i] != g {
return false
}
}
for i, l := range s.Labels {
if l != v.Labels[i] {
return false
}
}
for i, id := range s.LastSearchIDs {
if id != v.LastSearchIDs[i] {
return false
}
}
if s.LastRun != v.LastRun || s.LastRunDuration != v.LastRunDuration || s.LastError != v.LastError {
return false
}
if !bytes.Equal(v.DebugOutput, s.DebugOutput) {
return false
}
if (s.PersistentMaps == nil) != (v.PersistentMaps == nil) {
return false
} else if s.PersistentMaps == nil {
return true //both are nil
}
//just check the first level of keys
for k, val := range s.PersistentMaps {
if vv, ok := v.PersistentMaps[k]; !ok {
return false
} else if len(val) != len(vv) {
return false
}
}
return true
}
const scheduledScriptAnko string = `anko`
const scheduledScriptGo string = `go`
func (sl ScriptLang) String() string {
switch sl {
case ScriptAnko:
return scheduledScriptAnko
case ScriptGo:
return scheduledScriptGo
}
return `UNKNOWN`
}
func (sl ScriptLang) Valid() (err error) {
switch sl {
case ScriptAnko:
case ScriptGo:
default:
err = ErrUnknownScriptLanguage
}
return
}
func ParseScriptLang(v string) (l ScriptLang, err error) {
switch strings.TrimSpace(strings.ToLower(v)) {
case scheduledScriptAnko:
l = ScriptAnko
case scheduledScriptGo:
l = ScriptGo
default:
err = ErrUnknownScriptLanguage
}
return
}