-
Notifications
You must be signed in to change notification settings - Fork 55
/
util.go
295 lines (268 loc) · 9.33 KB
/
util.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
// Package transform package contains canonical implementations of Kazaam transforms.
package transform
import (
"bytes"
"fmt"
"regexp"
"strconv"
"strings"
"github.com/qntfy/jsonparser"
)
// ParseError should be thrown when there is an issue with parsing any of the specification or data
type ParseError string
func (p ParseError) Error() string {
return string(p)
}
// RequireError should be thrown if a required key is missing in the data
type RequireError string
func (r RequireError) Error() string {
return string(r)
}
// SpecError should be thrown if the spec for a transform is malformed
type SpecError string
func (s SpecError) Error() string {
return string(s)
}
// Config contains the options that dictate the behavior of a transform. The internal
// `spec` object can be an arbitrary json configuration for the transform.
type Config struct {
Spec *map[string]interface{} `json:"spec"`
Require bool `json:"require,omitempty"`
InPlace bool `json:"inplace,omitempty"`
KeySeparator string `json:"keySeparator"`
}
var (
NonExistentPath = RequireError("Path does not exist")
jsonPathRe = regexp.MustCompile("([^\\[\\]]+)\\[(.*?)\\]")
)
// Given a json byte slice `data` and a kazaam `path` string, return the object at the path in data if it exists.
func getJSONRaw(data []byte, path string, pathRequired bool, keySeparator string) ([]byte, error) {
objectKeys := strings.Split(path, keySeparator)
numOfInserts := 0
for element, k := range objectKeys {
// check the object key to see if it also contains an array reference
arrayRefs := jsonPathRe.FindAllStringSubmatch(k, -1)
if arrayRefs != nil && len(arrayRefs) > 0 {
objKey := arrayRefs[0][1] // the key
arrayKeyStr := arrayRefs[0][2] // the array index
err := validateArrayKeyString(arrayKeyStr)
if err != nil {
return nil, err
}
// if there's a wildcard array reference
if arrayKeyStr == "*" {
// ArrayEach setup
objectKeys[element+numOfInserts] = objKey
beforePath := objectKeys[:element+numOfInserts+1]
newPath := strings.Join(objectKeys[element+numOfInserts+1:], keySeparator)
var results [][]byte
// use jsonparser.ArrayEach to copy the array into results
_, err := jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
results = append(results, HandleUnquotedStrings(value, dataType))
}, beforePath...)
if err == jsonparser.KeyPathNotFoundError {
if pathRequired {
return nil, NonExistentPath
}
} else if err != nil {
return nil, err
}
// GetJSONRaw() the rest of path for each element in results
if newPath != "" {
for i, value := range results {
intermediate, err := getJSONRaw(value, newPath, pathRequired, keySeparator)
if err == jsonparser.KeyPathNotFoundError {
if pathRequired {
return nil, NonExistentPath
}
} else if err != nil {
return nil, err
}
results[i] = intermediate
}
}
// copy into raw []byte format and return
var buffer bytes.Buffer
buffer.WriteByte('[')
for i := 0; i < len(results)-1; i++ {
buffer.Write(results[i])
buffer.WriteByte(',')
}
if len(results) > 0 {
buffer.Write(results[len(results)-1])
}
buffer.WriteByte(']')
return buffer.Bytes(), nil
}
// separate the array key as a new element in objectKeys
objectKeys = makePathWithIndex(arrayKeyStr, objKey, objectKeys, element+numOfInserts)
numOfInserts++
} else {
// no array reference, good to go
continue
}
}
result, dataType, _, err := jsonparser.Get(data, objectKeys...)
// jsonparser strips quotes from Strings
if dataType == jsonparser.String {
// bookend() is destructive to underlying slice, need to copy.
// extra capacity saves an allocation and copy during bookend.
result = HandleUnquotedStrings(result, dataType)
}
if len(result) == 0 {
result = []byte("null")
}
if err == jsonparser.KeyPathNotFoundError {
if pathRequired {
return nil, NonExistentPath
}
} else if err != nil {
return nil, err
}
return result, nil
}
// setJSONRaw sets the value at a key and handles array indexing
func setJSONRaw(data, out []byte, path, keySeparator string) ([]byte, error) {
var err error
splitPath := strings.Split(path, keySeparator)
numOfInserts := 0
for element, k := range splitPath {
arrayRefs := jsonPathRe.FindAllStringSubmatch(k, -1)
if arrayRefs != nil && len(arrayRefs) > 0 {
objKey := arrayRefs[0][1] // the key
arrayKeyStr := arrayRefs[0][2] // the array index
err = validateArrayKeyString(arrayKeyStr)
if err != nil {
return nil, err
}
// Note: this branch of the function is not currently used by any
// existing transforms. It is simpy here to support he generalized
// form of this operation
if arrayKeyStr == "*" {
// ArrayEach setup
splitPath[element+numOfInserts] = objKey
beforePath := splitPath[:element+numOfInserts+1]
afterPath := strings.Join(splitPath[element+numOfInserts+1:], keySeparator)
// use jsonparser.ArrayEach to count the number of items in the
// array
var arraySize int
_, err = jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
arraySize++
}, beforePath...)
if err != nil {
return nil, err
}
// setJSONRaw() the rest of path for each element in results
for i := 0; i < arraySize; i++ {
var newPath string
// iterate through each item in the array by replacing the
// wildcard with an int and joining the path back together
newArrayKey := strings.Join([]string{"[", strconv.Itoa(i), "]"}, "")
beforePathStr := strings.Join(beforePath, keySeparator)
beforePathArrayKeyStr := strings.Join([]string{beforePathStr, newArrayKey}, "")
// if there's nothing that comes after the array index,
// don't join so that we avoid trailing cruft
if len(afterPath) > 0 {
newPath = strings.Join([]string{beforePathArrayKeyStr, afterPath}, keySeparator)
} else {
newPath = beforePathArrayKeyStr
}
// now call the function, but this time with an array index
// instead of a wildcard
data, err = setJSONRaw(data, out, newPath, keySeparator)
if err != nil {
return nil, err
}
}
return data, nil
}
// if not a wildcard then piece that path back together with the
// array index as an entry in the splitPath slice
splitPath = makePathWithIndex(arrayKeyStr, objKey, splitPath, element+numOfInserts)
numOfInserts++
} else {
continue
}
}
data, err = jsonparser.Set(data, out, splitPath...)
if err != nil {
return nil, err
}
return data, nil
}
// delJSONRaw deletes the value at a path and handles array indexing
func delJSONRaw(data []byte, path string, pathRequired bool, keySeparator string) ([]byte, error) {
var err error
splitPath := strings.Split(path, keySeparator)
numOfInserts := 0
for element, k := range splitPath {
arrayRefs := jsonPathRe.FindAllStringSubmatch(k, -1)
if arrayRefs != nil && len(arrayRefs) > 0 {
objKey := arrayRefs[0][1] // the key
arrayKeyStr := arrayRefs[0][2] // the array index
err = validateArrayKeyString(arrayKeyStr)
if err != nil {
return nil, err
}
// not currently supported
if arrayKeyStr == "*" {
return nil, SpecError("Array wildcard not supported for this operation.")
}
// if not a wildcard then piece that path back together with the
// array index as an entry in the splitPath slice
splitPath = makePathWithIndex(arrayKeyStr, objKey, splitPath, element+numOfInserts)
numOfInserts++
} else {
// no array reference, good to go
continue
}
}
if pathRequired {
_, _, _, err = jsonparser.Get(data, splitPath...)
if err == jsonparser.KeyPathNotFoundError {
return nil, NonExistentPath
} else if err != nil {
return nil, err
}
}
data = jsonparser.Delete(data, splitPath...)
return data, nil
}
// validateArrayKeyString is a helper function to make sure the array index is
// legal
func validateArrayKeyString(arrayKeyStr string) error {
if arrayKeyStr != "*" && arrayKeyStr != "+" && arrayKeyStr != "-" {
val, err := strconv.Atoi(arrayKeyStr)
if val < 0 || err != nil {
return ParseError(fmt.Sprintf("Warn: Unable to coerce index to integer: %v", arrayKeyStr))
}
}
return nil
}
// makePathWithIndex generats a path slice to pass to jsonparser
func makePathWithIndex(arrayKeyStr, objectKey string, pathSlice []string, pathIndex int) []string {
arrayKey := string(bookend([]byte(arrayKeyStr), '[', ']'))
pathSlice[pathIndex] = objectKey
pathSlice = append(pathSlice, "")
copy(pathSlice[pathIndex+2:], pathSlice[pathIndex+1:])
pathSlice[pathIndex+1] = arrayKey
return pathSlice
}
// add characters at beginning and end of []byte
func bookend(value []byte, bef, aft byte) []byte {
value = append(value, ' ', aft)
copy(value[1:], value[:len(value)-2])
value[0] = bef
return value
}
// jsonparser strips quotes from returned strings, this adds them back
func HandleUnquotedStrings(value []byte, dt jsonparser.ValueType) []byte {
if dt == jsonparser.String {
// bookend() is destructive to underlying slice, need to copy.
// extra capacity saves an allocation and copy during bookend.
tmp := make([]byte, len(value), len(value)+2)
copy(tmp, value)
value = bookend(tmp, '"', '"')
}
return value
}