Skip to content

Commit

Permalink
Merge 596b83c into d3a8cbb
Browse files Browse the repository at this point in the history
  • Loading branch information
@henry-megarry
henry-megarry committed Jul 17, 2019
2 parents d3a8cbb + 596b83c commit be7b380
Show file tree
Hide file tree
Showing 12 changed files with 140 additions and 35 deletions.
47 changes: 47 additions & 0 deletions kazaam_int_test.go
View file
Expand Up @@ -168,6 +168,53 @@ func TestKazaamCoalesceTransformAndShift(t *testing.T) {
}
}

func TestKazaamCoalesceTransformAndShiftWithKeySeparator(t *testing.T) {
spec := `[{
"operation": "coalesce",
"spec": {"foo": ["rating.foo", "rating.primary"]},
"keySeparator": "."
}, {
"operation": "shift",
"spec": {"rating.foo": "foo", "rating.example.value": "rating.primary.value"},
"keySeparator": "."
}]`
jsonOut := `{"rating":{"foo":{"value":3},"example":{"value":3}}}`

kazaamTransform, _ := kazaam.NewKazaam(spec)
kazaamOut, _ := kazaamTransform.TransformJSONStringToString(testJSONInput)
areEqual, _ := checkJSONStringsEqual(kazaamOut, jsonOut)

if !areEqual {
t.Error("Transformed data does not match expectation.")
t.Log("Expected: ", jsonOut)
t.Log("Actual: ", kazaamOut)
t.FailNow()
}
}

func TestKazaamCoalesceTransformAndShiftWithKeySeparatorNonDefault(t *testing.T) {
spec := `[{
"operation": "coalesce",
"spec": {"foo": ["rating>foo", "rating>primary"]},
"keySeparator": ">"
}, {
"operation": "shift",
"spec": {"rating.foo": "foo", "rating.example.value": "rating.primary.value"}
}]`
jsonOut := `{"rating":{"foo":{"value":3},"example":{"value":3}}}`

kazaamTransform, _ := kazaam.NewKazaam(spec)
kazaamOut, _ := kazaamTransform.TransformJSONStringToString(testJSONInput)
areEqual, _ := checkJSONStringsEqual(kazaamOut, jsonOut)

if !areEqual {
t.Error("Transformed data does not match expectation.")
t.Log("Expected: ", jsonOut)
t.Log("Actual: ", kazaamOut)
t.FailNow()
}
}

func TestKazaamShiftTransformWithTimestamp(t *testing.T) {
spec := `[{
"operation": "shift",
Expand Down
3 changes: 3 additions & 0 deletions spec.go
View file
Expand Up @@ -31,6 +31,9 @@ func (s *spec) UnmarshalJSON(b []byte) (err error) {
err = &Error{ErrMsg: "Spec must contain at least one element", ErrType: SpecError}
return
}
if s.Config != nil && s.KeySeparator == "" {
s.KeySeparator = "."
}
return
}
return
Expand Down
4 changes: 2 additions & 2 deletions transform/coalesce.go
View file
Expand Up @@ -60,12 +60,12 @@ func Coalesce(spec *Config, data []byte) ([]byte, error) {
var err error

// grab the data
dataForV, err = getJSONRaw(data, v, false)
dataForV, err = getJSONRaw(data, v, false, spec.KeySeparator)
if err != nil {
return nil, err
}
if !inArray(dataForV, ignoreSlice) {
data, err = setJSONRaw(data, dataForV, k)
data, err = setJSONRaw(data, dataForV, k, spec.KeySeparator)
if err != nil {
return nil, err
}
Expand Down
4 changes: 2 additions & 2 deletions transform/concat.go
View file
Expand Up @@ -33,7 +33,7 @@ func Concat(spec *Config, data []byte) ([]byte, error) {
if !ok {
path, ok := vItem.(map[string]interface{})["path"]
if ok {
zed, err := getJSONRaw(data, path.(string), spec.Require)
zed, err := getJSONRaw(data, path.(string), spec.Require, spec.KeySeparator)
switch {
case err != nil && spec.Require == true:
return nil, RequireError("Path does not exist")
Expand Down Expand Up @@ -63,7 +63,7 @@ func Concat(spec *Config, data []byte) ([]byte, error) {

applyDelim = true
}
data, err := setJSONRaw(data, bookend([]byte(outString), '"', '"'), targetPath.(string))
data, err := setJSONRaw(data, bookend([]byte(outString), '"', '"'), targetPath.(string), spec.KeySeparator)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion transform/default.go
View file
Expand Up @@ -13,7 +13,7 @@ func Default(spec *Config, data []byte) ([]byte, error) {
if err != nil {
return nil, ParseError(fmt.Sprintf("Warn: Unable to coerce element to json string: %v", v))
}
data, err = setJSONRaw(data, dataForV, k)
data, err = setJSONRaw(data, dataForV, k, spec.KeySeparator)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion transform/delete.go
View file
Expand Up @@ -22,7 +22,7 @@ func Delete(spec *Config, data []byte) ([]byte, error) {
}

var err error
data, err = delJSONRaw(data, path, spec.Require)
data, err = delJSONRaw(data, path, spec.Require, spec.KeySeparator)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion transform/extract.go
View file
Expand Up @@ -6,7 +6,7 @@ func Extract(spec *Config, data []byte) ([]byte, error) {
if !ok {
return nil, SpecError("Unable to get path")
}
result, err := getJSONRaw(data, outPath.(string), spec.Require)
result, err := getJSONRaw(data, outPath.(string), spec.Require, spec.KeySeparator)
if err != nil {
return nil, err
}
Expand Down
4 changes: 2 additions & 2 deletions transform/shift.go
View file
Expand Up @@ -48,7 +48,7 @@ func Shift(spec *Config, data []byte) ([]byte, error) {
if v == "$" {
dataForV = data
} else {
dataForV, err = getJSONRaw(data, v, spec.Require)
dataForV, err = getJSONRaw(data, v, spec.Require, spec.KeySeparator)
if err != nil {
return nil, err
}
Expand All @@ -65,7 +65,7 @@ func Shift(spec *Config, data []byte) ([]byte, error) {
// Note: following pattern from current Shift() - if multiple elements are included in an array,
// they will each successively overwrite each other and only the last element will be included
// in the transformed data.
outData, err = setJSONRaw(outData, dataForV, k)
outData, err = setJSONRaw(outData, dataForV, k, spec.KeySeparator)
if err != nil {
return nil, err
}
Expand Down
6 changes: 3 additions & 3 deletions transform/timestamp.go
View file
Expand Up @@ -48,7 +48,7 @@ func Timestamp(spec *Config, data []byte) ([]byte, error) {
inputFormat = time.RFC3339
} else {
// grab the data
dataForV, err = getJSONRaw(data, k, spec.Require)
dataForV, err = getJSONRaw(data, k, spec.Require, spec.KeySeparator)
if err != nil {
return nil, err
}
Expand All @@ -65,7 +65,7 @@ func Timestamp(spec *Config, data []byte) ([]byte, error) {
if err != nil {
return nil, err
}
data, err = setJSONRaw(data, []byte(formattedItem), k)
data, err = setJSONRaw(data, []byte(formattedItem), k, spec.KeySeparator)
if err != nil {
return nil, err
}
Expand All @@ -84,7 +84,7 @@ func Timestamp(spec *Config, data []byte) ([]byte, error) {
}
// replacing the wildcard here feels hacky, but seems to be the
// quickest way to achieve the outcome we want
data, err = setJSONRaw(data, []byte(formattedItem), strings.Replace(k, "*", strconv.Itoa(idx), -1))
data, err = setJSONRaw(data, []byte(formattedItem), strings.Replace(k, "*", strconv.Itoa(idx), -1), spec.KeySeparator)
if err != nil {
return nil, err
}
Expand Down
32 changes: 16 additions & 16 deletions transform/util.go
View file
Expand Up @@ -35,9 +35,10 @@ func (s SpecError) Error() string {
// Config contains the options that dictate the behavior of a transform. The internal
// `spec` object can be an arbitrary json configuration for the transform.
type Config struct {
Spec *map[string]interface{} `json:"spec"`
Require bool `json:"require,omitempty"`
InPlace bool `json:"inplace,omitempty"`
Spec *map[string]interface{} `json:"spec"`
Require bool `json:"require,omitempty"`
InPlace bool `json:"inplace,omitempty"`
KeySeparator string `json:"keySeparator"`
}

var (
Expand All @@ -46,8 +47,8 @@ var (
)

// Given a json byte slice `data` and a kazaam `path` string, return the object at the path in data if it exists.
func getJSONRaw(data []byte, path string, pathRequired bool) ([]byte, error) {
objectKeys := strings.Split(path, ".")
func getJSONRaw(data []byte, path string, pathRequired bool, keySeparator string) ([]byte, error) {
objectKeys := strings.Split(path, keySeparator)
numOfInserts := 0
for element, k := range objectKeys {
// check the object key to see if it also contains an array reference
Expand All @@ -64,7 +65,7 @@ func getJSONRaw(data []byte, path string, pathRequired bool) ([]byte, error) {
// ArrayEach setup
objectKeys[element+numOfInserts] = objKey
beforePath := objectKeys[:element+numOfInserts+1]
newPath := strings.Join(objectKeys[element+numOfInserts+1:], ".")
newPath := strings.Join(objectKeys[element+numOfInserts+1:], keySeparator)
var results [][]byte

// use jsonparser.ArrayEach to copy the array into results
Expand All @@ -82,7 +83,7 @@ func getJSONRaw(data []byte, path string, pathRequired bool) ([]byte, error) {
// GetJSONRaw() the rest of path for each element in results
if newPath != "" {
for i, value := range results {
intermediate, err := getJSONRaw(value, newPath, pathRequired)
intermediate, err := getJSONRaw(value, newPath, pathRequired, keySeparator)
if err == jsonparser.KeyPathNotFoundError {
if pathRequired {
return nil, NonExistentPath
Expand Down Expand Up @@ -137,11 +138,10 @@ func getJSONRaw(data []byte, path string, pathRequired bool) ([]byte, error) {
}

// setJSONRaw sets the value at a key and handles array indexing
func setJSONRaw(data, out []byte, path string) ([]byte, error) {
func setJSONRaw(data, out []byte, path, keySeparator string) ([]byte, error) {
var err error
splitPath := strings.Split(path, ".")
splitPath := strings.Split(path, keySeparator)
numOfInserts := 0

for element, k := range splitPath {
arrayRefs := jsonPathRe.FindAllStringSubmatch(k, -1)
if arrayRefs != nil && len(arrayRefs) > 0 {
Expand All @@ -158,7 +158,7 @@ func setJSONRaw(data, out []byte, path string) ([]byte, error) {
// ArrayEach setup
splitPath[element+numOfInserts] = objKey
beforePath := splitPath[:element+numOfInserts+1]
afterPath := strings.Join(splitPath[element+numOfInserts+1:], ".")
afterPath := strings.Join(splitPath[element+numOfInserts+1:], keySeparator)
// use jsonparser.ArrayEach to count the number of items in the
// array
var arraySize int
Expand All @@ -175,18 +175,18 @@ func setJSONRaw(data, out []byte, path string) ([]byte, error) {
// iterate through each item in the array by replacing the
// wildcard with an int and joining the path back together
newArrayKey := strings.Join([]string{"[", strconv.Itoa(i), "]"}, "")
beforePathStr := strings.Join(beforePath, ".")
beforePathStr := strings.Join(beforePath, keySeparator)
beforePathArrayKeyStr := strings.Join([]string{beforePathStr, newArrayKey}, "")
// if there's nothing that comes after the array index,
// don't join so that we avoid trailing cruft
if len(afterPath) > 0 {
newPath = strings.Join([]string{beforePathArrayKeyStr, afterPath}, ".")
newPath = strings.Join([]string{beforePathArrayKeyStr, afterPath}, keySeparator)
} else {
newPath = beforePathArrayKeyStr
}
// now call the function, but this time with an array index
// instead of a wildcard
data, err = setJSONRaw(data, out, newPath)
data, err = setJSONRaw(data, out, newPath, keySeparator)
if err != nil {
return nil, err
}
Expand All @@ -209,9 +209,9 @@ func setJSONRaw(data, out []byte, path string) ([]byte, error) {
}

// delJSONRaw deletes the value at a path and handles array indexing
func delJSONRaw(data []byte, path string, pathRequired bool) ([]byte, error) {
func delJSONRaw(data []byte, path string, pathRequired bool, keySeparator string) ([]byte, error) {
var err error
splitPath := strings.Split(path, ".")
splitPath := strings.Split(path, keySeparator)
numOfInserts := 0

for element, k := range splitPath {
Expand Down
65 changes: 60 additions & 5 deletions transform/util_test.go
View file
Expand Up @@ -11,7 +11,7 @@ const testJSONInput = `{"rating":{"example":{"value":3},"primary":{"value":3}}}`
func getConfig(spec string, require bool) Config {
var f map[string]interface{}
json.Unmarshal([]byte(spec), &f)
return Config{Spec: &f, Require: require}
return Config{Spec: &f, Require: require, KeySeparator: "."}
}

func getTransformTestWrapper(tform func(spec *Config, data []byte) ([]byte, error), cfg Config, input string) ([]byte, error) {
Expand Down Expand Up @@ -104,7 +104,35 @@ func TestSetJSONRaw(t *testing.T) {
{[]byte(`{"data":["value"]}`), []byte(`"newValue"`), "data[+]", []byte(`{"data":["value","newValue"]}`)},
}
for _, testItem := range setPathTests {
actual, _ := setJSONRaw(testItem.inputData, testItem.inputValue, testItem.path)
actual, _ := setJSONRaw(testItem.inputData, testItem.inputValue, testItem.path, ".")
areEqual, _ := checkJSONBytesEqual(actual, testItem.expectedOutput)
if !areEqual {
t.Error("Error data does not match expectation.")
t.Log("Expected: ", testItem.expectedOutput)
t.Log("Actual: ", string(actual))
}
}
}

func TestSetJSONRawNonDefaultKeySeparator(t *testing.T) {
setPathTests := []struct {
inputData []byte
inputValue []byte
path string
expectedOutput []byte
}{
{[]byte(`{"data":"value"}`), []byte(`"newValue"`), "data", []byte(`{"data":"newValue"}`)},
{[]byte(`{"data":["value", "notValue"]}`), []byte(`"newValue"`), "data[0]", []byte(`{"data":["newValue", "notValue"]}`)},
{[]byte(`{"data":["value", "notValue"]}`), []byte(`"newValue"`), "data[*]", []byte(`{"data":["newValue", "newValue"]}`)},
{[]byte(`{"data":[{"key": "value"}, {"key": "value"}]}`), []byte(`"newValue"`), "data[*]>key", []byte(`{"data":[{"key": "newValue"}, {"key": "newValue"}]}`)},
{[]byte(`{"data":[{"key": "value"}, {"key": "value"}]}`), []byte(`"newValue"`), "data[1]>key", []byte(`{"data":[{"key": "value"}, {"key": "newValue"}]}`)},
{[]byte(`{"data":{"subData":[{"key": "value"}, {"key": "value"}]}}`), []byte(`"newValue"`), "data>subData[*]>key", []byte(`{"data":{"subData":[{"key": "newValue"}, {"key": "newValue"}]}}`)},
{[]byte(`{"data":"value"}`), []byte(`"newValue"`), "data[1]", []byte(`{"data":[null,"newValue"]}`)},
{[]byte(`{"data":["value"]}`), []byte(`"newValue"`), "data[-]>key", []byte(`{"data":[{"key":"newValue"},"value"]}`)},
{[]byte(`{"data":["value"]}`), []byte(`"newValue"`), "data[+]", []byte(`{"data":["value","newValue"]}`)},
}
for _, testItem := range setPathTests {
actual, _ := setJSONRaw(testItem.inputData, testItem.inputValue, testItem.path, ">")
areEqual, _ := checkJSONBytesEqual(actual, testItem.expectedOutput)
if !areEqual {
t.Error("Error data does not match expectation.")
Expand All @@ -115,7 +143,7 @@ func TestSetJSONRaw(t *testing.T) {
}

func TestSetJSONRawBadIndex(t *testing.T) {
_, err := setJSONRaw([]byte(`{"data":["value"]}`), []byte(`"newValue"`), "data[g].key")
_, err := setJSONRaw([]byte(`{"data":["value"]}`), []byte(`"newValue"`), "data[g].key", ".")

errMsg := `Warn: Unable to coerce index to integer: g`
if err.Error() != errMsg {
Expand Down Expand Up @@ -143,7 +171,34 @@ func TestGetJSONRaw(t *testing.T) {
{[]byte(`{"data":{"subData":[{"key": "value"}, {"key": "value"}]}}`), "data.subData[*].key", true, []byte(`["value","value"]`)},
}
for _, testItem := range getPathTests {
actual, _ := getJSONRaw(testItem.inputData, testItem.path, testItem.required)
actual, _ := getJSONRaw(testItem.inputData, testItem.path, testItem.required, ".")
areEqual, _ := checkJSONBytesEqual(actual, testItem.expectedOutput)
if !areEqual {
t.Error("Error data does not match expectation.")
t.Log("Expected: ", string(testItem.expectedOutput))
t.Log("Actual: ", string(actual))
}
}
}

func TestGetJSONRawNonDefaultKeySeparator(t *testing.T) {
getPathTests := []struct {
inputData []byte
path string
required bool
expectedOutput []byte
}{
{[]byte(`{"data":"value"}`), "data", true, []byte(`"value"`)},
{[]byte(`{"data":"value"}`), "data", false, []byte(`"value"`)},
{[]byte(`{"notData":"value"}`), "data", false, []byte(`null`)},
{[]byte(`{"data":["value", "notValue"]}`), "data[0]", true, []byte(`"value"`)},
{[]byte(`{"data":["value", "notValue"]}`), "data[*]", true, []byte(`["value","notValue"]`)},
{[]byte(`{"data":[{"key": "value"}, {"key": "value"}]}`), "data[*]>key", true, []byte(`["value","value"]`)},
{[]byte(`{"data":[{"key": "value"}, {"key": "otherValue"}]}`), "data[1]>key", true, []byte(`"otherValue"`)},
{[]byte(`{"data":{"subData":[{"key": "value"}, {"key": "value"}]}}`), "data>subData[*]>key", true, []byte(`["value","value"]`)},
}
for _, testItem := range getPathTests {
actual, _ := getJSONRaw(testItem.inputData, testItem.path, testItem.required, ">")
areEqual, _ := checkJSONBytesEqual(actual, testItem.expectedOutput)
if !areEqual {
t.Error("Error data does not match expectation.")
Expand All @@ -154,7 +209,7 @@ func TestGetJSONRaw(t *testing.T) {
}

func TestGetJSONRawBadIndex(t *testing.T) {
_, err := getJSONRaw([]byte(`{"data":["value"]}`), "data[-1].key", true)
_, err := getJSONRaw([]byte(`{"data":["value"]}`), "data[-1].key", true, ".")

errMsg := `Warn: Unable to coerce index to integer: -1`
if err.Error() != errMsg {
Expand Down
4 changes: 2 additions & 2 deletions transform/uuid.go
View file
Expand Up @@ -67,7 +67,7 @@ func UUID(spec *Config, data []byte) ([]byte, error) {
for _, field := range nameFields {
p, _ := field.(map[string]interface{})["path"].(string)

name, pathErr := getJSONRaw(data, p, true)
name, pathErr := getJSONRaw(data, p, true, spec.KeySeparator)
// if a string, remove the heading and trailing quote
nameString := strings.TrimPrefix(strings.TrimSuffix(string(name), "\""), "\"")
if pathErr == NonExistentPath {
Expand All @@ -84,7 +84,7 @@ func UUID(spec *Config, data []byte) ([]byte, error) {

}
// set the uuid in the appropriate place
data, err = setJSONRaw(data, bookend([]byte(u.String()), '"', '"'), k)
data, err = setJSONRaw(data, bookend([]byte(u.String()), '"', '"'), k, spec.KeySeparator)
if err != nil {
return nil, err
}
Expand Down

0 comments on commit be7b380

Please sign in to comment.