Skip to content

Commit

Permalink
feat(changes): make change calculation optional
Browse files Browse the repository at this point in the history 
also finally properly identified the big in the Patch method: map values in go can't be mutated in-place. need to re-set the key
  • Loading branch information
@b5
b5 committed Nov 27, 2019
1 parent 6bf7c9a commit e9dbc5f
Show file tree
Hide file tree
Showing 7 changed files with 155 additions and 110 deletions.
36 changes: 21 additions & 15 deletions deepdiff.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ type Config struct {
// If true Diff will calculate "moves" that describe changing the parent of
// a subtree
MoveDeltas bool
// Setting Changes to true will have diff represent in-place value shifts
// as changes instead of add-delete pairs
Changes bool
}

// DiffOption is a function that adjust a config, zero or more DiffOptions
Expand All @@ -37,7 +40,7 @@ func NewDeepDiff(opts ...DiffOption) *DeepDiff {
}

return &DeepDiff{
changes: false,
changes: cfg.Changes,
moveDeltas: cfg.MoveDeltas,
}
}
Expand All @@ -48,33 +51,28 @@ func NewDeepDiff(opts ...DiffOption) *DeepDiff {
// future use. specifically: bailing before delta calculation based on a
// configurable threshold
func (dd *DeepDiff) Diff(ctx context.Context, a, b interface{}) ([]*Delta, error) {
deepdiff := &diff{cfg: dd.config(), d1: a, d2: b}
deepdiff := &diff{moves: dd.moveDeltas, changes: dd.changes, d1: a, d2: b}
return deepdiff.diff(ctx), nil
}

// StatDiff calculates a diff script and diff stats
func (dd *DeepDiff) StatDiff(ctx context.Context, a, b interface{}) ([]*Delta, *Stats, error) {
deepdiff := &diff{cfg: dd.config(), d1: a, d2: b, stats: &Stats{}}
deepdiff := &diff{moves: dd.moveDeltas, changes: dd.changes, d1: a, d2: b, stats: &Stats{}}
return deepdiff.diff(ctx), deepdiff.stats, nil
}

// Stat calculates the DiffStata between two documents
func (dd *DeepDiff) Stat(ctx context.Context, a, b interface{}) (*Stats, error) {
deepdiff := &diff{cfg: dd.config(), d1: a, d2: b, stats: &Stats{}}
deepdiff := &diff{moves: dd.moveDeltas, changes: dd.changes, d1: a, d2: b, stats: &Stats{}}
deepdiff.diff(ctx)
return deepdiff.stats, nil
}

func (dd *DeepDiff) config() *Config {
return &Config{
MoveDeltas: dd.moveDeltas,
}
}

// diff is a state machine for calculating an edit script that transitions
// between two state trees
type diff struct {
cfg *Config
moves bool // calculate moves flag
changes bool // calculate changes flag
stats *Stats
d1, d2 interface{}
t1, t2 node
Expand Down Expand Up @@ -287,7 +285,8 @@ func propagateMatchToChildren(n node) {
}
}

// calculate inserts, changes, deletes, & moves
// calculate inserts, deletes, and/or changes & moves by walking the matched
// tree checking for edits
func (d *diff) calcDeltas(t1, t2 node) (dts []*Delta) {
walkSorted(t1, "", func(p string, n node) bool {
if n.Match() == nil {
Expand Down Expand Up @@ -363,7 +362,7 @@ func (d *diff) calcDeltas(t1, t2 node) (dts []*Delta) {
return false
}

if d.cfg.MoveDeltas {
if d.moves {
// If we have a match & parents are different, this corresponds to a move
if path(match.Parent()) != path(n.Parent()) {
delta := &Delta{
Expand Down Expand Up @@ -407,13 +406,20 @@ func (d *diff) calcDeltas(t1, t2 node) (dts []*Delta) {
// TODO (b5): this needs to be a check to see if it's a leaf node
// (eg, empty object is a leaf node)
if delta := compareScalar(match, n, p); delta != nil {
dts = append(dts, delta)
if d.changes {
dts = append(dts, delta)
} else {
dts = append(dts,
&Delta{Type: DTDelete, Path: delta.Path, Value: delta.SourceValue},
&Delta{Type: DTInsert, Path: delta.Path, Value: delta.Value},
)
}
}
}
return true
})

if d.cfg.MoveDeltas {
if d.moves {
var cleanups []string
walkSorted(t2, "", func(p string, n node) bool {
if n.Type() == ntArray && n.Match() != nil {
Expand Down
129 changes: 76 additions & 53 deletions deepdiff_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,35 +27,34 @@ func RunTestCases(t *testing.T, cases []TestCase, opts ...DiffOption) {
ctx = context.Background()
)

for i, c := range cases {
if err := json.Unmarshal([]byte(c.src), &src); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal([]byte(c.dst), &dst); err != nil {
t.Fatal(err)
}
for _, c := range cases {
t.Run(c.description, func(t *testing.T) {
if err := json.Unmarshal([]byte(c.src), &src); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal([]byte(c.dst), &dst); err != nil {
t.Fatal(err)
}

diff, err := dd.Diff(ctx, src, dst)
if err != nil {
t.Fatalf("%d, %s Diff error: %s", i, c.description, err)
}
diff, err := dd.Diff(ctx, src, dst)
if err != nil {
t.Fatalf("Diff error: %s", err)
}

if err := CompareDiffs(c.expect, diff); err != nil {
t.Errorf("%d. '%s' result mismatch: %s", i, c.description, err)
}
if err := CompareDiffs(c.expect, diff); err != nil {
t.Errorf("result mismatch: %s", err)
}

if err := Patch(&src, diff); err != nil {
t.Errorf("error patching source: %s", err)
}
if !reflect.DeepEqual(src, dst) {
t.Errorf("%d. '%s' patched result mismatch:", i, c.description)
srcData, _ := json.Marshal(src)
dstData, _ := json.Marshal(dst)
patchData, _ := json.Marshal(diff)
t.Log("src :", string(srcData))
t.Log("dst :", string(dstData))
t.Log("patch:", string(patchData))
}
if err := Patch(&src, diff); err != nil {
t.Errorf("error patching source: %s", err)
}
if !reflect.DeepEqual(src, dst) {
srcData, _ := json.Marshal(src)
dstData, _ := json.Marshal(dst)
patchData, _ := json.Marshal(diff)
t.Errorf("patched result mismatch:\nsrc : %s\ndst : %s\npatch:%s", string(srcData), string(dstData), string(patchData))
}
})
}
}

Expand Down Expand Up @@ -105,17 +104,20 @@ func TestBasicDiffing(t *testing.T) {
`[[0,1,2]]`,
`[[0,1,3]]`,
[]*Delta{
{Type: DTUpdate, Path: "/0/2", SourceValue: float64(2), Value: float64(3)},
},
},
{
"scalar change object",
`{"a":[0,1,2],"b":true}`,
`{"a":[0,1,3],"b":true}`,
[]*Delta{
{Type: DTUpdate, Path: "/a/2", SourceValue: float64(2), Value: float64(3)},
{Type: DTDelete, Path: "/0/2", Value: float64(2)},
{Type: DTInsert, Path: "/0/2", Value: float64(3)},
},
},
// TODO (b5) - skipping because of unassignable object key problem
// {
// "scalar change object",
// `{"a":[0,1,2],"b":true}`,
// `{"a":[0,1,3],"b":true}`,
// []*Delta{
// {Type: DTDelete, Path: "/a/2", Value: float64(2)},
// {Type: DTInsert, Path: "/a/2", Value: float64(3)},
// },
// },
{
"insert array",
`[[1]]`,
Expand Down Expand Up @@ -166,6 +168,29 @@ func TestBasicDiffing(t *testing.T) {
RunTestCases(t, cases)
}

func TestChangeDiffs(t *testing.T) {
cases := []TestCase{
{
"scalar change array",
`[[0,1,2]]`,
`[[0,1,3]]`,
[]*Delta{
{Type: DTUpdate, Path: "/0/2", SourceValue: float64(2), Value: float64(3)},
},
},
{
"scalar change object",
`{"a":[0,1,2],"b":true}`,
`{"a":[0,1,3],"b":true}`,
[]*Delta{
{Type: DTUpdate, Path: "/a/2", SourceValue: float64(2), Value: float64(3)},
},
},
}

RunTestCases(t, cases, func(c *Config) { c.Changes = true })
}

func TestMoveDiffs(t *testing.T) {
cases := []TestCase{
{
Expand All @@ -186,6 +211,7 @@ func TestMoveDiffs(t *testing.T) {
},
}
RunTestCases(t, cases, func(o *Config) {
o.Changes = true
o.MoveDeltas = true
})
}
Expand All @@ -212,11 +238,16 @@ func TestInsertGeneralizing(t *testing.T) {
`{"bodyPath":"/ipfs/QmUNYnjzjTJyBEY3gXzQuGaXeawoFpmCi3UxjpbN4mvnib","commit":{"author":{"id":"QmSyDX5LYTiwQi861F5NAwdHrrnd1iRGsoEvCyzQMUyZ4W"},"path":"/ipfs/QmcHeeUmiDQE97rHw8GSCKWfsMXsLyqw1xrwxDA34XSqNE","qri":"cm:0","signature":"jq8TIriZaUqWyoXwr/vhPZyuZkxFttL9Bse67yoPszWPdKn8KhO7+DGBkVc/VQYdNaGoWRLajRtlcv8avp5RADyJEA3hc2SGsfYW4X+I5Wyj6ckD9p4UfRMrYakJT5yGDlfa0OW0T306k6VTt3v4O93Jj1hBNS45xsZ/TKSRGwiA9l5uh2Xt2XMTRPeFvDImdTomhB5mZBfLCHp7tj2i7G892JQPz9lidiyq0KrF7I6xRXbCoW3DMq9q63xWCnN8dnUpOEn+mupv+KL36Dzl3cE78fcKL0M/6WHP9T4OxyaQ/CEYOQA4RlJbcXMX9jLFnYsCht8Vxq7ffqTlRKP8lA==","timestamp":"2019-02-22T14:21:27.038532Z","title":"created dataset"},"meta":{"accessPath":"https://theunitedstates.io/","citations":[{}],"description":"Processed list of current legislators from @unitedstates.\n\n@unitedstates is a shared commons of data and tools for the United States. Made by the public, used by the public. ","downloadPath":"https://theunitedstates.io/congress-legislators/legislators-current.json","keywords":["us","gov","congress","538"],"license":{"type":"CC0 - Creative Commons Zero Public Domain Dedication","url":"https://creativecommons.org/publicdomain/zero/1.0/"},"qri":"md:0","theme":["government"],"title":"US Members of Congress"},"name":"us_current_legislators","path":"/ipfs/QmST56YbcS7Um3vpwLwWDTZozPHPttc1N6Jd19uU1W2z4t/dataset.json","peername":"b5","qri":"ds:0","structure":{"checksum":"QmXzzSj4UNqdCo4yX3t6ELfFi5QoEyj8zi9mkqiJofN1PC","depth":2,"errCount":0,"entries":538,"format":"json","length":87453,"qri":"st:0","schema":{"type":"array"}},"transform":{"qri":"tf:0","scriptPath":"/ipfs/QmSzYwaciz5C75BGzqVho24ngmhwMm5CcqVUPrPAwqPNWc","syntax":"starlark","syntaxVersion":"0.2.2-dev"}}`,
`{"bodyPath":"/ipfs/QmUNYnjzjTJyBEY3gXzQuGaXeawoFpmCi3UxjpbN4mvnib","commit":{"author":{"id":"QmSyDX5LYTiwQi861F5NAwdHrrnd1iRGsoEvCyzQMUyZ4W"},"path":"/ipfs/QmR5JTQxxjJPrZBL4neynAyv2WLuXQujR9NoLkfcahc34W","qri":"cm:0","signature":"jy3JiFNVgcGn8pcm1Vuv9Z3AbVl18Yh7z3Bj+N8t5lz0/OY+ZxbBrNPXVx/M6FgbPA9RzFGzgJ8xKudBsqS94kJaQ9yg2zvNmZxufiFs3YxoIhxPabod0fY5Whq91Ns3Ov3AOCKarIYpXyAdFDvpRQ3VSyqwaTNc9lheutEDeFHmW5BGFNsA/NXhbPIocgE3G48PYUXIRInwaFhsLjpcFSwn/cG+Xbkly0OrOYtCTS5hZ0aBPbk6FAAu6l6BVGbxDduflYyt8UFpdiinJf8S7G+l5nwO0VlQwTT47q3CkcPAdQTtTxHnz4mYwaWPGeqryBi4TO6PXlmbRDLaQ8v3dQ==","timestamp":"2019-02-23T23:12:25.886874Z","title":"forced update"},"meta":{"accessPath":"https://theunitedstates.io/","citations":[{}],"description":"Processed list of current legislators from @unitedstates.\n\n@unitedstates is a shared commons of data and tools for the United States. Made by the public, used by the public. ","downloadPath":"https://theunitedstates.io/congress-legislators/legislators-current.json","keywords":["us","gov","congress","538"],"license":{"type":"CC0 - Creative Commons Zero Public Domain Dedication","url":"https://creativecommons.org/publicdomain/zero/1.0/"},"qri":"md:0","theme":["government"],"title":"US Members of Congress"},"name":"us_current_legislators","path":"/ipfs/QmTV1n5BfQnG4EigyRJUP3466FRPgDFEbckva6mEmtLR97/dataset.json","peername":"b5","previousPath":"/ipfs/QmST56YbcS7Um3vpwLwWDTZozPHPttc1N6Jd19uU1W2z4t","qri":"ds:0","stats":{"bioguide":{"count":538,"maxLength":7,"minLength":7,"type":"string"},"birthday":{"count":538,"maxLength":10,"minLength":10,"type":"string"},"first":{"count":538,"maxLength":11,"minLength":2,"type":"string"},"full":{"count":538,"maxLength":30,"minLength":6,"type":"string"},"gender":{"count":538,"maxLength":1,"minLength":1,"type":"string"},"last":{"count":538,"maxLength":17,"minLength":3,"type":"string"},"party":{"count":538,"maxLength":11,"minLength":8,"type":"string"},"religion":{"count":538,"max":0,"min":0,"type":"integer"},"state":{"count":538,"maxLength":2,"minLength":2,"type":"string"}},"structure":{"checksum":"QmXzzSj4UNqdCo4yX3t6ELfFi5QoEyj8zi9mkqiJofN1PC","depth":2,"errCount":0,"entries":538,"format":"json","length":87453,"qri":"st:0","schema":{"type":"array"}},"transform":{"qri":"tf:0","scriptPath":"/ipfs/QmSzYwaciz5C75BGzqVho24ngmhwMm5CcqVUPrPAwqPNWc","syntax":"starlark","syntaxVersion":"0.2.2-dev"}}`,
[]*Delta{
{Type: DTUpdate, Path: "/commit/path", SourceValue: "/ipfs/QmcHeeUmiDQE97rHw8GSCKWfsMXsLyqw1xrwxDA34XSqNE", Value: "/ipfs/QmR5JTQxxjJPrZBL4neynAyv2WLuXQujR9NoLkfcahc34W"},
{Type: DTUpdate, Path: "/commit/signature", SourceValue: "jq8TIriZaUqWyoXwr/vhPZyuZkxFttL9Bse67yoPszWPdKn8KhO7+DGBkVc/VQYdNaGoWRLajRtlcv8avp5RADyJEA3hc2SGsfYW4X+I5Wyj6ckD9p4UfRMrYakJT5yGDlfa0OW0T306k6VTt3v4O93Jj1hBNS45xsZ/TKSRGwiA9l5uh2Xt2XMTRPeFvDImdTomhB5mZBfLCHp7tj2i7G892JQPz9lidiyq0KrF7I6xRXbCoW3DMq9q63xWCnN8dnUpOEn+mupv+KL36Dzl3cE78fcKL0M/6WHP9T4OxyaQ/CEYOQA4RlJbcXMX9jLFnYsCht8Vxq7ffqTlRKP8lA==", Value: "jy3JiFNVgcGn8pcm1Vuv9Z3AbVl18Yh7z3Bj+N8t5lz0/OY+ZxbBrNPXVx/M6FgbPA9RzFGzgJ8xKudBsqS94kJaQ9yg2zvNmZxufiFs3YxoIhxPabod0fY5Whq91Ns3Ov3AOCKarIYpXyAdFDvpRQ3VSyqwaTNc9lheutEDeFHmW5BGFNsA/NXhbPIocgE3G48PYUXIRInwaFhsLjpcFSwn/cG+Xbkly0OrOYtCTS5hZ0aBPbk6FAAu6l6BVGbxDduflYyt8UFpdiinJf8S7G+l5nwO0VlQwTT47q3CkcPAdQTtTxHnz4mYwaWPGeqryBi4TO6PXlmbRDLaQ8v3dQ=="},
{Type: DTUpdate, Path: "/commit/timestamp", SourceValue: "2019-02-22T14:21:27.038532Z", Value: "2019-02-23T23:12:25.886874Z"},
{Type: DTUpdate, Path: "/commit/title", SourceValue: "created dataset", Value: "forced update"},
{Type: DTUpdate, Path: "/path", SourceValue: "/ipfs/QmST56YbcS7Um3vpwLwWDTZozPHPttc1N6Jd19uU1W2z4t/dataset.json", Value: "/ipfs/QmTV1n5BfQnG4EigyRJUP3466FRPgDFEbckva6mEmtLR97/dataset.json"},
{Type: DTDelete, Path: "/commit/path", Value: "/ipfs/QmcHeeUmiDQE97rHw8GSCKWfsMXsLyqw1xrwxDA34XSqNE"},
{Type: DTInsert, Path: "/commit/path", Value: "/ipfs/QmR5JTQxxjJPrZBL4neynAyv2WLuXQujR9NoLkfcahc34W"},
{Type: DTDelete, Path: "/commit/signature", Value: "jq8TIriZaUqWyoXwr/vhPZyuZkxFttL9Bse67yoPszWPdKn8KhO7+DGBkVc/VQYdNaGoWRLajRtlcv8avp5RADyJEA3hc2SGsfYW4X+I5Wyj6ckD9p4UfRMrYakJT5yGDlfa0OW0T306k6VTt3v4O93Jj1hBNS45xsZ/TKSRGwiA9l5uh2Xt2XMTRPeFvDImdTomhB5mZBfLCHp7tj2i7G892JQPz9lidiyq0KrF7I6xRXbCoW3DMq9q63xWCnN8dnUpOEn+mupv+KL36Dzl3cE78fcKL0M/6WHP9T4OxyaQ/CEYOQA4RlJbcXMX9jLFnYsCht8Vxq7ffqTlRKP8lA=="},
{Type: DTInsert, Path: "/commit/signature", Value: "jy3JiFNVgcGn8pcm1Vuv9Z3AbVl18Yh7z3Bj+N8t5lz0/OY+ZxbBrNPXVx/M6FgbPA9RzFGzgJ8xKudBsqS94kJaQ9yg2zvNmZxufiFs3YxoIhxPabod0fY5Whq91Ns3Ov3AOCKarIYpXyAdFDvpRQ3VSyqwaTNc9lheutEDeFHmW5BGFNsA/NXhbPIocgE3G48PYUXIRInwaFhsLjpcFSwn/cG+Xbkly0OrOYtCTS5hZ0aBPbk6FAAu6l6BVGbxDduflYyt8UFpdiinJf8S7G+l5nwO0VlQwTT47q3CkcPAdQTtTxHnz4mYwaWPGeqryBi4TO6PXlmbRDLaQ8v3dQ=="},
{Type: DTDelete, Path: "/commit/timestamp", Value: "2019-02-22T14:21:27.038532Z"},
{Type: DTInsert, Path: "/commit/timestamp", Value: "2019-02-23T23:12:25.886874Z"},
{Type: DTDelete, Path: "/commit/title", Value: "created dataset"},
{Type: DTInsert, Path: "/commit/title", Value: "forced update"},
{Type: DTDelete, Path: "/path", Value: "/ipfs/QmST56YbcS7Um3vpwLwWDTZozPHPttc1N6Jd19uU1W2z4t/dataset.json"},
{Type: DTInsert, Path: "/path", Value: "/ipfs/QmTV1n5BfQnG4EigyRJUP3466FRPgDFEbckva6mEmtLR97/dataset.json"},
{Type: DTInsert, Path: "/previousPath", Value: "/ipfs/QmST56YbcS7Um3vpwLwWDTZozPHPttc1N6Jd19uU1W2z4t"},
{Type: DTInsert, Path: "/stats", Value: map[string]interface{}{
"bioguide": map[string]interface{}{"count": float64(538), "maxLength": float64(7), "minLength": float64(7), "type": "string"},
Expand Down Expand Up @@ -278,7 +309,7 @@ func TestDiffDotGraph(t *testing.T) {
panic(err)
}

d := &diff{cfg: &Config{}, d1: a, d2: b}
d := &diff{d1: a, d2: b}
d.t1, d.t2, d.t1Nodes = d.prepTrees(context.Background())
d.queueMatch(d.t1Nodes, d.t2)
d.optimize(d.t1, d.t2)
Expand Down Expand Up @@ -358,18 +389,10 @@ func TestDiffIntData(t *testing.T) {
}

expect := []*Delta{
&Delta{
Type: DTUpdate,
Path: "/1/1",
Value: int64(0),
SourceValue: int64(5),
},
&Delta{
Type: DTUpdate,
Path: "/2/0",
Value: int64(10),
SourceValue: int64(7),
},
&Delta{Type: DTDelete, Path: "/1/1", Value: int64(5)},
&Delta{Type: DTInsert, Path: "/1/1", Value: int64(0)},
&Delta{Type: DTDelete, Path: "/2/0", Value: int64(7)},
&Delta{Type: DTInsert, Path: "/2/0", Value: int64(10)},
}
if err := CompareDiffs(expect, diff); err != nil {
t.Errorf("Compare result mismatch: %s", err)
Expand Down
10 changes: 6 additions & 4 deletions delta.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,20 @@ package deepdiff
type Operation string

const (
// DTContext indicates unchanged contextual details present in both A and B
DTContext = Operation(" ")
// DTDelete means making the children of a node
// become the children of a node's parent
DTDelete = Operation("delete")
DTDelete = Operation("-")
// DTInsert is the compliment of deleting, adding
// children of a parent node to a new node, and making
// that node a child of the original parent
DTInsert = Operation("insert")
DTInsert = Operation("+")
// DTMove is the succession of a deletion & insertion
// of the same node
DTMove = Operation("move")
DTMove = Operation(">")
// DTUpdate is an alteration of a scalar data type (string, bool, float, etc)
DTUpdate = Operation("update")
DTUpdate = Operation("~")
)

// Delta represents a change between a source & destination document
Expand Down
11 changes: 7 additions & 4 deletions examples_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,14 @@ func ExampleDiffJSON() {
}

fmt.Println(change)
// Output: baz:
// Output: + a: 99
// - a: 100
// baz:
// + e: "thirty-thousand-something-dogecoin"
// + f: false
// - e: null
// - g: "apples-and-oranges"
// a:
// ~ b: 5
// ~ e: "thirty-thousand-something-dogecoin"
// ~ a: 99
// + b: 5
// - b: 4
}
Loading

0 comments on commit e9dbc5f

Please sign in to comment.