Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support merging JSON arrays, objects #148

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/cli/go-gh/v2
go 1.21

require (
dario.cat/mergo v1.0.0
github.com/AlecAivazis/survey/v2 v2.3.7
github.com/MakeNowJust/heredoc v1.0.0
github.com/charmbracelet/glamour v0.6.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ=
github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo=
github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
Expand Down
76 changes: 76 additions & 0 deletions pkg/jsonmerge/array.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package jsonmerge

import "io"

type arrayMerger struct {
isFirstPage bool
}

// NewArrayMerger creates a Merger for JSON arrays.
func NewArrayMerger() Merger {
return &arrayMerger{
isFirstPage: true,
}
}

func (merger *arrayMerger) NewPage(r io.Reader, isLastPage bool) io.ReadCloser {
return &arrayMergerPage{
merger: merger,
Reader: r,
isLastPage: isLastPage,
}
}

func (m *arrayMerger) Close() error {
// arrayMerger merges when reading, so any output was already written
// and there's nothing to do on Close.
return nil
}

type arrayMergerPage struct {
merger *arrayMerger

io.Reader
isLastPage bool

isSubsequentRead bool
cachedByte byte
}

func (page *arrayMergerPage) Read(p []byte) (int, error) {
var n int
var err error

if page.cachedByte != 0 && len(p) > 0 {
p[0] = page.cachedByte
n, err = page.Reader.Read(p[1:])
n += 1
page.cachedByte = 0
} else {
n, err = page.Reader.Read(p)
}

if !page.isSubsequentRead && !page.merger.isFirstPage && n > 0 && p[0] == '[' {
if n > 1 && p[1] == ']' {
// Empty array case.
p[0] = ' '
} else {
// Avoid starting a new array and continue with a comma instead.
p[0] = ','
}
}

if !page.isLastPage && n > 0 && p[n-1] == ']' {
// Avoid closing off an array in case we determine we are at EOF.
page.cachedByte = p[n-1]
n -= 1
}

page.isSubsequentRead = true
return n, err
}

func (page *arrayMergerPage) Close() error {
page.merger.isFirstPage = false
return nil
}
86 changes: 86 additions & 0 deletions pkg/jsonmerge/array_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package jsonmerge

import (
"bytes"
"io"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestArrayMerger_singleEmptyArray(t *testing.T) {
merger := NewArrayMerger()
w := &bytes.Buffer{}

r1 := bytes.NewBufferString(`[]`)
p1 := merger.NewPage(r1, true)
n, err := io.Copy(w, p1)
require.NoError(t, err)
assert.Equal(t, int64(2), n)
assert.NoError(t, p1.Close())
assert.Equal(t, `[]`, w.String())

require.NoError(t, merger.Close())
}

func TestArrayMerger_finalEmptyArray(t *testing.T) {
merger := NewArrayMerger()
w := &bytes.Buffer{}

r1 := bytes.NewBufferString(`["a","b"]`)
p1 := merger.NewPage(r1, false)
n, err := io.Copy(w, p1)
require.NoError(t, err)
assert.Equal(t, int64(8), n)
assert.NoError(t, p1.Close())
assert.Equal(t, `["a","b"`, w.String())

r2 := bytes.NewBufferString(`[]`)
p2 := merger.NewPage(r2, true)
n, err = io.Copy(w, p2)
require.NoError(t, err)
assert.Equal(t, int64(2), n)
assert.NoError(t, p2.Close())
assert.Equal(t, `["a","b" ]`, w.String())

require.NoError(t, merger.Close())
}

func TestArrayMerger_multiplePages(t *testing.T) {
merger := NewArrayMerger()
w := &bytes.Buffer{}

r1 := bytes.NewBufferString(`["a","b"]`)
p1 := merger.NewPage(r1, false)
n, err := io.Copy(w, p1)
require.NoError(t, err)
assert.Equal(t, int64(8), n)
assert.NoError(t, p1.Close())
assert.Equal(t, `["a","b"`, w.String())

r2 := bytes.NewBufferString(`["c","d"]`)
p2 := merger.NewPage(r2, true)
n, err = io.Copy(w, p2)
require.NoError(t, err)
assert.Equal(t, int64(9), n)
assert.NoError(t, p2.Close())
assert.Equal(t, `["a","b","c","d"]`, w.String())

require.NoError(t, merger.Close())
}

func TestArrayMerger_emptyObject(t *testing.T) {
merger := NewArrayMerger()
w := &bytes.Buffer{}

r1 := bytes.NewBufferString(`{}`)
p1 := merger.NewPage(r1, true)
n, err := io.Copy(w, p1)
require.NoError(t, err)
assert.Equal(t, int64(2), n)
assert.NoError(t, p1.Close())
assert.Equal(t, `{}`, w.String())

require.NoError(t, merger.Close())
}
12 changes: 12 additions & 0 deletions pkg/jsonmerge/merge.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
// jsonmerge implements readers to merge JSON arrays or objects.
package jsonmerge

import (
"io"
)

// Merger is implemented to merge JSON arrays or objects.
type Merger interface {
NewPage(r io.Reader, isLastPage bool) io.ReadCloser
Close() error
}
66 changes: 66 additions & 0 deletions pkg/jsonmerge/object.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package jsonmerge

import (
"bytes"
"encoding/json"
"io"

"dario.cat/mergo"
)

type objectMerger struct {
io.Writer
dst map[string]interface{}
}

// NewObjectMerger creates a Merger for JSON objects.
func NewObjectMerger(w io.Writer) Merger {
return &objectMerger{
Writer: w,
dst: make(map[string]interface{}),
}
}

func (merger *objectMerger) NewPage(r io.Reader, isLastPage bool) io.ReadCloser {
williammartin marked this conversation as resolved.
Show resolved Hide resolved
return &objectMergerPage{
merger: merger,
Reader: r,
}
}

func (merger *objectMerger) Close() error {
// Marshal to JSON and write to output.
buf, err := json.Marshal(merger.dst)
if err != nil {
return err
}

_, err = merger.Writer.Write(buf)
return err
}

type objectMergerPage struct {
merger *objectMerger

io.Reader
buffer bytes.Buffer
}

// Read caches the data in an internal buffer to be merged in Close.
// No data is copied into p so it's not written to stdout.
func (page *objectMergerPage) Read(p []byte) (int, error) {
heaths marked this conversation as resolved.
Show resolved Hide resolved
_, err := io.CopyN(&page.buffer, page.Reader, int64(len(p)))
return 0, err
}

// Close converts the internal buffer to a JSON object and merges it with the final JSON object.
func (page *objectMergerPage) Close() error {
var src map[string]interface{}

err := json.Unmarshal(page.buffer.Bytes(), &src)
if err != nil {
return err
heaths marked this conversation as resolved.
Show resolved Hide resolved
}

return mergo.Merge(&page.merger.dst, src, mergo.WithAppendSlice, mergo.WithOverride)
}
98 changes: 98 additions & 0 deletions pkg/jsonmerge/object_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package jsonmerge

import (
"bytes"
"io"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestObjectMerger_singleEmptyObject(t *testing.T) {
w := &bytes.Buffer{}
merger := NewObjectMerger(w)

r1 := bytes.NewBufferString(`{}`)
p1 := merger.NewPage(r1, true)
n, err := io.Copy(w, p1)
require.NoError(t, err)
assert.Equal(t, int64(0), n)
assert.NoError(t, p1.Close())
assert.Equal(t, ``, w.String())

require.NoError(t, merger.Close())
assert.JSONEq(t, `{}`, w.String())
}

func TestObjectMerger_finalEmptyObject(t *testing.T) {
w := &bytes.Buffer{}
merger := NewObjectMerger(w)

r1 := bytes.NewBufferString(`{"a":1,"b":2}`)
p1 := merger.NewPage(r1, false)
n, err := io.Copy(w, p1)
require.NoError(t, err)
assert.Equal(t, int64(0), n)
assert.NoError(t, p1.Close())
assert.Equal(t, ``, w.String())

r2 := bytes.NewBufferString(`{}`)
p2 := merger.NewPage(r2, true)
n, err = io.Copy(w, p2)
require.NoError(t, err)
assert.Equal(t, int64(0), n)
assert.NoError(t, p2.Close())
assert.Equal(t, ``, w.String())

require.NoError(t, merger.Close())
assert.JSONEq(t, `{"a":1,"b":2}`, w.String())
}

func TestObjectMerger_multiplePages(t *testing.T) {
w := &bytes.Buffer{}
merger := NewObjectMerger(w)

r1 := bytes.NewBufferString(`{"a":1,"b":2,"arr":["a","b"]}`)
p1 := merger.NewPage(r1, false)
n, err := io.Copy(w, p1)
require.NoError(t, err)
assert.Equal(t, int64(0), n)
assert.NoError(t, p1.Close())
assert.Equal(t, ``, w.String())

r2 := bytes.NewBufferString(`{"b":3,"c":{"d":4},"arr":["c","d"]}`)
p2 := merger.NewPage(r2, true)
n, err = io.Copy(w, p2)
require.NoError(t, err)
assert.Equal(t, int64(0), n)
assert.NoError(t, p2.Close())
assert.Equal(t, ``, w.String())

require.NoError(t, merger.Close())
assert.JSONEq(t, `{"a":1,"b":3,"c":{"d":4},"arr":["a","b","c","d"]}`, w.String())
}

func TestObjectMerger_invalidJSON(t *testing.T) {
w := &bytes.Buffer{}
merger := NewObjectMerger(w)

r1 := bytes.NewBufferString(`invalid`)
p1 := merger.NewPage(r1, true)
n, err := io.Copy(w, p1)
require.NoError(t, err)
assert.Equal(t, int64(0), n)
assert.Error(t, p1.Close())
}

func TestObjectMerger_array(t *testing.T) {
w := &bytes.Buffer{}
merger := NewObjectMerger(w)

r1 := bytes.NewBufferString(`[]`)
p1 := merger.NewPage(r1, true)
n, err := io.Copy(w, p1)
require.NoError(t, err)
assert.Equal(t, int64(0), n)
assert.Error(t, p1.Close())
}
Loading