Fix the data error when loading json with path (#298)

* Fix the data error when loading json with path

* Add test case

runner/database.go

@@ -385,12 +385,12 @@ func writeRowFromDatabase(dbInfo DatabaseConnectorInfoDatabase, out *ResultWrite
 	return out.WriteRow(row)
 }
 
-func (ec EvalContext) loadJSONArrayPanel(projectId, panelId string) (chan map[string]any, error) {
+func (ec *EvalContext) loadJSONArrayPanel(projectId, panelId string) (chan map[string]any, error) {
 	f := ec.GetPanelResultsFile(projectId, panelId)
-	return loadJSONArrayFile(f)
+	return loadJSONArrayFileWithPath(f, ec.path)
 }
 
-func (ec EvalContext) EvalDatabasePanelWithWriter(
+func (ec *EvalContext) EvalDatabasePanelWithWriter(
 	project *ProjectState,
 	pageIndex int,
 	panel *PanelInfo,
@@ -479,7 +479,7 @@ func (ec EvalContext) EvalDatabasePanelWithWriter(
 	idMap := getIdMap(project.Pages[pageIndex])
 	idShapeMap := getIdShapeMap(project.Pages[pageIndex])
 
-	panelsToImport, query, err := transformDM_getPanelCalls(
+	panelsToImport, query, path, err := transformDM_getPanelCalls(
 		panel.Content,
 		idShapeMap,
 		idMap,
@@ -490,6 +490,7 @@ func (ec EvalContext) EvalDatabasePanelWithWriter(
 	if err != nil {
 		return err
 	}
+	ec.path = path
 
 	// Copy remote sqlite database to tmp file if remote
 	if dbInfo.Type == SQLiteDatabase && server != nil {
@@ -604,7 +605,7 @@ func (ec EvalContext) EvalDatabasePanelWithWriter(
 	})
 }
 
-func (ec EvalContext) EvalDatabasePanel(
+func (ec *EvalContext) EvalDatabasePanel(
 	project *ProjectState,
 	pageIndex int,
 	panel *PanelInfo,

runner/eval.go

@@ -138,6 +138,7 @@ func (ec EvalContext) evalMacros(content string, project *ProjectState, pageInde
 type EvalContext struct {
 	settings Settings
 	fsBase   string
+	path     string
 }
 
 func (ec EvalContext) decrypt(e *Encrypt) (string, error) {
@@ -180,7 +181,7 @@ func (ec EvalContext) decrypt(e *Encrypt) (string, error) {
 }
 
 func NewEvalContext(s Settings, fsBase string) EvalContext {
-	return EvalContext{s, fsBase}
+	return EvalContext{s, fsBase, ""}
 }
 
 func (ec EvalContext) Eval(projectId, panelId string) (error, string) {

runner/go.mod

@@ -29,6 +29,7 @@ require (
 	github.com/sijms/go-ora/v2 v2.5.3
 	github.com/snowflakedb/gosnowflake v1.6.13
 	github.com/stretchr/testify v1.8.0
+	github.com/tidwall/gjson v1.14.3
 	github.com/xitongsys/parquet-go v1.6.2
 	github.com/xitongsys/parquet-go-source v0.0.0-20220723234337-052319f3f36b
 	github.com/xuri/excelize/v2 v2.6.1
@@ -101,6 +102,8 @@ require (
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 // indirect
 	github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22 // indirect
 	go.opencensus.io v0.23.0 // indirect

runner/go.sum

@@ -564,6 +564,12 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw=
+github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=

runner/json.go

@@ -3,9 +3,12 @@ package runner
 import (
 	"encoding/json"
 	"io"
+	"io/ioutil"
 	"os"
+	"strings"
 
 	goccy_json "github.com/goccy/go-json"
+	"github.com/tidwall/gjson"
 )
 
 // goccy/go-json is the fastest library benchmarked in
@@ -73,17 +76,27 @@ func WriteJSONFile(file string, value any) error {
 	return encoder.Encode(value)
 }
 
-func loadJSONArrayFile(f string) (chan map[string]any, error) {
+func loadJSONArrayFileWithPath(f, path string) (chan map[string]any, error) {
 	out := make(chan map[string]any, 1000)
 
 	fd, err := os.Open(f)
 	if err != nil {
 		return nil, err
 	}
 
+	var reader io.Reader = fd
+	if path != "" {
+		data, err := ioutil.ReadAll(fd)
+		if err != nil {
+			return nil, err
+		}
+		value := gjson.GetBytes(data, path)
+		reader = strings.NewReader(value.String())
+	}
+
 	bs := make([]byte, 1)
 	for {
-		_, err := fd.Read(bs)
+		_, err := reader.Read(bs)
 		if err != nil {
 			return nil, err
 		}
@@ -97,7 +110,7 @@ func loadJSONArrayFile(f string) (chan map[string]any, error) {
 		defer fd.Close()
 		defer close(out)
 
-		var r io.Reader = fd
+		var r io.Reader = reader
 
 		// Stream all JSON objects
 		for {
@@ -139,3 +152,7 @@ func loadJSONArrayFile(f string) (chan map[string]any, error) {
 
 	return out, nil
 }
+
+func loadJSONArrayFile(f string) (chan map[string]any, error) {
+	return loadJSONArrayFileWithPath(f, "")
+}

runner/sql.go

@@ -119,14 +119,14 @@ func transformDM_getPanelCalls(
 	getPanelCallsAllowed bool,
 	qt quoteType,
 	cachePresent bool,
-) ([]panelToImport, string, error) {
+) ([]panelToImport, string, string, error) {
 	var panelsToImport []panelToImport
 
 	var insideErr error
+	path := ""
 	query = dmGetPanelRe.ReplaceAllStringFunc(query, func(m string) string {
 		matchForSubexps := dmGetPanelRe.FindStringSubmatch(m)
 		nameOrIndex := ""
-		path := ""
 		for i, name := range dmGetPanelRe.SubexpNames() {
 			if matchForSubexps[i] == "" {
 				continue
@@ -200,14 +200,14 @@ func transformDM_getPanelCalls(
 	})
 
 	if insideErr != nil {
-		return nil, "", insideErr
+		return nil, "", "", insideErr
 	}
 
 	if len(panelsToImport) > 0 && !getPanelCallsAllowed {
-		return nil, "", makeErrUnsupported("DM_getPanel() is not yet supported by this connector.")
+		return nil, "", "", makeErrUnsupported("DM_getPanel() is not yet supported by this connector.")
 	}
 
-	return panelsToImport, query, nil
+	return panelsToImport, query, path, nil
 }
 
 func GetObjectAtPath(obj map[string]any, path string) any {

runner/sql_test.go

@@ -33,7 +33,7 @@ func Test_transformDM_getPanelCalls(t *testing.T) {
 			},
 		},
 	}
-	panels, query, err := transformDM_getPanelCalls(
+	panels, query, path, err := transformDM_getPanelCalls(
 		"SELECT * FROM DM_getPanel(0) p0, DM_getPanel('my great panel')",
 		map[string]Shape{
 			"0":              shape,
@@ -53,6 +53,7 @@ func Test_transformDM_getPanelCalls(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, query, `SELECT * FROM "t_0" p0, "t_my great panel"`)
 	assert.Equal(t, len(panels), 2)
+	assert.Equal(t, path, "")
 	assert.Equal(t, panels[0], panelToImport{
 		tableName: "t_0",
 		columns: []column{
@@ -88,6 +89,7 @@ func Test_transformDM_getPanel_callsWithPaths(t *testing.T) {
 		data       string
 		query      string
 		expColumns []column
+		expPath    string
 	}{
 
 		{
@@ -97,6 +99,7 @@ func Test_transformDM_getPanel_callsWithPaths(t *testing.T) {
 				{name: "b", kind: "REAL"},
 				{name: "c", kind: "REAL"},
 			},
+			"a",
 		},
 		{
 			`{"a": [{"b": 2}, {"c": 3}]}`,
@@ -105,6 +108,7 @@ func Test_transformDM_getPanel_callsWithPaths(t *testing.T) {
 				{name: "b", kind: "REAL"},
 				{name: "c", kind: "REAL"},
 			},
+			"a",
 		},
 		{
 			`[{"b": 2}, {"c": 3}]`,
@@ -113,6 +117,7 @@ func Test_transformDM_getPanel_callsWithPaths(t *testing.T) {
 				{name: "b", kind: "REAL"},
 				{name: "c", kind: "REAL"},
 			},
+			"",
 		},
 	}
 
@@ -123,7 +128,7 @@ func Test_transformDM_getPanel_callsWithPaths(t *testing.T) {
 		s := GetShape("", j, len(test.data))
 		assert.Nil(t, err)
 
-		panels, query, err := transformDM_getPanelCalls(
+		panels, query, path, err := transformDM_getPanelCalls(
 			test.query,
 			map[string]Shape{"0": s},
 			map[string]string{"0": " a great id 2"},
@@ -135,6 +140,7 @@ func Test_transformDM_getPanel_callsWithPaths(t *testing.T) {
 		assert.Nil(t, err)
 		assert.Equal(t, query, `SELECT * FROM "t_0"`)
 		assert.Equal(t, len(panels), 1)
+		assert.Equal(t, test.expPath, path)
 		assert.Equal(t, panels[0], panelToImport{
 			tableName: "t_0",
 			columns:   test.expColumns,
@@ -218,17 +224,20 @@ func makeTestEvalContext() (EvalContext, func()) {
 func Test_sqlIngest_e2e(t *testing.T) {
 	tests := []struct {
 		json      string
+		sql       string
 		expResult []any
 	}{
 		{
 			`[{"a": 1},{"b": 2}]`,
+			"SELECT * FROM DM_getPanel(0) ORDER BY a DESC",
 			[]any{
 				map[string]any{"a": float64(1), "b": nil},
 				map[string]any{"a": nil, "b": float64(2)},
 			},
 		},
 		{
 			`[{"a": 1},{"b": 2}, {"a": 1},{"b": 2}, {"a": 1},{"b": 2}, {"a": 1},{"b": 2}, {"a": 1},{"b": 2}]`,
+			"SELECT * FROM DM_getPanel(0) ORDER BY a DESC",
 			[]any{
 				map[string]any{"a": float64(1), "b": nil},
 				map[string]any{"a": float64(1), "b": nil},
@@ -244,6 +253,7 @@ func Test_sqlIngest_e2e(t *testing.T) {
 		},
 		{
 			`[{"a": 1},{"b": 2}, {"a": 1},{"b": 2}, {"a": 1},{"b": 2}, {"a": 1},{"b": 2}, {"a": 1},{"b": 2},{"b": 2}]`,
+			"SELECT * FROM DM_getPanel(0) ORDER BY a DESC",
 			[]any{
 				map[string]any{"a": float64(1), "b": nil},
 				map[string]any{"a": float64(1), "b": nil},
@@ -260,6 +270,7 @@ func Test_sqlIngest_e2e(t *testing.T) {
 		},
 		{
 			`[{"a": 1},{"b": 2},{"c": [1]}]`,
+			"SELECT * FROM DM_getPanel(0) ORDER BY a DESC",
 			[]any{
 				map[string]any{"a": float64(1), "b": nil, "c": nil},
 				map[string]any{"a": nil, "b": float64(2), "c": nil},
@@ -268,10 +279,19 @@ func Test_sqlIngest_e2e(t *testing.T) {
 		},
 		{
 			`[{"a": 1,"b": 2,"c": [1]}]`,
+			"SELECT * FROM DM_getPanel(0) ORDER BY a DESC",
 			[]any{
 				map[string]any{"a": float64(1), "b": float64(2), "c": "[1]"},
 			},
 		},
+		{
+			`{"data":{"data1":[{"id":1,"name":"Corah"},{"id":3,"name":"Minh"}],"data2":[{"id":2,"name":"Corah"},{"id":4,"name":"Minh"}]},"total":2}`,
+			"SELECT * FROM DM_getPanel(0, 'data.data2') ORDER BY id DESC",
+			[]any{
+				map[string]any{"id": float64(4), "name": "Minh"},
+				map[string]any{"id": float64(2), "name": "Corah"},
+			},
+		},
 	}
 
 	projectTmp, err := os.CreateTemp("", "dsq-project")
@@ -315,7 +335,7 @@ func Test_sqlIngest_e2e(t *testing.T) {
 
 		panel2 := &PanelInfo{
 			Type:    DatabasePanel,
-			Content: "SELECT * FROM DM_getPanel(0) ORDER BY a DESC",
+			Content: test.sql,
 			Id:      newId(),
 			Name:    newId(),
 			DatabasePanelInfo: &DatabasePanelInfo{
@@ -326,7 +346,7 @@ func Test_sqlIngest_e2e(t *testing.T) {
 		}
 
 		err = ec.EvalDatabasePanel(project, 0, panel2, func(projectId, panelId string) (chan map[string]any, error) {
-			return loadJSONArrayFile(readFile.Name())
+			return loadJSONArrayFileWithPath(readFile.Name(), ec.path)
 		}, *DefaultCacheSettings)
 		if err != nil {
 			// Otherwise the channel below gets weird to debug
@@ -397,7 +417,7 @@ func Test_sqlIngest_BENCHMARK(t *testing.T) {
 
 	ec := EvalContext{}
 	err = ec.EvalDatabasePanel(project, 0, panel2, func(projectId, panelId string) (chan map[string]any, error) {
-		return loadJSONArrayFile(readFile)
+		return loadJSONArrayFileWithPath(readFile, ec.path)
 	}, *DefaultCacheSettings)
 	assert.Nil(t, err)
 }