Improve multivalue field support for report load step

Same behavour as with record lists where the module field delimiter
is used.

compose/service/record_datasource.go

@@ -95,6 +95,13 @@ func (svc record) Datasource(ctx context.Context, ld *report.LoadStepDefinition)
 			c.Name = f.Name
 			c.Label = f.Label
 			c.Options = f.Options
+			c.Multivalue = f.Multi
+			if f.Multi {
+				c.MultivalueDelimiter = f.Options.String("multiDelimiter")
+				if c.MultivalueDelimiter == "" {
+					c.MultivalueDelimiter = "\n"
+				}
+			}
 			if c.Label == "" {
 				c.Label = c.Name
 			}

pkg/report/frame.go

@@ -39,8 +39,6 @@ type (
 
 		// params to help us perform things in place
 		startIndex int
-		size       int
-		sliced     bool
 	}
 
 	FrameRowSet []FrameRow
@@ -57,6 +55,9 @@ type (
 		System  bool          `json:"system"`
 		Options ColumnOptions `json:"-"`
 
+		Multivalue          bool   `json:"multivalue"`
+		MultivalueDelimiter string `json:"multivalueDelimiter"`
+
 		Caster frameCellCaster `json:"-" yaml:"-"`
 	}
 
@@ -279,14 +280,7 @@ func (f *Frame) String() string {
 
 	f.WalkRows(func(i int, r FrameRow) error {
 		out += fmt.Sprintf("%d| ", i+1)
-		for _, c := range r {
-			if c == nil {
-				out += "<N/A>, "
-			} else {
-				v := cast.ToString(c.Get())
-				out += fmt.Sprintf("%s, ", v)
-			}
-		}
+		out += fmt.Sprintf("%s, ", r.String())
 		out = strings.TrimRight(out, ", ") + "\n"
 		return nil
 	})
@@ -480,38 +474,32 @@ func (dd FrameDefinitionSet) FindBySourceRef(source, ref string) *FrameDefinitio
 	return nil
 }
 
-func (r FrameRow) MarshalJSON() (out []byte, err error) {
-	aux := make([]string, len(r))
+func (row FrameRow) String() string {
+	out := ""
 	var s string
 
-	for i, c := range r {
-		if c == nil {
-			continue
-		}
-
-		s, err = cast.ToStringE(c.Get())
+	format := func(c expr.TypedValue) (out string) {
+		out, err := cast.ToStringE(c.Get())
 		if err != nil {
-			return nil, err
+			return fmt.Sprintf("[STRING CAST ERROR]%s", err.Error())
 		}
-
-		aux[i] = s
+		return out
 	}
 
-	return json.Marshal(aux)
-}
-
-func (r FrameRow) String() string {
-	out := ""
-	var s string
-	var err error
-	for _, c := range r {
-		if c == nil {
+	for _, cell := range row {
+		if cell == nil {
 			out += "<N/A>, "
 		} else {
-			s, err = cast.ToStringE(c.Get())
-			if err != nil {
-				out = fmt.Sprintf("%s, [STRING CAST ERROR]%s", out, err.Error())
-			} else {
+			switch c := cell.(type) {
+			case *expr.Array:
+				aux := make([]string, 0, 3)
+				for _, c := range c.GetValue() {
+					aux = append(aux, format(c))
+				}
+
+				out = fmt.Sprintf("%s, %s", out, fmt.Sprintf("[%s]", strings.Join(aux, ", ")))
+			default:
+				s = format(cell)
 				out = fmt.Sprintf("%s, %s", out, s)
 			}
 		}
@@ -530,6 +518,26 @@ func (a FrameRow) Compare(b FrameRow, cols ...int) int {
 	return 0
 }
 
+// AppendCell adds an extra value to the cell at index i
+func (a FrameRow) AppendCell(i int, v expr.TypedValue) (err error) {
+	out := make([]expr.TypedValue, 0, 10)
+
+	handle := func(v expr.TypedValue) {
+		switch c := v.(type) {
+		case *expr.Array:
+			out = append(out, c.GetValue()...)
+		default:
+			out = append(out, v)
+		}
+	}
+
+	handle(a[i])
+	handle(v)
+
+	a[i], err = expr.NewArray(out)
+	return err
+}
+
 func (dd FrameDescriptionSet) FilterBySource(source string) FrameDescriptionSet {
 	out := make(FrameDescriptionSet, 0, len(dd))
 

store/rdbms/compose_record_datasource.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/Masterminds/squirrel"
 	"github.com/cortezaproject/corteza-server/compose/types"
+	"github.com/cortezaproject/corteza-server/pkg/expr"
 	"github.com/cortezaproject/corteza-server/pkg/filter"
 	"github.com/cortezaproject/corteza-server/pkg/qlng"
 	"github.com/cortezaproject/corteza-server/pkg/report"
@@ -325,7 +326,9 @@ func (r *recordDatasource) preloadQuery(def *report.FrameDefinition) (squirrel.S
 }
 
 func (r *recordDatasource) load(ctx context.Context, def *report.FrameDefinition, q squirrel.SelectBuilder) (l report.Loader, c report.Closer, err error) {
-	sort := def.Sort
+	var (
+		sort filter.SortExprSet
+	)
 
 	// - paging related stuff
 	if def.Paging.PageCursor != nil {
@@ -633,12 +636,39 @@ func (b *recordDatasource) cast(row sqlx.ColScanner, out *report.Frame) (err err
 				return err
 			}
 			r[i] = c
-
 		}
+	}
 
+	// Handling multi value fields; rows with duplicated PKs get their multi value fields
+	// merged together.
+	next := false
+	hasPrimary := false
+	if len(out.Rows) > 0 {
+		for i, c := range out.Columns {
+			hasPrimary = hasPrimary || c.Primary
+			if c.Primary {
+				a := out.Rows[len(out.Rows)-1][i].(expr.Comparable)
+				if o, err := a.Compare(r[i]); err != nil || o != 0 {
+					next = true
+					break
+				}
+			}
+		}
+	} else {
+		next = true
 	}
 
-	out.Rows = append(out.Rows, r)
+	next = next || !hasPrimary
+	if next {
+		out.Rows = append(out.Rows, r)
+	} else {
+		lastR := out.Rows[len(out.Rows)-1]
+		for i, c := range out.Columns {
+			if c.Multivalue {
+				lastR.AppendCell(i, r[i])
+			}
+		}
+	}
 	return nil
 }
 

system/rest/report.go

@@ -2,14 +2,18 @@ package rest
 
 import (
 	"context"
+	"encoding/json"
+	"strings"
 
 	"github.com/cortezaproject/corteza-server/pkg/api"
+	"github.com/cortezaproject/corteza-server/pkg/expr"
 	"github.com/cortezaproject/corteza-server/pkg/filter"
 	"github.com/cortezaproject/corteza-server/pkg/report"
 	"github.com/cortezaproject/corteza-server/system/rest/request"
 	"github.com/cortezaproject/corteza-server/system/service"
 	"github.com/cortezaproject/corteza-server/system/types"
 	"github.com/pkg/errors"
+	"github.com/spf13/cast"
 )
 
 var _ = errors.Wrap
@@ -56,7 +60,17 @@ type (
 	}
 
 	reportFramePayload struct {
-		Frames []*report.Frame `json:"frames"`
+		Frames []*rspFrame `json:"frames"`
+	}
+
+	rspFrame struct {
+		*report.Frame
+		Rows []rspRows `json:"rows"`
+	}
+
+	rspRows struct {
+		row  report.FrameRow
+		cols report.FrameColumnSet
 	}
 )
 
@@ -182,7 +196,64 @@ func (ctrl Report) makeReportFramePayload(ctx context.Context, ff []*report.Fram
 		return nil, err
 	}
 
+	out := make([]*rspFrame, len(ff))
+	for i, f := range ff {
+		out[i] = &rspFrame{
+			Frame: f,
+		}
+
+		// We're including cols here so we can use then when marshling JSON
+		for _, r := range f.Rows {
+			out[i].Rows = append(out[i].Rows, rspRows{
+				row:  r,
+				cols: f.Columns,
+			})
+		}
+	}
+
 	return &reportFramePayload{
-		Frames: ff,
+		Frames: out,
 	}, nil
 }
+
+func (rr rspRows) MarshalJSON() (out []byte, err error) {
+	row := rr.row
+	cc := rr.cols
+
+	aux := make([]string, len(row))
+	var ss []string
+
+	enc := func(e expr.TypedValue) error {
+		s, err := cast.ToStringE(e.Get())
+		if err != nil {
+			return err
+		}
+
+		ss = append(ss, s)
+		return nil
+	}
+
+	for i, c := range row {
+		ss = make([]string, 0, 2)
+		if c == nil {
+			continue
+		}
+
+		switch cc := c.(type) {
+		case *expr.Array:
+			for _, v := range cc.GetValue() {
+				if err = enc(v); err != nil {
+					return
+				}
+			}
+		default:
+			if err = enc(c); err != nil {
+				return
+			}
+		}
+
+		aux[i] = strings.Join(ss, cc[i].MultivalueDelimiter)
+	}
+
+	return json.Marshal(aux)
+}