[charts-by-type] histogram broken down by type

docs/swagger/swagger.yaml

@@ -1420,6 +1420,15 @@ paths:
                   count:
                     type: string
                   timestamp: {}
+                  types:
+                    items:
+                      properties:
+                        count:
+                          type: string
+                        type:
+                          type: string
+                      type: object
+                    type: array
                 type: object
           description: Success
         default:

internal/database/sqlcommon/chart_sql.go

@@ -1,4 +1,4 @@
-// Copyright © 2021 Kaleido, Inc.
+// Copyright © 2022 Kaleido, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -20,20 +20,22 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"strconv"
 
 	sq "github.com/Masterminds/squirrel"
 	"github.com/hyperledger/firefly/internal/i18n"
 	"github.com/hyperledger/firefly/pkg/database"
 	"github.com/hyperledger/firefly/pkg/fftypes"
 )
 
-func (s *SQLCommon) getCaseQueries(ns string, intervals []fftypes.ChartHistogramInterval) (caseQueries []sq.CaseBuilder) {
-	for _, interval := range intervals {
+func (s *SQLCommon) getCaseQueries(ns string, dataTypes []string, interval fftypes.ChartHistogramInterval, typeColName string) (caseQueries []sq.CaseBuilder) {
+	for _, dataType := range dataTypes {
 		caseQueries = append(caseQueries, sq.Case().
 			When(
 				sq.And{
 					sq.GtOrEq{"created": interval.StartTime},
 					sq.Lt{"created": interval.EndTime},
+					sq.Eq{typeColName: dataType},
 					sq.Eq{"namespace": ns},
 				},
 				"1",
@@ -44,63 +46,128 @@ func (s *SQLCommon) getCaseQueries(ns string, intervals []fftypes.ChartHistogram
 	return caseQueries
 }
 
-func (s *SQLCommon) getTableNameFromCollection(ctx context.Context, collection database.CollectionName) (tableName string, err error) {
+func (s *SQLCommon) getTableNameFromCollection(ctx context.Context, collection database.CollectionName) (tableName string, fieldMap map[string]string, err error) {
 	switch collection {
 	case database.CollectionName(database.CollectionMessages):
-		return "messages", nil
+		return "messages", msgFilterFieldMap, nil
 	case database.CollectionName(database.CollectionTransactions):
-		return "transactions", nil
+		return "transactions", transactionFilterFieldMap, nil
 	case database.CollectionName(database.CollectionOperations):
-		return "operations", nil
+		return "operations", opFilterFieldMap, nil
 	case database.CollectionName(database.CollectionEvents):
-		return "events", nil
+		return "events", eventFilterFieldMap, nil
+	case database.CollectionName(database.CollectionTokenTransfers):
+		return "tokentransfers", tokenTransferFilterFieldMap, nil
 	default:
-		return "", i18n.NewError(ctx, i18n.MsgUnsupportedCollection, collection)
+		return "", nil, i18n.NewError(ctx, i18n.MsgUnsupportedCollection, collection)
+	}
+}
+
+func (s *SQLCommon) getDistinctTypesFromTable(ctx context.Context, tableName string, fieldMap map[string]string) ([]string, error) {
+	qb := sq.Select(fieldMap["type"]).Distinct().From(tableName)
+
+	rows, _, err := s.query(ctx, qb.From(tableName))
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var dataTypes []string
+	for rows.Next() {
+		var dataType string
+		err := rows.Scan(&dataType)
+		if err != nil {
+			return []string{}, i18n.WrapError(ctx, err, i18n.MsgDBReadErr, tableName)
+		}
+		dataTypes = append(dataTypes, dataType)
 	}
+	rows.Close()
+
+	return dataTypes, nil
 }
 
-func (s *SQLCommon) histogramResult(ctx context.Context, rows *sql.Rows, cols []*fftypes.ChartHistogram) ([]*fftypes.ChartHistogram, error) {
+func (s *SQLCommon) histogramResult(ctx context.Context, rows *sql.Rows, cols []*fftypes.ChartHistogramType, tableName string) ([]*fftypes.ChartHistogramType, error) {
 	results := []interface{}{}
 
 	for i := range cols {
 		results = append(results, &cols[i].Count)
 	}
 	err := rows.Scan(results...)
 	if err != nil {
-		return nil, i18n.NewError(ctx, i18n.MsgDBReadErr, "histogram")
+		return nil, i18n.NewError(ctx, i18n.MsgDBReadErr, tableName)
 	}
 
 	return cols, nil
 }
 
-func (s *SQLCommon) GetChartHistogram(ctx context.Context, ns string, intervals []fftypes.ChartHistogramInterval, collection database.CollectionName) (histogram []*fftypes.ChartHistogram, err error) {
-	tableName, err := s.getTableNameFromCollection(ctx, collection)
-	if err != nil {
-		return nil, err
+func (s *SQLCommon) getBucketTotal(typeBuckets []*fftypes.ChartHistogramType) (string, error) {
+	total := 0
+	for _, typeBucket := range typeBuckets {
+		typeBucketInt, err := strconv.Atoi(typeBucket.Count)
+		if err != nil {
+			return "", err
+		}
+		total += typeBucketInt
 	}
+	return strconv.Itoa(total), nil
+}
 
-	qb := sq.Select()
-
-	for i, caseQuery := range s.getCaseQueries(ns, intervals) {
-		query, args, _ := caseQuery.ToSql()
-
-		histogram = append(histogram, &fftypes.ChartHistogram{
-			Count:     "",
-			Timestamp: intervals[i].StartTime,
-		})
-
-		qb = qb.Column(sq.Alias(sq.Expr("SUM("+query+")", args...), fmt.Sprintf("case_%d", i)))
+func (s *SQLCommon) GetChartHistogram(ctx context.Context, ns string, intervals []fftypes.ChartHistogramInterval, collection database.CollectionName) (histogramList []*fftypes.ChartHistogram, err error) {
+	tableName, fieldMap, err := s.getTableNameFromCollection(ctx, collection)
+	if err != nil {
+		return nil, err
 	}
 
-	rows, _, err := s.query(ctx, qb.From(tableName))
+	dataTypes, err := s.getDistinctTypesFromTable(ctx, tableName, fieldMap)
 	if err != nil {
 		return nil, err
 	}
-	defer rows.Close()
 
-	if !rows.Next() {
-		return []*fftypes.ChartHistogram{}, nil
+	for _, interval := range intervals {
+		qb := sq.Select()
+		histogramTypes := make([]*fftypes.ChartHistogramType, 0)
+
+		for i, caseQuery := range s.getCaseQueries(ns, dataTypes, interval, fieldMap["type"]) {
+			query, args, _ := caseQuery.ToSql()
+			histogramTypes = append(histogramTypes, &fftypes.ChartHistogramType{
+				Count: "",
+				Type:  dataTypes[i],
+			})
+
+			qb = qb.Column(sq.Alias(sq.Expr("SUM("+query+")", args...), fmt.Sprintf("case_%d", i)))
+		}
+
+		rows, _, err := s.query(ctx, qb.From(tableName))
+		if err != nil {
+			return nil, err
+		}
+		defer rows.Close()
+
+		if rows.Next() {
+			hist, err := s.histogramResult(ctx, rows, histogramTypes, tableName)
+			rows.Close()
+			if err != nil {
+				return nil, err
+			}
+
+			total, err := s.getBucketTotal(hist)
+			if err != nil {
+				return nil, err
+			}
+
+			histogramList = append(histogramList, &fftypes.ChartHistogram{
+				Count:     total,
+				Timestamp: interval.StartTime,
+				Types:     hist,
+			})
+		} else {
+			histogramList = append(histogramList, &fftypes.ChartHistogram{
+				Count:     "0",
+				Timestamp: interval.StartTime,
+				Types:     make([]*fftypes.ChartHistogramType, 0),
+			})
+		}
 	}
 
-	return s.histogramResult(ctx, rows, histogram)
+	return histogramList, nil
 }

internal/database/sqlcommon/chart_sql_test.go

@@ -28,13 +28,30 @@ import (
 )
 
 var (
-	emptyHistogramResult    = make([]*fftypes.ChartHistogram, 0)
+	emptyHistogramResult = []*fftypes.ChartHistogram{
+		{
+			Count:     "0",
+			Timestamp: fftypes.UnixTime(1000000000),
+			Types:     make([]*fftypes.ChartHistogramType, 0),
+		},
+	}
 	expectedHistogramResult = []*fftypes.ChartHistogram{
 		{
-			Count:     "123",
+			Count:     "10",
 			Timestamp: fftypes.UnixTime(1000000000),
+			Types: []*fftypes.ChartHistogramType{
+				{
+					Count: "5",
+					Type:  "typeA",
+				},
+				{
+					Count: "5",
+					Type:  "typeB",
+				},
+			},
 		},
 	}
+
 	mockHistogramInterval = []fftypes.ChartHistogramInterval{
 		{
 			StartTime: fftypes.UnixTime(1000000000),
@@ -46,6 +63,7 @@ var (
 		"messages",
 		"operations",
 		"transactions",
+		"tokentransfers",
 	}
 )
 
@@ -59,7 +77,8 @@ func TestGetChartHistogramInvalidCollectionName(t *testing.T) {
 func TestGetChartHistogramValidCollectionName(t *testing.T) {
 	for i := range validCollections {
 		s, mock := newMockProvider().init()
-		mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0"}).AddRow("123"))
+		mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB"))
+		mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "case_1"}).AddRow("5", "5"))
 
 		histogram, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName(validCollections[i]))
 
@@ -71,25 +90,56 @@ func TestGetChartHistogramValidCollectionName(t *testing.T) {
 
 func TestGetChartHistogramsQueryFail(t *testing.T) {
 	s, mock := newMockProvider().init()
+	mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB"))
 	mock.ExpectQuery("SELECT *").WillReturnError(fmt.Errorf("pop"))
 
 	_, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("messages"))
 	assert.Regexp(t, "FF10115", err)
 	assert.NoError(t, mock.ExpectationsWereMet())
 }
 
+func TestGetChartHistogramQueryFailBadDistinctTypes(t *testing.T) {
+	s, mock := newMockProvider().init()
+	mock.ExpectQuery("SELECT DISTINCT .*").WillReturnError(fmt.Errorf("pop"))
+
+	_, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("messages"))
+	assert.Regexp(t, "FF10115", err)
+	assert.NoError(t, mock.ExpectationsWereMet())
+}
+
+func TestGetChartHistogramScanFailInvalidRowType(t *testing.T) {
+	s, mock := newMockProvider().init()
+	mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow(nil).AddRow("typeB"))
+
+	_, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("messages"))
+	assert.Regexp(t, "FF10121", err)
+	assert.NoError(t, mock.ExpectationsWereMet())
+}
+
 func TestGetChartHistogramScanFailTooManyCols(t *testing.T) {
 	s, mock := newMockProvider().init()
-	mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "unexpected_column"}).AddRow("one", "two"))
+	mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB"))
+	mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "case_1", "unexpected_col"}).AddRow("one", "two", "three"))
 
 	_, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("messages"))
 	assert.Regexp(t, "FF10121", err)
 	assert.NoError(t, mock.ExpectationsWereMet())
 }
 
+func TestGetChartHistogramFailStringToIntConversion(t *testing.T) {
+	s, mock := newMockProvider().init()
+	mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB"))
+	mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "case_1"}).AddRow("5", "NotInt"))
+
+	_, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("messages"))
+	assert.Error(t, err)
+	assert.NoError(t, mock.ExpectationsWereMet())
+}
+
 func TestGetChartHistogramSuccessNoRows(t *testing.T) {
 	s, mock := newMockProvider().init()
-	mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0"}))
+	mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB"))
+	mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "case_1"}))
 
 	histogram, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("messages"))
 	assert.NoError(t, err)
@@ -99,7 +149,8 @@ func TestGetChartHistogramSuccessNoRows(t *testing.T) {
 
 func TestGetChartHistogramSuccess(t *testing.T) {
 	s, mock := newMockProvider().init()
-	mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0"}).AddRow("123"))
+	mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB"))
+	mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "case_1"}).AddRow("5", "5"))
 
 	histogram, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("messages"))
 

internal/database/sqlcommon/tokentransfer_sql.go

@@ -49,6 +49,7 @@ var (
 		"created",
 	}
 	tokenTransferFilterFieldMap = map[string]string{
+		"type":            "type",
 		"localid":         "local_id",
 		"pool":            "pool_id",
 		"tokenindex":      "token_index",

pkg/fftypes/charthistogram.go

@@ -1,4 +1,4 @@
-// Copyright © 2021 Kaleido, Inc.
+// Copyright © 2022 Kaleido, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -23,12 +23,22 @@ const (
 	ChartHistogramMinBuckets = 1
 )
 
-// ChartHistogram is a timestamp and count
+// ChartHistogram is a list of buckets with types
 type ChartHistogram struct {
-	// Timestamp of bucket in histogram
+	// Count for entire timestamp in histogram
+	Count string `json:"count"`
+	// Timestamp of bucket
 	Timestamp *FFTime `json:"timestamp"`
-	// Count for timestamp in histogram
+	// Types list of histogram types and their count
+	Types []*ChartHistogramType `json:"types"`
+}
+
+// ChartHistogramType is a type and count
+type ChartHistogramType struct {
+	// Count for type in histogram bucket
 	Count string `json:"count"`
+	// Type of bucket in histogram
+	Type string `json:"type"`
 }
 
 // ChartHistogramInterval specifies lower and upper timestamps for histogram bucket