diff --git a/docs/swagger/swagger.yaml b/docs/swagger/swagger.yaml index f82bd14ccf..67325ce7b0 100644 --- a/docs/swagger/swagger.yaml +++ b/docs/swagger/swagger.yaml @@ -1420,6 +1420,15 @@ paths: count: type: string timestamp: {} + types: + items: + properties: + count: + type: string + type: + type: string + type: object + type: array type: object description: Success default: diff --git a/internal/database/sqlcommon/chart_sql.go b/internal/database/sqlcommon/chart_sql.go index a93f946552..17c1bf29a8 100644 --- a/internal/database/sqlcommon/chart_sql.go +++ b/internal/database/sqlcommon/chart_sql.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -20,6 +20,7 @@ import ( "context" "database/sql" "fmt" + "strconv" sq "github.com/Masterminds/squirrel" "github.com/hyperledger/firefly/internal/i18n" @@ -27,13 +28,14 @@ import ( "github.com/hyperledger/firefly/pkg/fftypes" ) -func (s *SQLCommon) getCaseQueries(ns string, intervals []fftypes.ChartHistogramInterval) (caseQueries []sq.CaseBuilder) { - for _, interval := range intervals { +func (s *SQLCommon) getCaseQueries(ns string, dataTypes []string, interval fftypes.ChartHistogramInterval, typeColName string) (caseQueries []sq.CaseBuilder) { + for _, dataType := range dataTypes { caseQueries = append(caseQueries, sq.Case(). When( sq.And{ sq.GtOrEq{"created": interval.StartTime}, sq.Lt{"created": interval.EndTime}, + sq.Eq{typeColName: dataType}, sq.Eq{"namespace": ns}, }, "1", @@ -44,22 +46,47 @@ func (s *SQLCommon) getCaseQueries(ns string, intervals []fftypes.ChartHistogram return caseQueries } -func (s *SQLCommon) getTableNameFromCollection(ctx context.Context, collection database.CollectionName) (tableName string, err error) { +func (s *SQLCommon) getTableNameFromCollection(ctx context.Context, collection database.CollectionName) (tableName string, fieldMap map[string]string, err error) { switch collection { case database.CollectionName(database.CollectionMessages): - return "messages", nil + return "messages", msgFilterFieldMap, nil case database.CollectionName(database.CollectionTransactions): - return "transactions", nil + return "transactions", transactionFilterFieldMap, nil case database.CollectionName(database.CollectionOperations): - return "operations", nil + return "operations", opFilterFieldMap, nil case database.CollectionName(database.CollectionEvents): - return "events", nil + return "events", eventFilterFieldMap, nil + case database.CollectionName(database.CollectionTokenTransfers): + return "tokentransfers", tokenTransferFilterFieldMap, nil default: - return "", i18n.NewError(ctx, i18n.MsgUnsupportedCollection, collection) + return "", nil, i18n.NewError(ctx, i18n.MsgUnsupportedCollection, collection) + } +} + +func (s *SQLCommon) getDistinctTypesFromTable(ctx context.Context, tableName string, fieldMap map[string]string) ([]string, error) { + qb := sq.Select(fieldMap["type"]).Distinct().From(tableName) + + rows, _, err := s.query(ctx, qb.From(tableName)) + if err != nil { + return nil, err + } + defer rows.Close() + + var dataTypes []string + for rows.Next() { + var dataType string + err := rows.Scan(&dataType) + if err != nil { + return []string{}, i18n.WrapError(ctx, err, i18n.MsgDBReadErr, tableName) + } + dataTypes = append(dataTypes, dataType) } + rows.Close() + + return dataTypes, nil } -func (s *SQLCommon) histogramResult(ctx context.Context, rows *sql.Rows, cols []*fftypes.ChartHistogram) ([]*fftypes.ChartHistogram, error) { +func (s *SQLCommon) histogramResult(ctx context.Context, rows *sql.Rows, cols []*fftypes.ChartHistogramType, tableName string) ([]*fftypes.ChartHistogramType, error) { results := []interface{}{} for i := range cols { @@ -67,40 +94,80 @@ func (s *SQLCommon) histogramResult(ctx context.Context, rows *sql.Rows, cols [] } err := rows.Scan(results...) if err != nil { - return nil, i18n.NewError(ctx, i18n.MsgDBReadErr, "histogram") + return nil, i18n.NewError(ctx, i18n.MsgDBReadErr, tableName) } return cols, nil } -func (s *SQLCommon) GetChartHistogram(ctx context.Context, ns string, intervals []fftypes.ChartHistogramInterval, collection database.CollectionName) (histogram []*fftypes.ChartHistogram, err error) { - tableName, err := s.getTableNameFromCollection(ctx, collection) - if err != nil { - return nil, err +func (s *SQLCommon) getBucketTotal(typeBuckets []*fftypes.ChartHistogramType) (string, error) { + total := 0 + for _, typeBucket := range typeBuckets { + typeBucketInt, err := strconv.Atoi(typeBucket.Count) + if err != nil { + return "", err + } + total += typeBucketInt } + return strconv.Itoa(total), nil +} - qb := sq.Select() - - for i, caseQuery := range s.getCaseQueries(ns, intervals) { - query, args, _ := caseQuery.ToSql() - - histogram = append(histogram, &fftypes.ChartHistogram{ - Count: "", - Timestamp: intervals[i].StartTime, - }) - - qb = qb.Column(sq.Alias(sq.Expr("SUM("+query+")", args...), fmt.Sprintf("case_%d", i))) +func (s *SQLCommon) GetChartHistogram(ctx context.Context, ns string, intervals []fftypes.ChartHistogramInterval, collection database.CollectionName) (histogramList []*fftypes.ChartHistogram, err error) { + tableName, fieldMap, err := s.getTableNameFromCollection(ctx, collection) + if err != nil { + return nil, err } - rows, _, err := s.query(ctx, qb.From(tableName)) + dataTypes, err := s.getDistinctTypesFromTable(ctx, tableName, fieldMap) if err != nil { return nil, err } - defer rows.Close() - if !rows.Next() { - return []*fftypes.ChartHistogram{}, nil + for _, interval := range intervals { + qb := sq.Select() + histogramTypes := make([]*fftypes.ChartHistogramType, 0) + + for i, caseQuery := range s.getCaseQueries(ns, dataTypes, interval, fieldMap["type"]) { + query, args, _ := caseQuery.ToSql() + histogramTypes = append(histogramTypes, &fftypes.ChartHistogramType{ + Count: "", + Type: dataTypes[i], + }) + + qb = qb.Column(sq.Alias(sq.Expr("SUM("+query+")", args...), fmt.Sprintf("case_%d", i))) + } + + rows, _, err := s.query(ctx, qb.From(tableName)) + if err != nil { + return nil, err + } + defer rows.Close() + + if rows.Next() { + hist, err := s.histogramResult(ctx, rows, histogramTypes, tableName) + rows.Close() + if err != nil { + return nil, err + } + + total, err := s.getBucketTotal(hist) + if err != nil { + return nil, err + } + + histogramList = append(histogramList, &fftypes.ChartHistogram{ + Count: total, + Timestamp: interval.StartTime, + Types: hist, + }) + } else { + histogramList = append(histogramList, &fftypes.ChartHistogram{ + Count: "0", + Timestamp: interval.StartTime, + Types: make([]*fftypes.ChartHistogramType, 0), + }) + } } - return s.histogramResult(ctx, rows, histogram) + return histogramList, nil } diff --git a/internal/database/sqlcommon/chart_sql_test.go b/internal/database/sqlcommon/chart_sql_test.go index e598d91a5f..3a790fb758 100644 --- a/internal/database/sqlcommon/chart_sql_test.go +++ b/internal/database/sqlcommon/chart_sql_test.go @@ -28,13 +28,30 @@ import ( ) var ( - emptyHistogramResult = make([]*fftypes.ChartHistogram, 0) + emptyHistogramResult = []*fftypes.ChartHistogram{ + { + Count: "0", + Timestamp: fftypes.UnixTime(1000000000), + Types: make([]*fftypes.ChartHistogramType, 0), + }, + } expectedHistogramResult = []*fftypes.ChartHistogram{ { - Count: "123", + Count: "10", Timestamp: fftypes.UnixTime(1000000000), + Types: []*fftypes.ChartHistogramType{ + { + Count: "5", + Type: "typeA", + }, + { + Count: "5", + Type: "typeB", + }, + }, }, } + mockHistogramInterval = []fftypes.ChartHistogramInterval{ { StartTime: fftypes.UnixTime(1000000000), @@ -46,6 +63,7 @@ var ( "messages", "operations", "transactions", + "tokentransfers", } ) @@ -59,7 +77,8 @@ func TestGetChartHistogramInvalidCollectionName(t *testing.T) { func TestGetChartHistogramValidCollectionName(t *testing.T) { for i := range validCollections { s, mock := newMockProvider().init() - mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0"}).AddRow("123")) + mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB")) + mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "case_1"}).AddRow("5", "5")) histogram, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName(validCollections[i])) @@ -71,6 +90,7 @@ func TestGetChartHistogramValidCollectionName(t *testing.T) { func TestGetChartHistogramsQueryFail(t *testing.T) { s, mock := newMockProvider().init() + mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB")) mock.ExpectQuery("SELECT *").WillReturnError(fmt.Errorf("pop")) _, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("messages")) @@ -78,18 +98,48 @@ func TestGetChartHistogramsQueryFail(t *testing.T) { assert.NoError(t, mock.ExpectationsWereMet()) } +func TestGetChartHistogramQueryFailBadDistinctTypes(t *testing.T) { + s, mock := newMockProvider().init() + mock.ExpectQuery("SELECT DISTINCT .*").WillReturnError(fmt.Errorf("pop")) + + _, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("messages")) + assert.Regexp(t, "FF10115", err) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetChartHistogramScanFailInvalidRowType(t *testing.T) { + s, mock := newMockProvider().init() + mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow(nil).AddRow("typeB")) + + _, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("messages")) + assert.Regexp(t, "FF10121", err) + assert.NoError(t, mock.ExpectationsWereMet()) +} + func TestGetChartHistogramScanFailTooManyCols(t *testing.T) { s, mock := newMockProvider().init() - mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "unexpected_column"}).AddRow("one", "two")) + mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB")) + mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "case_1", "unexpected_col"}).AddRow("one", "two", "three")) _, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("messages")) assert.Regexp(t, "FF10121", err) assert.NoError(t, mock.ExpectationsWereMet()) } +func TestGetChartHistogramFailStringToIntConversion(t *testing.T) { + s, mock := newMockProvider().init() + mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB")) + mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "case_1"}).AddRow("5", "NotInt")) + + _, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("messages")) + assert.Error(t, err) + assert.NoError(t, mock.ExpectationsWereMet()) +} + func TestGetChartHistogramSuccessNoRows(t *testing.T) { s, mock := newMockProvider().init() - mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0"})) + mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB")) + mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "case_1"})) histogram, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("messages")) assert.NoError(t, err) @@ -99,7 +149,8 @@ func TestGetChartHistogramSuccessNoRows(t *testing.T) { func TestGetChartHistogramSuccess(t *testing.T) { s, mock := newMockProvider().init() - mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0"}).AddRow("123")) + mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB")) + mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "case_1"}).AddRow("5", "5")) histogram, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("messages")) diff --git a/internal/database/sqlcommon/tokentransfer_sql.go b/internal/database/sqlcommon/tokentransfer_sql.go index 279b20b0b1..813558b4b6 100644 --- a/internal/database/sqlcommon/tokentransfer_sql.go +++ b/internal/database/sqlcommon/tokentransfer_sql.go @@ -49,6 +49,7 @@ var ( "created", } tokenTransferFilterFieldMap = map[string]string{ + "type": "type", "localid": "local_id", "pool": "pool_id", "tokenindex": "token_index", diff --git a/pkg/fftypes/charthistogram.go b/pkg/fftypes/charthistogram.go index 218501a567..19716f08cc 100644 --- a/pkg/fftypes/charthistogram.go +++ b/pkg/fftypes/charthistogram.go @@ -1,4 +1,4 @@ -// Copyright © 2021 Kaleido, Inc. +// Copyright © 2022 Kaleido, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -23,12 +23,22 @@ const ( ChartHistogramMinBuckets = 1 ) -// ChartHistogram is a timestamp and count +// ChartHistogram is a list of buckets with types type ChartHistogram struct { - // Timestamp of bucket in histogram + // Count for entire timestamp in histogram + Count string `json:"count"` + // Timestamp of bucket Timestamp *FFTime `json:"timestamp"` - // Count for timestamp in histogram + // Types list of histogram types and their count + Types []*ChartHistogramType `json:"types"` +} + +// ChartHistogramType is a type and count +type ChartHistogramType struct { + // Count for type in histogram bucket Count string `json:"count"` + // Type of bucket in histogram + Type string `json:"type"` } // ChartHistogramInterval specifies lower and upper timestamps for histogram bucket