From 0dccdcc4c7063c14cb93c4f4ea9e36dc8a88f708 Mon Sep 17 00:00:00 2001 From: David Echelberger Date: Fri, 18 Mar 2022 08:21:05 -0400 Subject: [PATCH] [blockchainevents-charts] support for blockchain event charting Signed-off-by: David Echelberger --- internal/database/sqlcommon/chart_sql.go | 125 +++++++++++++++--- internal/database/sqlcommon/chart_sql_test.go | 69 +++++++++- pkg/database/plugin.go | 2 +- 3 files changed, 172 insertions(+), 24 deletions(-) diff --git a/internal/database/sqlcommon/chart_sql.go b/internal/database/sqlcommon/chart_sql.go index 17c1bf29a8..c61c3c0f55 100644 --- a/internal/database/sqlcommon/chart_sql.go +++ b/internal/database/sqlcommon/chart_sql.go @@ -28,7 +28,28 @@ import ( "github.com/hyperledger/firefly/pkg/fftypes" ) -func (s *SQLCommon) getCaseQueries(ns string, dataTypes []string, interval fftypes.ChartHistogramInterval, typeColName string) (caseQueries []sq.CaseBuilder) { +func (s *SQLCommon) getCaseQueriesByInterval(ns string, intervals []fftypes.ChartHistogramInterval) (caseQueries []sq.CaseBuilder) { + for _, interval := range intervals { + caseQueries = append(caseQueries, sq.Case(). + When( + sq.And{ + // Querying by 'timestamp' field for blockchain events + // If more tables are supported that have no "type" field, + // and a different date field name, + // this method will need to be refactored + sq.GtOrEq{"timestamp": interval.StartTime}, + sq.Lt{"timestamp": interval.EndTime}, + sq.Eq{"namespace": ns}, + }, + "1", + ). + Else("0")) + } + + return caseQueries +} + +func (s *SQLCommon) getCaseQueriesByType(ns string, dataTypes []string, interval fftypes.ChartHistogramInterval, typeColName string) (caseQueries []sq.CaseBuilder) { for _, dataType := range dataTypes { caseQueries = append(caseQueries, sq.Case(). When( @@ -57,13 +78,18 @@ func (s *SQLCommon) getTableNameFromCollection(ctx context.Context, collection d case database.CollectionName(database.CollectionEvents): return "events", eventFilterFieldMap, nil case database.CollectionName(database.CollectionTokenTransfers): - return "tokentransfers", tokenTransferFilterFieldMap, nil + return "tokentransfer", tokenTransferFilterFieldMap, nil + case database.CollectionName(database.CollectionBlockchainEvents): + return "blockchainevents", blockchainEventFilterFieldMap, nil default: return "", nil, i18n.NewError(ctx, i18n.MsgUnsupportedCollection, collection) } } func (s *SQLCommon) getDistinctTypesFromTable(ctx context.Context, tableName string, fieldMap map[string]string) ([]string, error) { + if _, ok := fieldMap["type"]; !ok { + return []string{}, nil + } qb := sq.Select(fieldMap["type"]).Distinct().From(tableName) rows, _, err := s.query(ctx, qb.From(tableName)) @@ -86,7 +112,19 @@ func (s *SQLCommon) getDistinctTypesFromTable(ctx context.Context, tableName str return dataTypes, nil } -func (s *SQLCommon) histogramResult(ctx context.Context, rows *sql.Rows, cols []*fftypes.ChartHistogramType, tableName string) ([]*fftypes.ChartHistogramType, error) { +func (s *SQLCommon) getBucketTotal(typeBuckets []*fftypes.ChartHistogramType) (string, error) { + total := 0 + for _, typeBucket := range typeBuckets { + typeBucketInt, err := strconv.Atoi(typeBucket.Count) + if err != nil { + return "", err + } + total += typeBucketInt + } + return strconv.Itoa(total), nil +} + +func (s *SQLCommon) histogramResultWithTypes(ctx context.Context, rows *sql.Rows, cols []*fftypes.ChartHistogramType, tableName string) ([]*fftypes.ChartHistogramType, error) { results := []interface{}{} for i := range cols { @@ -100,34 +138,55 @@ func (s *SQLCommon) histogramResult(ctx context.Context, rows *sql.Rows, cols [] return cols, nil } -func (s *SQLCommon) getBucketTotal(typeBuckets []*fftypes.ChartHistogramType) (string, error) { - total := 0 - for _, typeBucket := range typeBuckets { - typeBucketInt, err := strconv.Atoi(typeBucket.Count) - if err != nil { - return "", err - } - total += typeBucketInt +func (s *SQLCommon) histogramResultNoType(ctx context.Context, rows *sql.Rows, cols []*fftypes.ChartHistogram, tableName string) ([]*fftypes.ChartHistogram, error) { + results := []interface{}{} + + for i := range cols { + results = append(results, &cols[i].Count) } - return strconv.Itoa(total), nil + err := rows.Scan(results...) + if err != nil { + return nil, i18n.NewError(ctx, i18n.MsgDBReadErr, tableName) + } + + return cols, nil } -func (s *SQLCommon) GetChartHistogram(ctx context.Context, ns string, intervals []fftypes.ChartHistogramInterval, collection database.CollectionName) (histogramList []*fftypes.ChartHistogram, err error) { - tableName, fieldMap, err := s.getTableNameFromCollection(ctx, collection) - if err != nil { - return nil, err +func (s *SQLCommon) getHistogramNoTypes(ctx context.Context, ns string, intervals []fftypes.ChartHistogramInterval, tableName string) (histogramList []*fftypes.ChartHistogram, err error) { + qb := sq.Select() + + for i, caseQuery := range s.getCaseQueriesByInterval(ns, intervals) { + query, args, _ := caseQuery.ToSql() + + histogramList = append(histogramList, &fftypes.ChartHistogram{ + Count: "0", + Timestamp: intervals[i].StartTime, + Types: make([]*fftypes.ChartHistogramType, 0), + }) + + qb = qb.Column(sq.Alias(sq.Expr("SUM("+query+")", args...), fmt.Sprintf("case_%d", i))) + } - dataTypes, err := s.getDistinctTypesFromTable(ctx, tableName, fieldMap) + rows, _, err := s.query(ctx, qb.From(tableName)) if err != nil { return nil, err } + defer rows.Close() + + if !rows.Next() { + return histogramList, nil + } + return s.histogramResultNoType(ctx, rows, histogramList, tableName) +} + +func (s *SQLCommon) getHistogramWithTypes(ctx context.Context, ns string, intervals []fftypes.ChartHistogramInterval, dataTypes []string, fieldMap map[string]string, tableName string) (histogramList []*fftypes.ChartHistogram, err error) { for _, interval := range intervals { qb := sq.Select() histogramTypes := make([]*fftypes.ChartHistogramType, 0) - for i, caseQuery := range s.getCaseQueries(ns, dataTypes, interval, fieldMap["type"]) { + for i, caseQuery := range s.getCaseQueriesByType(ns, dataTypes, interval, fieldMap["type"]) { query, args, _ := caseQuery.ToSql() histogramTypes = append(histogramTypes, &fftypes.ChartHistogramType{ Count: "", @@ -144,7 +203,7 @@ func (s *SQLCommon) GetChartHistogram(ctx context.Context, ns string, intervals defer rows.Close() if rows.Next() { - hist, err := s.histogramResult(ctx, rows, histogramTypes, tableName) + hist, err := s.histogramResultWithTypes(ctx, rows, histogramTypes, tableName) rows.Close() if err != nil { return nil, err @@ -171,3 +230,31 @@ func (s *SQLCommon) GetChartHistogram(ctx context.Context, ns string, intervals return histogramList, nil } + +func (s *SQLCommon) GetChartHistogram(ctx context.Context, ns string, intervals []fftypes.ChartHistogramInterval, collection database.CollectionName) (histogramList []*fftypes.ChartHistogram, err error) { + tableName, fieldMap, err := s.getTableNameFromCollection(ctx, collection) + if err != nil { + return nil, err + } + + dataTypes, err := s.getDistinctTypesFromTable(ctx, tableName, fieldMap) + if err != nil { + return nil, err + } + + if len(dataTypes) > 0 { + histogramList, err = s.getHistogramWithTypes(ctx, ns, intervals, dataTypes, fieldMap, tableName) + if err != nil { + return nil, err + } + + return histogramList, nil + } + + histogramList, err = s.getHistogramNoTypes(ctx, ns, intervals, tableName) + if err != nil { + return nil, err + } + + return histogramList, nil +} diff --git a/internal/database/sqlcommon/chart_sql_test.go b/internal/database/sqlcommon/chart_sql_test.go index 3a790fb758..38639b3c9a 100644 --- a/internal/database/sqlcommon/chart_sql_test.go +++ b/internal/database/sqlcommon/chart_sql_test.go @@ -51,6 +51,13 @@ var ( }, }, } + expectedHistogramResultNoTypes = []*fftypes.ChartHistogram{ + { + Count: "10", + Timestamp: fftypes.UnixTime(1000000000), + Types: make([]*fftypes.ChartHistogramType, 0), + }, + } mockHistogramInterval = []fftypes.ChartHistogramInterval{ { @@ -58,13 +65,16 @@ var ( EndTime: fftypes.UnixTime(1000000001), }, } - validCollections = []string{ + validCollectionsWithTypes = []string{ "events", "messages", "operations", "transactions", "tokentransfers", } + validCollectionsNoTypes = []string{ + "blockchainevents", + } ) func TestGetChartHistogramInvalidCollectionName(t *testing.T) { @@ -74,13 +84,13 @@ func TestGetChartHistogramInvalidCollectionName(t *testing.T) { assert.Regexp(t, "FF10301", err) } -func TestGetChartHistogramValidCollectionName(t *testing.T) { - for i := range validCollections { +func TestGetChartHistogramValidCollectionNameWithTypes(t *testing.T) { + for i := range validCollectionsWithTypes { s, mock := newMockProvider().init() mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB")) mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "case_1"}).AddRow("5", "5")) - histogram, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName(validCollections[i])) + histogram, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName(validCollectionsWithTypes[i])) assert.NoError(t, err) assert.Equal(t, histogram, expectedHistogramResult) @@ -88,6 +98,18 @@ func TestGetChartHistogramValidCollectionName(t *testing.T) { } } +func TestGetChartHistogramValidCollectionNameNoTypes(t *testing.T) { + for i := range validCollectionsNoTypes { + s, mock := newMockProvider().init() + mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0"}).AddRow("10")) + + histogram, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName(validCollectionsNoTypes[i])) + assert.NoError(t, err) + assert.Equal(t, expectedHistogramResultNoTypes, histogram) + assert.NoError(t, mock.ExpectationsWereMet()) + } +} + func TestGetChartHistogramsQueryFail(t *testing.T) { s, mock := newMockProvider().init() mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB")) @@ -98,6 +120,15 @@ func TestGetChartHistogramsQueryFail(t *testing.T) { assert.NoError(t, mock.ExpectationsWereMet()) } +func TestGetChartHistogramsQueryFailNoTypes(t *testing.T) { + s, mock := newMockProvider().init() + mock.ExpectQuery("SELECT *").WillReturnError(fmt.Errorf("pop")) + + _, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("blockchainevents")) + assert.Regexp(t, "FF10115", err) + assert.NoError(t, mock.ExpectationsWereMet()) +} + func TestGetChartHistogramQueryFailBadDistinctTypes(t *testing.T) { s, mock := newMockProvider().init() mock.ExpectQuery("SELECT DISTINCT .*").WillReturnError(fmt.Errorf("pop")) @@ -126,6 +157,15 @@ func TestGetChartHistogramScanFailTooManyCols(t *testing.T) { assert.NoError(t, mock.ExpectationsWereMet()) } +func TestGetChartHistogramScanFailTooManyColsNoTypes(t *testing.T) { + s, mock := newMockProvider().init() + mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "unexpected"}).AddRow("10", "abc")) + + _, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("blockchainevents")) + assert.Regexp(t, "FF10121", err) + assert.NoError(t, mock.ExpectationsWereMet()) +} + func TestGetChartHistogramFailStringToIntConversion(t *testing.T) { s, mock := newMockProvider().init() mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB")) @@ -147,6 +187,27 @@ func TestGetChartHistogramSuccessNoRows(t *testing.T) { assert.NoError(t, mock.ExpectationsWereMet()) } +func TestGetChartHistogramSuccessNoRowsNoTypes(t *testing.T) { + s, mock := newMockProvider().init() + mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "case_1"})) + + histogram, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("blockchainevents")) + assert.NoError(t, err) + + assert.Equal(t, emptyHistogramResult, histogram) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetChartHistogramSuccessNoTypes(t *testing.T) { + s, mock := newMockProvider().init() + mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0"}).AddRow("10")) + + histogram, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("blockchainevents")) + assert.NoError(t, err) + assert.Equal(t, expectedHistogramResultNoTypes, histogram) + assert.NoError(t, mock.ExpectationsWereMet()) +} + func TestGetChartHistogramSuccess(t *testing.T) { s, mock := newMockProvider().init() mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB")) diff --git a/pkg/database/plugin.go b/pkg/database/plugin.go index 5b07f5644d..6c64c79814 100644 --- a/pkg/database/plugin.go +++ b/pkg/database/plugin.go @@ -577,7 +577,7 @@ type OrderedUUIDCollectionNS CollectionName const ( CollectionMessages OrderedUUIDCollectionNS = "messages" CollectionEvents OrderedUUIDCollectionNS = "events" - CollectionBlockchainEvents OrderedUUIDCollectionNS = "contractevents" + CollectionBlockchainEvents OrderedUUIDCollectionNS = "blockchainevents" ) // OrderedCollection is a collection that is ordered, and that sequence is the only key