Skip to content

[blockchainevents-charts] support for blockchain event charting #612

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 18, 2022
Merged

[blockchainevents-charts] support for blockchain event charting #612

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 106 additions & 19 deletions internal/database/sqlcommon/chart_sql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,28 @@ import (
"github.com/hyperledger/firefly/pkg/fftypes"
)

func (s *SQLCommon) getCaseQueries(ns string, dataTypes []string, interval fftypes.ChartHistogramInterval, typeColName string) (caseQueries []sq.CaseBuilder) {
func (s *SQLCommon) getCaseQueriesByInterval(ns string, intervals []fftypes.ChartHistogramInterval) (caseQueries []sq.CaseBuilder) {
for _, interval := range intervals {
caseQueries = append(caseQueries, sq.Case().
When(
sq.And{
// Querying by 'timestamp' field for blockchain events
// If more tables are supported that have no "type" field,
// and a different date field name,
// this method will need to be refactored
sq.GtOrEq{"timestamp": interval.StartTime},
sq.Lt{"timestamp": interval.EndTime},
sq.Eq{"namespace": ns},
},
"1",
).
Else("0"))
}

return caseQueries
}

func (s *SQLCommon) getCaseQueriesByType(ns string, dataTypes []string, interval fftypes.ChartHistogramInterval, typeColName string) (caseQueries []sq.CaseBuilder) {
for _, dataType := range dataTypes {
caseQueries = append(caseQueries, sq.Case().
When(
Expand Down Expand Up @@ -57,13 +78,18 @@ func (s *SQLCommon) getTableNameFromCollection(ctx context.Context, collection d
case database.CollectionName(database.CollectionEvents):
return "events", eventFilterFieldMap, nil
case database.CollectionName(database.CollectionTokenTransfers):
return "tokentransfers", tokenTransferFilterFieldMap, nil
return "tokentransfer", tokenTransferFilterFieldMap, nil
case database.CollectionName(database.CollectionBlockchainEvents):
return "blockchainevents", blockchainEventFilterFieldMap, nil
default:
return "", nil, i18n.NewError(ctx, i18n.MsgUnsupportedCollection, collection)
}
}

func (s *SQLCommon) getDistinctTypesFromTable(ctx context.Context, tableName string, fieldMap map[string]string) ([]string, error) {
if _, ok := fieldMap["type"]; !ok {
return []string{}, nil
}
qb := sq.Select(fieldMap["type"]).Distinct().From(tableName)

rows, _, err := s.query(ctx, qb.From(tableName))
Expand All @@ -86,7 +112,19 @@ func (s *SQLCommon) getDistinctTypesFromTable(ctx context.Context, tableName str
return dataTypes, nil
}

func (s *SQLCommon) histogramResult(ctx context.Context, rows *sql.Rows, cols []*fftypes.ChartHistogramType, tableName string) ([]*fftypes.ChartHistogramType, error) {
func (s *SQLCommon) getBucketTotal(typeBuckets []*fftypes.ChartHistogramType) (string, error) {
total := 0
for _, typeBucket := range typeBuckets {
typeBucketInt, err := strconv.Atoi(typeBucket.Count)
if err != nil {
return "", err
}
total += typeBucketInt
}
return strconv.Itoa(total), nil
}

func (s *SQLCommon) histogramResultWithTypes(ctx context.Context, rows *sql.Rows, cols []*fftypes.ChartHistogramType, tableName string) ([]*fftypes.ChartHistogramType, error) {
results := []interface{}{}

for i := range cols {
Expand All @@ -100,34 +138,55 @@ func (s *SQLCommon) histogramResult(ctx context.Context, rows *sql.Rows, cols []
return cols, nil
}

func (s *SQLCommon) getBucketTotal(typeBuckets []*fftypes.ChartHistogramType) (string, error) {
total := 0
for _, typeBucket := range typeBuckets {
typeBucketInt, err := strconv.Atoi(typeBucket.Count)
if err != nil {
return "", err
}
total += typeBucketInt
func (s *SQLCommon) histogramResultNoType(ctx context.Context, rows *sql.Rows, cols []*fftypes.ChartHistogram, tableName string) ([]*fftypes.ChartHistogram, error) {
results := []interface{}{}

for i := range cols {
results = append(results, &cols[i].Count)
}
return strconv.Itoa(total), nil
err := rows.Scan(results...)
if err != nil {
return nil, i18n.NewError(ctx, i18n.MsgDBReadErr, tableName)
}

return cols, nil
}

func (s *SQLCommon) GetChartHistogram(ctx context.Context, ns string, intervals []fftypes.ChartHistogramInterval, collection database.CollectionName) (histogramList []*fftypes.ChartHistogram, err error) {
tableName, fieldMap, err := s.getTableNameFromCollection(ctx, collection)
if err != nil {
return nil, err
func (s *SQLCommon) getHistogramNoTypes(ctx context.Context, ns string, intervals []fftypes.ChartHistogramInterval, tableName string) (histogramList []*fftypes.ChartHistogram, err error) {
qb := sq.Select()

for i, caseQuery := range s.getCaseQueriesByInterval(ns, intervals) {
query, args, _ := caseQuery.ToSql()

histogramList = append(histogramList, &fftypes.ChartHistogram{
Count: "0",
Timestamp: intervals[i].StartTime,
Types: make([]*fftypes.ChartHistogramType, 0),
})

qb = qb.Column(sq.Alias(sq.Expr("SUM("+query+")", args...), fmt.Sprintf("case_%d", i)))

}

dataTypes, err := s.getDistinctTypesFromTable(ctx, tableName, fieldMap)
rows, _, err := s.query(ctx, qb.From(tableName))
if err != nil {
return nil, err
}
defer rows.Close()

if !rows.Next() {
return histogramList, nil
}

return s.histogramResultNoType(ctx, rows, histogramList, tableName)
}

func (s *SQLCommon) getHistogramWithTypes(ctx context.Context, ns string, intervals []fftypes.ChartHistogramInterval, dataTypes []string, fieldMap map[string]string, tableName string) (histogramList []*fftypes.ChartHistogram, err error) {
for _, interval := range intervals {
qb := sq.Select()
histogramTypes := make([]*fftypes.ChartHistogramType, 0)

for i, caseQuery := range s.getCaseQueries(ns, dataTypes, interval, fieldMap["type"]) {
for i, caseQuery := range s.getCaseQueriesByType(ns, dataTypes, interval, fieldMap["type"]) {
query, args, _ := caseQuery.ToSql()
histogramTypes = append(histogramTypes, &fftypes.ChartHistogramType{
Count: "",
Expand All @@ -144,7 +203,7 @@ func (s *SQLCommon) GetChartHistogram(ctx context.Context, ns string, intervals
defer rows.Close()

if rows.Next() {
hist, err := s.histogramResult(ctx, rows, histogramTypes, tableName)
hist, err := s.histogramResultWithTypes(ctx, rows, histogramTypes, tableName)
rows.Close()
if err != nil {
return nil, err
Expand All @@ -171,3 +230,31 @@ func (s *SQLCommon) GetChartHistogram(ctx context.Context, ns string, intervals

return histogramList, nil
}

func (s *SQLCommon) GetChartHistogram(ctx context.Context, ns string, intervals []fftypes.ChartHistogramInterval, collection database.CollectionName) (histogramList []*fftypes.ChartHistogram, err error) {
tableName, fieldMap, err := s.getTableNameFromCollection(ctx, collection)
if err != nil {
return nil, err
}

dataTypes, err := s.getDistinctTypesFromTable(ctx, tableName, fieldMap)
if err != nil {
return nil, err
}

if len(dataTypes) > 0 {
histogramList, err = s.getHistogramWithTypes(ctx, ns, intervals, dataTypes, fieldMap, tableName)
if err != nil {
return nil, err
}

return histogramList, nil
}

histogramList, err = s.getHistogramNoTypes(ctx, ns, intervals, tableName)
if err != nil {
return nil, err
}

return histogramList, nil
}
69 changes: 65 additions & 4 deletions internal/database/sqlcommon/chart_sql_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,20 +51,30 @@ var (
},
},
}
expectedHistogramResultNoTypes = []*fftypes.ChartHistogram{
{
Count: "10",
Timestamp: fftypes.UnixTime(1000000000),
Types: make([]*fftypes.ChartHistogramType, 0),
},
}

mockHistogramInterval = []fftypes.ChartHistogramInterval{
{
StartTime: fftypes.UnixTime(1000000000),
EndTime: fftypes.UnixTime(1000000001),
},
}
validCollections = []string{
validCollectionsWithTypes = []string{
"events",
"messages",
"operations",
"transactions",
"tokentransfers",
}
validCollectionsNoTypes = []string{
"blockchainevents",
}
)

func TestGetChartHistogramInvalidCollectionName(t *testing.T) {
Expand All @@ -74,20 +84,32 @@ func TestGetChartHistogramInvalidCollectionName(t *testing.T) {
assert.Regexp(t, "FF10301", err)
}

func TestGetChartHistogramValidCollectionName(t *testing.T) {
for i := range validCollections {
func TestGetChartHistogramValidCollectionNameWithTypes(t *testing.T) {
for i := range validCollectionsWithTypes {
s, mock := newMockProvider().init()
mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB"))
mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "case_1"}).AddRow("5", "5"))

histogram, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName(validCollections[i]))
histogram, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName(validCollectionsWithTypes[i]))

assert.NoError(t, err)
assert.Equal(t, histogram, expectedHistogramResult)
assert.NoError(t, mock.ExpectationsWereMet())
}
}

func TestGetChartHistogramValidCollectionNameNoTypes(t *testing.T) {
for i := range validCollectionsNoTypes {
s, mock := newMockProvider().init()
mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0"}).AddRow("10"))

histogram, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName(validCollectionsNoTypes[i]))
assert.NoError(t, err)
assert.Equal(t, expectedHistogramResultNoTypes, histogram)
assert.NoError(t, mock.ExpectationsWereMet())
}
}

func TestGetChartHistogramsQueryFail(t *testing.T) {
s, mock := newMockProvider().init()
mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB"))
Expand All @@ -98,6 +120,15 @@ func TestGetChartHistogramsQueryFail(t *testing.T) {
assert.NoError(t, mock.ExpectationsWereMet())
}

func TestGetChartHistogramsQueryFailNoTypes(t *testing.T) {
s, mock := newMockProvider().init()
mock.ExpectQuery("SELECT *").WillReturnError(fmt.Errorf("pop"))

_, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("blockchainevents"))
assert.Regexp(t, "FF10115", err)
assert.NoError(t, mock.ExpectationsWereMet())
}

func TestGetChartHistogramQueryFailBadDistinctTypes(t *testing.T) {
s, mock := newMockProvider().init()
mock.ExpectQuery("SELECT DISTINCT .*").WillReturnError(fmt.Errorf("pop"))
Expand Down Expand Up @@ -126,6 +157,15 @@ func TestGetChartHistogramScanFailTooManyCols(t *testing.T) {
assert.NoError(t, mock.ExpectationsWereMet())
}

func TestGetChartHistogramScanFailTooManyColsNoTypes(t *testing.T) {
s, mock := newMockProvider().init()
mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "unexpected"}).AddRow("10", "abc"))

_, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("blockchainevents"))
assert.Regexp(t, "FF10121", err)
assert.NoError(t, mock.ExpectationsWereMet())
}

func TestGetChartHistogramFailStringToIntConversion(t *testing.T) {
s, mock := newMockProvider().init()
mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB"))
Expand All @@ -147,6 +187,27 @@ func TestGetChartHistogramSuccessNoRows(t *testing.T) {
assert.NoError(t, mock.ExpectationsWereMet())
}

func TestGetChartHistogramSuccessNoRowsNoTypes(t *testing.T) {
s, mock := newMockProvider().init()
mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0", "case_1"}))

histogram, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("blockchainevents"))
assert.NoError(t, err)

assert.Equal(t, emptyHistogramResult, histogram)
assert.NoError(t, mock.ExpectationsWereMet())
}

func TestGetChartHistogramSuccessNoTypes(t *testing.T) {
s, mock := newMockProvider().init()
mock.ExpectQuery("SELECT .*").WillReturnRows(sqlmock.NewRows([]string{"case_0"}).AddRow("10"))

histogram, err := s.GetChartHistogram(context.Background(), "ns1", mockHistogramInterval, database.CollectionName("blockchainevents"))
assert.NoError(t, err)
assert.Equal(t, expectedHistogramResultNoTypes, histogram)
assert.NoError(t, mock.ExpectationsWereMet())
}

func TestGetChartHistogramSuccess(t *testing.T) {
s, mock := newMockProvider().init()
mock.ExpectQuery("SELECT DISTINCT .*").WillReturnRows(sqlmock.NewRows([]string{"type"}).AddRow("typeA").AddRow("typeB"))
Expand Down
2 changes: 1 addition & 1 deletion pkg/database/plugin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -577,7 +577,7 @@ type OrderedUUIDCollectionNS CollectionName
const (
CollectionMessages OrderedUUIDCollectionNS = "messages"
CollectionEvents OrderedUUIDCollectionNS = "events"
CollectionBlockchainEvents OrderedUUIDCollectionNS = "contractevents"
CollectionBlockchainEvents OrderedUUIDCollectionNS = "blockchainevents"
)

// OrderedCollection is a collection that is ordered, and that sequence is the only key
Expand Down