Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: add support for table view in traces #3047

Merged
merged 14 commits into from
Jul 12, 2023
Merged

feat: add support for table view in traces #3047

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
15ae7c9
feat: add support for table view in traces
makeavish Jul 6, 2023
2d27dc4
fix: alignment issue
makeavish Jul 6, 2023
e4202f4
Merge branch 'develop' into feat/support-table-view
makeavish Jul 7, 2023
00008d2
Merge branch 'develop' into feat/support-table-view
makeavish Jul 10, 2023
d27841d
feat: handle table view in traces
makeavish Jul 10, 2023
ad716bd
Merge branch 'develop' into feat/support-table-view
makeavish Jul 10, 2023
afc302d
Merge branch 'develop' into feat/support-table-view
makeavish Jul 11, 2023
335beb7
fix: value type panel
makeavish Jul 11, 2023
cfc2e5b
Merge branch 'develop' into feat/support-table-view
makeavish Jul 11, 2023
cb33630
fix: order by in table view
makeavish Jul 12, 2023
b444c5d
Merge branch 'develop' into feat/support-table-view
makeavish Jul 12, 2023
3c64fbe
chore: remove obsolete code
makeavish Jul 12, 2023
ee70682
fix: use now() as ts in query to support formula
makeavish Jul 12, 2023
06e07b6
test: update tests
makeavish Jul 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 42 additions & 23 deletions pkg/query-service/app/traces/v3/query_builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.Attri
} else {
for _, tag := range groupBy {
filterName := getColumnName(tag, keys)
selectLabels += fmt.Sprintf(", %s as `%s`", filterName, tag.Key)
selectLabels += fmt.Sprintf(" %s as `%s`,", filterName, tag.Key)
}
}
return selectLabels
Expand Down Expand Up @@ -235,25 +235,42 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str
having = " having " + having
}

// Select the aggregate value for interval
queryTmpl :=
"SELECT toStartOfInterval(timestamp, INTERVAL %d SECOND) AS ts" + selectLabels +
", %s as value " +
"from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME +
" where " + spanIndexTableTimeFilter + "%s " +
"group by %s%s " +
"order by %s"
var queryTmpl string

if panelType == v3.PanelTypeTable {
queryTmpl =
"SELECT now() as ts," + selectLabels +
" %s as value " +
"from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME +
" where " + spanIndexTableTimeFilter + "%s" +
"%s%s" +
"%s"
} else if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue {
// Select the aggregate value for interval
queryTmpl =
fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d SECOND) AS ts,", step) + selectLabels +
" %s as value " +
"from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME +
" where " + spanIndexTableTimeFilter + "%s" +
"%s%s" +
"%s"
}

emptyValuesInGroupByFilter, err := handleEmptyValuesInGroupBy(keys, mq.GroupBy)
if err != nil {
return "", err
}
filterSubQuery += emptyValuesInGroupByFilter

groupBy := groupByAttributeKeyTags(keys, mq.GroupBy...)
groupBy := groupByAttributeKeyTags(panelType, mq.GroupBy...)
if groupBy != "" {
groupBy = " group by " + groupBy
}
enrichedOrderBy := enrichOrderBy(mq.OrderBy, keys)
orderBy := orderByAttributeKeyTags(panelType, enrichedOrderBy, mq.GroupBy, keys)

if orderBy != "" {
orderBy = " order by " + orderBy
}
aggregationKey := ""
if mq.AggregateAttribute.Key != "" {
aggregationKey = getColumnName(mq.AggregateAttribute, keys)
Expand All @@ -266,7 +283,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str
v3.AggregateOperatorRateMin,
v3.AggregateOperatorRate:
op := fmt.Sprintf("%s(%s)/%d", aggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey, step)
query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy)
query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy)
return query, nil
case
v3.AggregateOperatorP05,
Expand All @@ -279,11 +296,11 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str
v3.AggregateOperatorP95,
v3.AggregateOperatorP99:
op := fmt.Sprintf("quantile(%v)(%s)", aggregateOperatorToPercentile[mq.AggregateOperator], aggregationKey)
query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy)
query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy)
return query, nil
case v3.AggregateOperatorAvg, v3.AggregateOperatorSum, v3.AggregateOperatorMin, v3.AggregateOperatorMax:
op := fmt.Sprintf("%s(%s)", aggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey)
query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy)
query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy)
return query, nil
case v3.AggregateOperatorCount:
if mq.AggregateAttribute.Key != "" {
Expand All @@ -299,11 +316,11 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str
}
}
op := "toFloat64(count())"
query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy)
query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy)
return query, nil
case v3.AggregateOperatorCountDistinct:
op := fmt.Sprintf("toFloat64(count(distinct(%s)))", aggregationKey)
query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy)
query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy)
return query, nil
case v3.AggregateOperatorNoOp:
var query string
Expand All @@ -319,7 +336,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str
return "", fmt.Errorf("select columns cannot be empty for panelType %s", panelType)
}
selectColumns := getSelectColumns(mq.SelectColumns, keys)
queryNoOpTmpl := fmt.Sprintf("SELECT timestamp as timestamp_datetime, spanID, traceID, "+"%s ", selectColumns) + "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME + " where %s %s" + " order by %s"
queryNoOpTmpl := fmt.Sprintf("SELECT timestamp as timestamp_datetime, spanID, traceID, "+"%s ", selectColumns) + "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME + " where %s %s" + "%s"
query = fmt.Sprintf(queryNoOpTmpl, spanIndexTableTimeFilter, filterSubQuery, orderBy)
} else {
return "", fmt.Errorf("unsupported aggregate operator %s for panelType %s", mq.AggregateOperator, panelType)
Expand Down Expand Up @@ -350,17 +367,19 @@ func enrichOrderBy(items []v3.OrderBy, keys map[string]v3.AttributeKey) []v3.Ord

// groupBy returns a string of comma separated tags for group by clause
// `ts` is always added to the group by clause
func groupBy(tags ...string) string {
tags = append(tags, "ts")
func groupBy(panelType v3.PanelType, tags ...string) string {
if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue {
tags = append(tags, "ts")
}
return strings.Join(tags, ",")
}

func groupByAttributeKeyTags(keys map[string]v3.AttributeKey, tags ...v3.AttributeKey) string {
func groupByAttributeKeyTags(panelType v3.PanelType, tags ...v3.AttributeKey) string {
groupTags := []string{}
for _, tag := range tags {
groupTags = append(groupTags, fmt.Sprintf("`%s`", tag.Key))
}
return groupBy(groupTags...)
return groupBy(panelType, groupTags...)
}

// orderBy returns a string of comma separated tags for order by clause
Expand Down Expand Up @@ -403,7 +422,7 @@ func orderBy(panelType v3.PanelType, items []v3.OrderBy, tags []string, keys map
if !addedToOrderBy[item.ColumnName] {
attr := v3.AttributeKey{Key: item.ColumnName, DataType: item.DataType, Type: item.Type, IsColumn: item.IsColumn}
name := getColumnName(attr, keys)

if item.IsColumn {
orderBy = append(orderBy, fmt.Sprintf("`%s` %s", name, item.Order))
} else {
Expand All @@ -424,7 +443,7 @@ func orderByAttributeKeyTags(panelType v3.PanelType, items []v3.OrderBy, tags []

if panelType == v3.PanelTypeList && len(orderByArray) == 0 {
orderByArray = append(orderByArray, constants.TIMESTAMP+" DESC")
} else if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeTable {
} else if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue {
orderByArray = append(orderByArray, "ts")
}

Expand Down
79 changes: 75 additions & 4 deletions pkg/query-service/app/traces/v3/query_builder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,13 +216,13 @@ var testGetSelectLabelsData = []struct {
Name: "select keys for groupBy attribute",
AggregateOperator: v3.AggregateOperatorCount,
GroupByTags: []v3.AttributeKey{{Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
SelectLabels: ", stringTagMap['user.name'] as `user.name`",
SelectLabels: " stringTagMap['user.name'] as `user.name`,",
},
{
Name: "select keys for groupBy resource",
AggregateOperator: v3.AggregateOperatorCount,
GroupByTags: []v3.AttributeKey{{Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}},
SelectLabels: ", resourceTagsMap['user.name'] as `user.name`",
SelectLabels: " resourceTagsMap['user.name'] as `user.name`,",
},
{
Name: "select keys for groupBy attribute and resource",
Expand All @@ -231,13 +231,13 @@ var testGetSelectLabelsData = []struct {
{Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource},
{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
},
SelectLabels: ", resourceTagsMap['user.name'] as `user.name`, stringTagMap['host'] as `host`",
SelectLabels: " resourceTagsMap['user.name'] as `user.name`, stringTagMap['host'] as `host`,",
},
{
Name: "select keys for groupBy fixed columns",
AggregateOperator: v3.AggregateOperatorCount,
GroupByTags: []v3.AttributeKey{{Key: "host", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
SelectLabels: ", host as `host`",
SelectLabels: " host as `host`,",
},
}

Expand Down Expand Up @@ -956,6 +956,77 @@ var testBuildTracesQueryData = []struct {
"AND stringTagMap['method'] = 'GET' group by ts having value > 10 order by ts",
PanelType: v3.PanelTypeGraph,
},
{
Name: "Test count with having clause and filters",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorCount,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="},
},
},
Having: []v3.Having{
{
ColumnName: "name",
Operator: ">",
Value: 10,
},
},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['method'] = 'GET' AND has(stringTagMap, 'name') group by ts having value > 10 order by ts",
PanelType: v3.PanelTypeValue,
},
{
Name: "Test aggregate PXX",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "durationNano", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorP05,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}},
GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT now() as ts, stringTagMap['method'] as `method`, " +
"quantile(0.05)(durationNano) as value " +
"from signoz_traces.distributed_signoz_index_v2 " +
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND has(stringTagMap, 'method') group by `method` " +
"order by `method` ASC",
PanelType: v3.PanelTypeTable,
},
{
Name: "Test aggregate PXX",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "durationNano", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorP05,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}},
GroupBy: []v3.AttributeKey{},
OrderBy: []v3.OrderBy{},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT now() as ts, quantile(0.05)(durationNano) as value " +
"from signoz_traces.distributed_signoz_index_v2 " +
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')",
PanelType: v3.PanelTypeTable,
},
{
Name: "Test Noop list view",
Start: 1680066360726210000,
Expand Down
Loading