Skip to content

Commit

Permalink
telemetry: interval partitioning (#36486)
Browse files Browse the repository at this point in the history 
close #36525
  • Loading branch information
@mjonss
mjonss committed Sep 14, 2022
1 parent 51e6536 commit f0d8d47
Show file tree
Hide file tree
Showing 5 changed files with 148 additions and 71 deletions.
3 changes: 3 additions & 0 deletions executor/adapter.go
View file
Expand Up @@ -206,6 +206,9 @@ type PartitionTelemetryInfo struct {
UseTablePartitionRangeColumns bool
UseTablePartitionListColumns bool
TablePartitionMaxPartitionsNum uint64
UseCreateIntervalPartition bool
UseAddIntervalPartition bool
UseDropIntervalPartition bool
}

// AccountLockTelemetryInfo records account lock/unlock information during execution
Expand Down
89 changes: 55 additions & 34 deletions executor/builder.go
View file
Expand Up @@ -1017,51 +1017,72 @@ func (b *executorBuilder) buildRevoke(revoke *ast.RevokeStmt) Executor {
return e
}

func (b *executorBuilder) buildDDL(v *plannercore.DDL) Executor {
switch v.Statement.(type) {
func (b *executorBuilder) setTelemetryInfo(v *plannercore.DDL) {
if v == nil || b.Ti == nil {
return
}
switch s := v.Statement.(type) {
case *ast.AlterTableStmt:
if len(v.Statement.(*ast.AlterTableStmt).Specs) > 1 && b.Ti != nil {
if len(s.Specs) > 1 {
b.Ti.UseMultiSchemaChange = true
}
case *ast.CreateTableStmt:
stmt := v.Statement.(*ast.CreateTableStmt)
if stmt != nil && stmt.Partition != nil {
if strings.EqualFold(b.ctx.GetSessionVars().EnableTablePartition, "OFF") {
break
for _, spec := range s.Specs {
switch spec.Tp {
case ast.AlterTableDropFirstPartition:
if b.Ti.PartitionTelemetry == nil {
b.Ti.PartitionTelemetry = &PartitionTelemetryInfo{}
}
b.Ti.PartitionTelemetry.UseDropIntervalPartition = true
case ast.AlterTableAddLastPartition:
if b.Ti.PartitionTelemetry == nil {
b.Ti.PartitionTelemetry = &PartitionTelemetryInfo{}
}
b.Ti.PartitionTelemetry.UseAddIntervalPartition = true
}
}
case *ast.CreateTableStmt:
if s.Partition == nil || strings.EqualFold(b.ctx.GetSessionVars().EnableTablePartition, "OFF") {
break
}

s := stmt.Partition
if b.Ti.PartitionTelemetry == nil {
b.Ti.PartitionTelemetry = &PartitionTelemetryInfo{}
}
b.Ti.PartitionTelemetry.TablePartitionMaxPartitionsNum = mathutil.Max(s.Num, uint64(len(s.Definitions)))
b.Ti.PartitionTelemetry.UseTablePartition = true

switch s.Tp {
case model.PartitionTypeRange:
if s.Sub == nil {
if len(s.ColumnNames) > 0 {
b.Ti.PartitionTelemetry.UseTablePartitionRangeColumns = true
} else {
b.Ti.PartitionTelemetry.UseTablePartitionRange = true
}
p := s.Partition
if b.Ti.PartitionTelemetry == nil {
b.Ti.PartitionTelemetry = &PartitionTelemetryInfo{}
}
b.Ti.PartitionTelemetry.TablePartitionMaxPartitionsNum = mathutil.Max(p.Num, uint64(len(p.Definitions)))
b.Ti.PartitionTelemetry.UseTablePartition = true

switch p.Tp {
case model.PartitionTypeRange:
if p.Sub == nil {
if len(p.ColumnNames) > 0 {
b.Ti.PartitionTelemetry.UseTablePartitionRangeColumns = true
} else {
b.Ti.PartitionTelemetry.UseTablePartitionRange = true
}
case model.PartitionTypeHash:
if !s.Linear && s.Sub == nil {
b.Ti.PartitionTelemetry.UseTablePartitionHash = true
if p.Interval != nil {
b.Ti.PartitionTelemetry.UseCreateIntervalPartition = true
}
case model.PartitionTypeList:
enable := b.ctx.GetSessionVars().EnableListTablePartition
if s.Sub == nil && enable {
if len(s.ColumnNames) > 0 {
b.Ti.PartitionTelemetry.UseTablePartitionListColumns = true
} else {
b.Ti.PartitionTelemetry.UseTablePartitionList = true
}
}
case model.PartitionTypeHash:
if !p.Linear && p.Sub == nil {
b.Ti.PartitionTelemetry.UseTablePartitionHash = true
}
case model.PartitionTypeList:
enable := b.ctx.GetSessionVars().EnableListTablePartition
if p.Sub == nil && enable {
if len(p.ColumnNames) > 0 {
b.Ti.PartitionTelemetry.UseTablePartitionListColumns = true
} else {
b.Ti.PartitionTelemetry.UseTablePartitionList = true
}
}
}
}
}

func (b *executorBuilder) buildDDL(v *plannercore.DDL) Executor {
b.setTelemetryInfo(v)

e := &DDLExec{
baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()),
Expand Down
72 changes: 51 additions & 21 deletions metrics/telemetry.go
View file
Expand Up @@ -92,6 +92,27 @@ var (
Name: "account_lock_usage",
Help: "Counter of locked/unlocked users",
}, []string{LblAccountLock})
TelemetryTablePartitionCreateIntervalPartitionsCnt = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: "tidb",
Subsystem: "telemetry",
Name: "table_partition_create_interval_partition_usage",
Help: "Counter of partitions created by CREATE TABLE INTERVAL statements",
})
TelemetryTablePartitionAddIntervalPartitionsCnt = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: "tidb",
Subsystem: "telemetry",
Name: "table_partition_add_interval_partition_usage",
Help: "Counter of partitions added by ALTER TABLE LAST PARTITION statements",
})
TelemetryTablePartitionDropIntervalPartitionsCnt = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: "tidb",
Subsystem: "telemetry",
Name: "table_partition_drop_interval_partition_usage",
Help: "Counter of partitions added by ALTER TABLE FIRST PARTITION statements",
})
)

// readCounter reads the value of a prometheus.Counter.
Expand Down Expand Up @@ -178,25 +199,31 @@ func GetMultiSchemaCounter() MultiSchemaChangeUsageCounter {

// TablePartitionUsageCounter records the usages of table partition.
type TablePartitionUsageCounter struct {
TablePartitionCnt int64 `json:"table_partition_cnt"`
TablePartitionListCnt int64 `json:"table_partition_list_cnt"`
TablePartitionRangeCnt int64 `json:"table_partition_range_cnt"`
TablePartitionHashCnt int64 `json:"table_partition_hash_cnt"`
TablePartitionRangeColumnsCnt int64 `json:"table_partition_range_columns_cnt"`
TablePartitionListColumnsCnt int64 `json:"table_partition_list_columns_cnt"`
TablePartitionMaxPartitionsCnt int64 `json:"table_partition_max_partitions_cnt"`
TablePartitionCnt int64 `json:"table_partition_cnt"`
TablePartitionListCnt int64 `json:"table_partition_list_cnt"`
TablePartitionRangeCnt int64 `json:"table_partition_range_cnt"`
TablePartitionHashCnt int64 `json:"table_partition_hash_cnt"`
TablePartitionRangeColumnsCnt int64 `json:"table_partition_range_columns_cnt"`
TablePartitionListColumnsCnt int64 `json:"table_partition_list_columns_cnt"`
TablePartitionMaxPartitionsCnt int64 `json:"table_partition_max_partitions_cnt"`
TablePartitionCreateIntervalPartitionsCnt int64 `json:"table_partition_create_interval_partitions_cnt"`
TablePartitionAddIntervalPartitionsCnt int64 `json:"table_partition_add_interval_partitions_cnt"`
TablePartitionDropIntervalPartitionsCnt int64 `json:"table_partition_drop_interval_partitions_cnt"`
}

// Cal returns the difference of two counters.
func (c TablePartitionUsageCounter) Cal(rhs TablePartitionUsageCounter) TablePartitionUsageCounter {
return TablePartitionUsageCounter{
TablePartitionCnt: c.TablePartitionCnt - rhs.TablePartitionCnt,
TablePartitionListCnt: c.TablePartitionListCnt - rhs.TablePartitionListCnt,
TablePartitionRangeCnt: c.TablePartitionRangeCnt - rhs.TablePartitionRangeCnt,
TablePartitionHashCnt: c.TablePartitionHashCnt - rhs.TablePartitionHashCnt,
TablePartitionRangeColumnsCnt: c.TablePartitionRangeColumnsCnt - rhs.TablePartitionRangeColumnsCnt,
TablePartitionListColumnsCnt: c.TablePartitionListColumnsCnt - rhs.TablePartitionListColumnsCnt,
TablePartitionMaxPartitionsCnt: mathutil.Max(c.TablePartitionMaxPartitionsCnt-rhs.TablePartitionMaxPartitionsCnt, rhs.TablePartitionMaxPartitionsCnt),
TablePartitionCnt: c.TablePartitionCnt - rhs.TablePartitionCnt,
TablePartitionListCnt: c.TablePartitionListCnt - rhs.TablePartitionListCnt,
TablePartitionRangeCnt: c.TablePartitionRangeCnt - rhs.TablePartitionRangeCnt,
TablePartitionHashCnt: c.TablePartitionHashCnt - rhs.TablePartitionHashCnt,
TablePartitionRangeColumnsCnt: c.TablePartitionRangeColumnsCnt - rhs.TablePartitionRangeColumnsCnt,
TablePartitionListColumnsCnt: c.TablePartitionListColumnsCnt - rhs.TablePartitionListColumnsCnt,
TablePartitionMaxPartitionsCnt: mathutil.Max(c.TablePartitionMaxPartitionsCnt-rhs.TablePartitionMaxPartitionsCnt, rhs.TablePartitionMaxPartitionsCnt),
TablePartitionCreateIntervalPartitionsCnt: c.TablePartitionCreateIntervalPartitionsCnt - rhs.TablePartitionCreateIntervalPartitionsCnt,
TablePartitionAddIntervalPartitionsCnt: c.TablePartitionAddIntervalPartitionsCnt - rhs.TablePartitionAddIntervalPartitionsCnt,
TablePartitionDropIntervalPartitionsCnt: c.TablePartitionDropIntervalPartitionsCnt - rhs.TablePartitionDropIntervalPartitionsCnt,
}
}

Expand All @@ -216,13 +243,16 @@ func ResetTablePartitionCounter(pre TablePartitionUsageCounter) TablePartitionUs
// GetTablePartitionCounter gets the TxnCommitCounter.
func GetTablePartitionCounter() TablePartitionUsageCounter {
return TablePartitionUsageCounter{
TablePartitionCnt: readCounter(TelemetryTablePartitionCnt),
TablePartitionListCnt: readCounter(TelemetryTablePartitionListCnt),
TablePartitionRangeCnt: readCounter(TelemetryTablePartitionRangeCnt),
TablePartitionHashCnt: readCounter(TelemetryTablePartitionHashCnt),
TablePartitionRangeColumnsCnt: readCounter(TelemetryTablePartitionRangeColumnsCnt),
TablePartitionListColumnsCnt: readCounter(TelemetryTablePartitionListColumnsCnt),
TablePartitionMaxPartitionsCnt: readCounter(TelemetryTablePartitionMaxPartitionsCnt),
TablePartitionCnt: readCounter(TelemetryTablePartitionCnt),
TablePartitionListCnt: readCounter(TelemetryTablePartitionListCnt),
TablePartitionRangeCnt: readCounter(TelemetryTablePartitionRangeCnt),
TablePartitionHashCnt: readCounter(TelemetryTablePartitionHashCnt),
TablePartitionRangeColumnsCnt: readCounter(TelemetryTablePartitionRangeColumnsCnt),
TablePartitionListColumnsCnt: readCounter(TelemetryTablePartitionListColumnsCnt),
TablePartitionMaxPartitionsCnt: readCounter(TelemetryTablePartitionMaxPartitionsCnt),
TablePartitionCreateIntervalPartitionsCnt: readCounter(TelemetryTablePartitionCreateIntervalPartitionsCnt),
TablePartitionAddIntervalPartitionsCnt: readCounter(TelemetryTablePartitionAddIntervalPartitionsCnt),
TablePartitionDropIntervalPartitionsCnt: readCounter(TelemetryTablePartitionDropIntervalPartitionsCnt),
}
}

Expand Down
40 changes: 26 additions & 14 deletions session/session.go
View file
Expand Up @@ -121,20 +121,23 @@ var (
sessionExecuteParseDurationInternal = metrics.SessionExecuteParseDuration.WithLabelValues(metrics.LblInternal)
sessionExecuteParseDurationGeneral = metrics.SessionExecuteParseDuration.WithLabelValues(metrics.LblGeneral)

telemetryCTEUsageRecurCTE = metrics.TelemetrySQLCTECnt.WithLabelValues("recurCTE")
telemetryCTEUsageNonRecurCTE = metrics.TelemetrySQLCTECnt.WithLabelValues("nonRecurCTE")
telemetryCTEUsageNotCTE = metrics.TelemetrySQLCTECnt.WithLabelValues("notCTE")
telemetryMultiSchemaChangeUsage = metrics.TelemetryMultiSchemaChangeCnt
telemetryTablePartitionUsage = metrics.TelemetryTablePartitionCnt
telemetryTablePartitionListUsage = metrics.TelemetryTablePartitionListCnt
telemetryTablePartitionRangeUsage = metrics.TelemetryTablePartitionRangeCnt
telemetryTablePartitionHashUsage = metrics.TelemetryTablePartitionHashCnt
telemetryTablePartitionRangeColumnsUsage = metrics.TelemetryTablePartitionRangeColumnsCnt
telemetryTablePartitionListColumnsUsage = metrics.TelemetryTablePartitionListColumnsCnt
telemetryTablePartitionMaxPartitionsUsage = metrics.TelemetryTablePartitionMaxPartitionsCnt
telemetryLockUserUsage = metrics.TelemetryAccountLockCnt.WithLabelValues("lockUser")
telemetryUnlockUserUsage = metrics.TelemetryAccountLockCnt.WithLabelValues("unlockUser")
telemetryCreateOrAlterUserUsage = metrics.TelemetryAccountLockCnt.WithLabelValues("createOrAlterUser")
telemetryCTEUsageRecurCTE = metrics.TelemetrySQLCTECnt.WithLabelValues("recurCTE")
telemetryCTEUsageNonRecurCTE = metrics.TelemetrySQLCTECnt.WithLabelValues("nonRecurCTE")
telemetryCTEUsageNotCTE = metrics.TelemetrySQLCTECnt.WithLabelValues("notCTE")
telemetryMultiSchemaChangeUsage = metrics.TelemetryMultiSchemaChangeCnt
telemetryTablePartitionUsage = metrics.TelemetryTablePartitionCnt
telemetryTablePartitionListUsage = metrics.TelemetryTablePartitionListCnt
telemetryTablePartitionRangeUsage = metrics.TelemetryTablePartitionRangeCnt
telemetryTablePartitionHashUsage = metrics.TelemetryTablePartitionHashCnt
telemetryTablePartitionRangeColumnsUsage = metrics.TelemetryTablePartitionRangeColumnsCnt
telemetryTablePartitionListColumnsUsage = metrics.TelemetryTablePartitionListColumnsCnt
telemetryTablePartitionMaxPartitionsUsage = metrics.TelemetryTablePartitionMaxPartitionsCnt
telemetryLockUserUsage = metrics.TelemetryAccountLockCnt.WithLabelValues("lockUser")
telemetryUnlockUserUsage = metrics.TelemetryAccountLockCnt.WithLabelValues("unlockUser")
telemetryCreateOrAlterUserUsage = metrics.TelemetryAccountLockCnt.WithLabelValues("createOrAlterUser")
telemetryTablePartitionCreateIntervalUsage = metrics.TelemetryTablePartitionCreateIntervalPartitionsCnt
telemetryTablePartitionAddIntervalUsage = metrics.TelemetryTablePartitionAddIntervalPartitionsCnt
telemetryTablePartitionDropIntervalUsage = metrics.TelemetryTablePartitionDropIntervalPartitionsCnt
)

// Session context, it is consistent with the lifecycle of a client connection.
Expand Down Expand Up @@ -3310,6 +3313,15 @@ func (s *session) updateTelemetryMetric(es *executor.ExecStmt) {
if ti.PartitionTelemetry.UseTablePartitionListColumns {
telemetryTablePartitionListColumnsUsage.Inc()
}
if ti.PartitionTelemetry.UseCreateIntervalPartition {
telemetryTablePartitionCreateIntervalUsage.Inc()
}
if ti.PartitionTelemetry.UseAddIntervalPartition {
telemetryTablePartitionAddIntervalUsage.Inc()
}
if ti.PartitionTelemetry.UseDropIntervalPartition {
telemetryTablePartitionDropIntervalUsage.Inc()
}
}

if ti.AccountLockTelemetry != nil {
Expand Down
15 changes: 13 additions & 2 deletions telemetry/data_feature_usage_test.go
View file
Expand Up @@ -212,6 +212,9 @@ func TestTablePartition(t *testing.T) {
require.Equal(t, int64(0), usage.TablePartition.TablePartitionRangeCnt)
require.Equal(t, int64(0), usage.TablePartition.TablePartitionRangeColumnsCnt)
require.Equal(t, int64(0), usage.TablePartition.TablePartitionListColumnsCnt)
require.Equal(t, int64(0), usage.TablePartition.TablePartitionCreateIntervalPartitionsCnt)
require.Equal(t, int64(0), usage.TablePartition.TablePartitionAddIntervalPartitionsCnt)
require.Equal(t, int64(0), usage.TablePartition.TablePartitionDropIntervalPartitionsCnt)

telemetry.PostReportTelemetryDataForTest()
tk.MustExec("drop table if exists pt1")
Expand All @@ -221,15 +224,23 @@ func TestTablePartition(t *testing.T) {
"partition p2 values less than (9)," +
"partition p3 values less than (12)," +
"partition p4 values less than (15))")
tk.MustExec("alter table pt1 first partition less than (9)")
tk.MustExec("alter table pt1 last partition less than (21)")
tk.MustExec("drop table if exists pt1")
tk.MustExec("create table pt1 (d datetime primary key, v varchar(255)) partition by range columns(d)" +
" interval (1 day) first partition less than ('2022-01-01') last partition less than ('2022-02-22')")
usage, err = telemetry.GetFeatureUsage(tk.Session())
require.NoError(t, err)
require.Equal(t, int64(1), usage.TablePartition.TablePartitionCnt)
require.Equal(t, int64(2), usage.TablePartition.TablePartitionCnt)
require.Equal(t, int64(0), usage.TablePartition.TablePartitionHashCnt)
require.Equal(t, int64(5), usage.TablePartition.TablePartitionMaxPartitionsCnt)
require.Equal(t, int64(0), usage.TablePartition.TablePartitionListCnt)
require.Equal(t, int64(1), usage.TablePartition.TablePartitionRangeCnt)
require.Equal(t, int64(0), usage.TablePartition.TablePartitionRangeColumnsCnt)
require.Equal(t, int64(1), usage.TablePartition.TablePartitionRangeColumnsCnt)
require.Equal(t, int64(0), usage.TablePartition.TablePartitionListColumnsCnt)
require.Equal(t, int64(1), usage.TablePartition.TablePartitionCreateIntervalPartitionsCnt)
require.Equal(t, int64(1), usage.TablePartition.TablePartitionAddIntervalPartitionsCnt)
require.Equal(t, int64(1), usage.TablePartition.TablePartitionDropIntervalPartitionsCnt)
}

func TestPlacementPolicies(t *testing.T) {
Expand Down

0 comments on commit f0d8d47

Please sign in to comment.