From 0ba1555e40057453533f8977ae419b2cb63a3d1f Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Thu, 30 Nov 2023 11:56:44 +0800 Subject: [PATCH 01/22] [perf] optimizing unconditional delete through truncate --- pkg/sql/colexec/deletion/types.go | 8 +- pkg/sql/compile/compile.go | 30 +++++++ pkg/sql/compile/dml.go | 144 +++++++++++++++++++++++++++--- pkg/sql/compile/operator.go | 17 ++++ pkg/sql/plan/build_dml_util.go | 39 ++++---- 5 files changed, 199 insertions(+), 39 deletions(-) diff --git a/pkg/sql/colexec/deletion/types.go b/pkg/sql/colexec/deletion/types.go index 028aa6b9b48b..5749ba37a360 100644 --- a/pkg/sql/colexec/deletion/types.go +++ b/pkg/sql/colexec/deletion/types.go @@ -98,9 +98,11 @@ func (arg *Argument) AppendChild(child vm.Operator) { type DeleteCtx struct { CanTruncate bool - RowIdIdx int // The array index position of the rowid column - PartitionTableIDs []uint64 // Align array index with the partition number - PartitionTableNames []string // Align array index with the partition number + RowIdIdx int // The array index position of the rowid column + PartitionTableIDs []uint64 // Align array index with the partition number + PartitionTableNames []string // Align array index with the partition number + IndexTableNames []string + ForeignTbl []uint64 PartitionIndexInBatch int // The array index position of the partition expression column PartitionSources []engine.Relation // Align array index with the partition number Source engine.Relation diff --git a/pkg/sql/compile/compile.go b/pkg/sql/compile/compile.go index abe6df8ef74e..4dbbe19c8105 100644 --- a/pkg/sql/compile/compile.go +++ b/pkg/sql/compile/compile.go @@ -1131,6 +1131,23 @@ func (c *Compile) compilePlanScope(ctx context.Context, step int32, curNodeIdx i return c.compileSort(n, c.compileUnionAll(left, right)), nil case plan.Node_DELETE: c.appendMetaTables(n.DeleteCtx.Ref) + if n.DeleteCtx.CanTruncate { + arg, err := constructDeletion(n, c.e, c.proc) + if err != nil { + return nil, err + } + ss := []*Scope{{ + Magic: Deletion, + Plan: c.pn, + Instructions: vm.Instructions{ + { + Op: vm.Deletion, + Arg: arg, + }, + }, + }} + return ss, nil + } curr := c.anal.curr c.setAnalyzeCurrent(nil, int(n.Children[0])) @@ -2794,6 +2811,19 @@ func (c *Compile) newDeleteMergeScope(arg *deletion.Argument, ss []*Scope) *Scop return c.newMergeScope(rs) } +func (c *Compile) newDeleteScope(arg *deletion.Argument, ss []*Scope) []*Scope { + deleteIns := &vm.Instruction{ + Op: vm.Deletion, + Arg: arg, + } + for i := range ss { + ss[i].Instructions = append( + ss[i].Instructions, + dupInstruction(deleteIns, nil, 0)) + } + return ss +} + func (c *Compile) newMergeScope(ss []*Scope) *Scope { rs := &Scope{ PreScopes: ss, diff --git a/pkg/sql/compile/dml.go b/pkg/sql/compile/dml.go index 190329f5dc20..d276f31be5b8 100644 --- a/pkg/sql/compile/dml.go +++ b/pkg/sql/compile/dml.go @@ -15,9 +15,14 @@ package compile import ( + "fmt" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/incrservice" + "github.com/matrixorigin/matrixone/pkg/pb/lock" "github.com/matrixorigin/matrixone/pkg/sql/colexec/deletion" "github.com/matrixorigin/matrixone/pkg/sql/colexec/insert" + "github.com/matrixorigin/matrixone/pkg/vm/engine" ) func (s *Scope) Delete(c *Compile) (uint64, error) { @@ -25,35 +30,142 @@ func (s *Scope) Delete(c *Compile) (uint64, error) { arg := s.Instructions[len(s.Instructions)-1].Arg.(*deletion.Argument) if arg.DeleteCtx.CanTruncate { + var dbSource engine.Database + var rel engine.Relation var err error - var affectRows int64 + var isTemp bool + var newId uint64 + delCtx := arg.DeleteCtx - err = delCtx.Source.UpdateObjectInfos(c.ctx) + dbName := delCtx.Ref.SchemaName + tblName := delCtx.Ref.ObjName + oldId := uint64(delCtx.Ref.Obj) + affectRows, err := delCtx.Source.Rows(c.ctx) + if err != nil { return 0, err } - affectRow, err := delCtx.Source.Rows(s.Proc.Ctx) + + dbSource, err = c.e.Database(c.ctx, dbName, c.proc.TxnOperator) if err != nil { return 0, err } - affectRows = affectRows + affectRow - dbName := delCtx.Ref.SchemaName - tblName := delCtx.Ref.ObjName - oldId := uint64(delCtx.Ref.Obj) - dbSource, err := c.e.Database(c.ctx, dbName, c.proc.TxnOperator) - if err != nil { - return 0, err + if rel, err = dbSource.Relation(c.ctx, tblName, nil); err != nil { + var e error // avoid contamination of error messages + dbSource, e = c.e.Database(c.ctx, defines.TEMPORARY_DBNAME, c.proc.TxnOperator) + if e != nil { + return 0, err + } + rel, e = dbSource.Relation(c.ctx, engine.GetTempTableName(dbName, tblName), nil) + if e != nil { + return 0, err + } + isTemp = true + } + + if !isTemp && c.proc.TxnOperator.Txn().IsPessimistic() { + var err error + if e := lockMoTable(c, dbName, tblName, lock.LockMode_Shared); e != nil { + if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && + !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { + return 0, e + } + err = e + } + // before dropping table, lock it. + if e := lockTable(c.ctx, c.e, c.proc, rel, dbName, delCtx.PartitionTableNames, false); e != nil { + if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && + !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { + return 0, e + } + err = e + } + if err != nil { + return 0, err + } + } + + if isTemp { + // memoryengine truncate always return 0, so for temporary table, just use origin tableId as newId + _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, tblName)) + newId = rel.GetTableID(c.ctx) + } else { + newId, err = dbSource.Truncate(c.ctx, tblName) } - // truncate origin table - newId, err := dbSource.Truncate(c.ctx, tblName) if err != nil { return 0, err } - // keep old offset. + // Truncate Index Tables if needed + for _, name := range delCtx.IndexTableNames { + var err error + if isTemp { + _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) + } else { + _, err = dbSource.Truncate(c.ctx, name) + } + if err != nil { + return 0, err + } + } + + //Truncate Partition subtable if needed + for _, name := range delCtx.PartitionTableNames { + var err error + if isTemp { + dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) + } else { + _, err = dbSource.Truncate(c.ctx, name) + } + if err != nil { + return 0, err + } + } + + // update tableDef of foreign key's table with new table id + for _, fTblId := range delCtx.ForeignTbl { + _, _, fkRelation, err := c.e.GetRelationById(c.ctx, c.proc.TxnOperator, fTblId) + if err != nil { + return 0, err + } + fkTableDef, err := fkRelation.TableDefs(c.ctx) + if err != nil { + return 0, err + } + var oldCt *engine.ConstraintDef + for _, def := range fkTableDef { + if ct, ok := def.(*engine.ConstraintDef); ok { + oldCt = ct + break + } + } + for _, ct := range oldCt.Cts { + if def, ok := ct.(*engine.RefChildTableDef); ok { + for idx, refTable := range def.Tables { + if refTable == oldId { + def.Tables[idx] = newId + break + } + } + break + } + } + if err != nil { + return 0, err + } + err = fkRelation.UpdateConstraint(c.ctx, oldCt) + if err != nil { + return 0, err + } + + } + + if isTemp { + oldId = rel.GetTableID(c.ctx) + } err = incrservice.GetAutoIncrementService(c.ctx).Reset( c.ctx, oldId, @@ -64,6 +176,12 @@ func (s *Scope) Delete(c *Compile) (uint64, error) { return 0, err } + // update index information in mo_catalog.mo_indexes + updateSql := fmt.Sprintf(updateMoIndexesTruncateTableFormat, newId, oldId) + err = c.runSql(updateSql) + if err != nil { + return 0, err + } return uint64(affectRows), nil } diff --git a/pkg/sql/compile/operator.go b/pkg/sql/compile/operator.go index 5d8a11cc64d5..e4442dc96a9f 100644 --- a/pkg/sql/compile/operator.go +++ b/pkg/sql/compile/operator.go @@ -540,6 +540,23 @@ func constructDeletion(n *plan.Node, eg engine.Engine, proc *process.Process) (* } } + if delCtx.CanTruncate { + tableDef := delCtx.Source.GetTableDef(proc.Ctx) + delCtx.IndexTableNames = make([]string, 0) + if tableDef.Indexes != nil { + for _, indexDef := range tableDef.Indexes { + if indexDef.TableExist { + delCtx.IndexTableNames = append(delCtx.IndexTableNames, indexDef.IndexTableName) + } + } + } + if tableDef.Fkeys != nil { + for _, fk := range tableDef.Fkeys { + delCtx.ForeignTbl = append(delCtx.ForeignTbl, fk.ForeignTbl) + } + } + } + return &deletion.Argument{ DeleteCtx: delCtx, }, nil diff --git a/pkg/sql/plan/build_dml_util.go b/pkg/sql/plan/build_dml_util.go index 0feb1a7118e2..e15a49088985 100644 --- a/pkg/sql/plan/build_dml_util.go +++ b/pkg/sql/plan/build_dml_util.go @@ -269,7 +269,8 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC // both UK and SK. To handle SK case, we will have flags to indicate if it's UK or SK. hasUniqueKey := haveUniqueKey(delCtx.tableDef) hasSecondaryKey := haveSecondaryKey(delCtx.tableDef) - if hasUniqueKey || hasSecondaryKey { + if (hasUniqueKey || hasSecondaryKey) && !delCtx.isDeleteWithoutFilters { + uniqueDeleteIdx := len(delCtx.tableDef.Cols) + delCtx.updateColLength typMap := make(map[string]*plan.Type) posMap := make(map[string]int) for idx, col := range delCtx.tableDef.Cols { @@ -302,26 +303,17 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC if uniqueTableDef == nil { return moerr.NewNoSuchTable(builder.GetContext(), delCtx.objRef.SchemaName, indexdef.IndexTableName) } - var lastNodeId int32 - var err error - var uniqueDeleteIdx int - var uniqueTblPkPos int - var uniqueTblPkTyp *Type - - if delCtx.isDeleteWithoutFilters { - lastNodeId, err = appendDeleteUniqueTablePlanWithoutFilters(builder, bindCtx, uniqueObjRef, uniqueTableDef) - uniqueDeleteIdx = getRowIdPos(uniqueTableDef) - uniqueTblPkPos, uniqueTblPkTyp = getPkPos(uniqueTableDef, false) - } else { - lastNodeId = appendSinkScanNode(builder, bindCtx, delCtx.sourceStep) - lastNodeId, err = appendDeleteUniqueTablePlan(builder, bindCtx, uniqueObjRef, uniqueTableDef, indexdef, typMap, posMap, lastNodeId, isUk) - uniqueDeleteIdx = len(delCtx.tableDef.Cols) + delCtx.updateColLength - uniqueTblPkPos = uniqueDeleteIdx + 1 - uniqueTblPkTyp = uniqueTableDef.Cols[0].Typ - } + lastNodeId := appendSinkScanNode(builder, bindCtx, delCtx.sourceStep) + + lastNodeId, err := appendDeleteUniqueTablePlan(builder, bindCtx, uniqueObjRef, uniqueTableDef, indexdef, typMap, posMap, lastNodeId, isUk) + if err != nil { return err } + + uniqueTblPkPos := uniqueDeleteIdx + 1 + uniqueTblPkTyp := uniqueTableDef.Cols[0].Typ + if isUpdate { // do it like simple update lastNodeId = appendSinkNode(builder, bindCtx, lastNodeId) @@ -331,7 +323,7 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC //sink_scan -> lock -> delete lastNodeId = appendSinkScanNode(builder, bindCtx, newSourceStep) delNodeInfo := makeDeleteNodeInfo(builder.compCtx, uniqueObjRef, uniqueTableDef, uniqueDeleteIdx, -1, false, uniqueTblPkPos, uniqueTblPkTyp, delCtx.lockTable, delCtx.partitionInfos) - lastNodeId, err = makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, isUk, isSK) + lastNodeId, err = makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, isUk, isSK, false) putDeleteNodeInfo(delNodeInfo) if err != nil { return err @@ -381,7 +373,7 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC } else { // it's more simple for delete hidden unique table .so we append nodes after the plan. not recursive call buildDeletePlans delNodeInfo := makeDeleteNodeInfo(builder.compCtx, uniqueObjRef, uniqueTableDef, uniqueDeleteIdx, -1, false, uniqueTblPkPos, uniqueTblPkTyp, delCtx.lockTable, delCtx.partitionInfos) - lastNodeId, err = makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, isUk, isSK) + lastNodeId, err = makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, isUk, isSK, delCtx.isDeleteWithoutFilters) putDeleteNodeInfo(delNodeInfo) if err != nil { return err @@ -401,7 +393,7 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC } pkPos, pkTyp := getPkPos(delCtx.tableDef, false) delNodeInfo := makeDeleteNodeInfo(ctx, delCtx.objRef, delCtx.tableDef, delCtx.rowIdPos, partExprIdx, true, pkPos, pkTyp, delCtx.lockTable, delCtx.partitionInfos) - lastNodeId, err := makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, false, false) + lastNodeId, err := makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, false, false, delCtx.isDeleteWithoutFilters) putDeleteNodeInfo(delNodeInfo) if err != nil { return err @@ -1488,8 +1480,9 @@ func makeOneDeletePlan( bindCtx *BindContext, lastNodeId int32, delNodeInfo *deleteNodeInfo, - isUK bool, // is delete unique key hidden table + isUK bool, isSK bool, + canTruncate bool, ) (int32, error) { if isUK || isSK { // append lock @@ -1521,7 +1514,7 @@ func makeOneDeletePlan( DeleteCtx: &plan.DeleteCtx{ RowIdIdx: int32(delNodeInfo.deleteIndex), Ref: delNodeInfo.objRef, - CanTruncate: false, + CanTruncate: canTruncate, AddAffectedRows: delNodeInfo.addAffectedRows, IsClusterTable: delNodeInfo.IsClusterTable, PartitionTableIds: delNodeInfo.partTableIDs, From 688bee7f571f74e4ffa2a06a297e4065585724c3 Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Thu, 30 Nov 2023 15:19:47 +0800 Subject: [PATCH 02/22] [fix] the issue of being unable to delete special tables --- pkg/sql/plan/build_dml_util.go | 42 ++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/pkg/sql/plan/build_dml_util.go b/pkg/sql/plan/build_dml_util.go index e15a49088985..08c77ec08edc 100644 --- a/pkg/sql/plan/build_dml_util.go +++ b/pkg/sql/plan/build_dml_util.go @@ -269,8 +269,17 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC // both UK and SK. To handle SK case, we will have flags to indicate if it's UK or SK. hasUniqueKey := haveUniqueKey(delCtx.tableDef) hasSecondaryKey := haveSecondaryKey(delCtx.tableDef) - if (hasUniqueKey || hasSecondaryKey) && !delCtx.isDeleteWithoutFilters { - uniqueDeleteIdx := len(delCtx.tableDef.Cols) + delCtx.updateColLength + + canTruncate := delCtx.isDeleteWithoutFilters + + if len(delCtx.tableDef.RefChildTbls) > 0 || + delCtx.tableDef.ViewSql != nil || + (util.TableIsClusterTable(delCtx.tableDef.GetTableType()) && ctx.GetAccountId() != catalog.System_Account) || + delCtx.objRef.PubInfo != nil { + canTruncate = false + } + + if (hasUniqueKey || hasSecondaryKey) && !canTruncate { typMap := make(map[string]*plan.Type) posMap := make(map[string]int) for idx, col := range delCtx.tableDef.Cols { @@ -303,17 +312,26 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC if uniqueTableDef == nil { return moerr.NewNoSuchTable(builder.GetContext(), delCtx.objRef.SchemaName, indexdef.IndexTableName) } - lastNodeId := appendSinkScanNode(builder, bindCtx, delCtx.sourceStep) - - lastNodeId, err := appendDeleteUniqueTablePlan(builder, bindCtx, uniqueObjRef, uniqueTableDef, indexdef, typMap, posMap, lastNodeId, isUk) - + var lastNodeId int32 + var err error + var uniqueDeleteIdx int + var uniqueTblPkPos int + var uniqueTblPkTyp *Type + + if delCtx.isDeleteWithoutFilters { + lastNodeId, err = appendDeleteUniqueTablePlanWithoutFilters(builder, bindCtx, uniqueObjRef, uniqueTableDef) + uniqueDeleteIdx = getRowIdPos(uniqueTableDef) + uniqueTblPkPos, uniqueTblPkTyp = getPkPos(uniqueTableDef, false) + } else { + lastNodeId = appendSinkScanNode(builder, bindCtx, delCtx.sourceStep) + lastNodeId, err = appendDeleteUniqueTablePlan(builder, bindCtx, uniqueObjRef, uniqueTableDef, indexdef, typMap, posMap, lastNodeId, isUk) + uniqueDeleteIdx = len(delCtx.tableDef.Cols) + delCtx.updateColLength + uniqueTblPkPos = uniqueDeleteIdx + 1 + uniqueTblPkTyp = uniqueTableDef.Cols[0].Typ + } if err != nil { return err } - - uniqueTblPkPos := uniqueDeleteIdx + 1 - uniqueTblPkTyp := uniqueTableDef.Cols[0].Typ - if isUpdate { // do it like simple update lastNodeId = appendSinkNode(builder, bindCtx, lastNodeId) @@ -373,7 +391,7 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC } else { // it's more simple for delete hidden unique table .so we append nodes after the plan. not recursive call buildDeletePlans delNodeInfo := makeDeleteNodeInfo(builder.compCtx, uniqueObjRef, uniqueTableDef, uniqueDeleteIdx, -1, false, uniqueTblPkPos, uniqueTblPkTyp, delCtx.lockTable, delCtx.partitionInfos) - lastNodeId, err = makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, isUk, isSK, delCtx.isDeleteWithoutFilters) + lastNodeId, err = makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, isUk, isSK, false) putDeleteNodeInfo(delNodeInfo) if err != nil { return err @@ -393,7 +411,7 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC } pkPos, pkTyp := getPkPos(delCtx.tableDef, false) delNodeInfo := makeDeleteNodeInfo(ctx, delCtx.objRef, delCtx.tableDef, delCtx.rowIdPos, partExprIdx, true, pkPos, pkTyp, delCtx.lockTable, delCtx.partitionInfos) - lastNodeId, err := makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, false, false, delCtx.isDeleteWithoutFilters) + lastNodeId, err := makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, false, false, canTruncate) putDeleteNodeInfo(delNodeInfo) if err != nil { return err From 682d31fefdf311406c3ae4120982a46a5d1cb539 Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Thu, 30 Nov 2023 16:01:59 +0800 Subject: [PATCH 03/22] [mod] remove useless codes --- pkg/sql/compile/compile.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/pkg/sql/compile/compile.go b/pkg/sql/compile/compile.go index 4dbbe19c8105..056f99387a03 100644 --- a/pkg/sql/compile/compile.go +++ b/pkg/sql/compile/compile.go @@ -2811,19 +2811,6 @@ func (c *Compile) newDeleteMergeScope(arg *deletion.Argument, ss []*Scope) *Scop return c.newMergeScope(rs) } -func (c *Compile) newDeleteScope(arg *deletion.Argument, ss []*Scope) []*Scope { - deleteIns := &vm.Instruction{ - Op: vm.Deletion, - Arg: arg, - } - for i := range ss { - ss[i].Instructions = append( - ss[i].Instructions, - dupInstruction(deleteIns, nil, 0)) - } - return ss -} - func (c *Compile) newMergeScope(ss []*Scope) *Scope { rs := &Scope{ PreScopes: ss, From 3855d65e5cefd90c44d29385dddd7aa046e19dd2 Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Thu, 30 Nov 2023 18:47:49 +0800 Subject: [PATCH 04/22] [refactor] truncate table operation --- pkg/sql/compile/ddl.go | 76 +++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/pkg/sql/compile/ddl.go b/pkg/sql/compile/ddl.go index 711ae9c266bc..e5fb8878e429 100644 --- a/pkg/sql/compile/ddl.go +++ b/pkg/sql/compile/ddl.go @@ -1238,31 +1238,33 @@ func (s *Scope) removeRefChildTbl(c *Compile, fkTblId uint64, tblId uint64) erro return fkRelation.UpdateConstraint(c.ctx, oldCt) } -// Truncation operations cannot be performed if the session holds an active table lock. -func (s *Scope) TruncateTable(c *Compile) error { +func doTruncateTable( + c *Compile, + dbName string, + tableName string, + tableId uint64, + partitionTableNames []string, + indexTableNames []string, + foreignTbl []uint64, + keepAutoIncrement bool, +) error { var dbSource engine.Database var rel engine.Relation var err error var isTemp bool - var newId uint64 - - tqry := s.Plan.GetDdl().GetTruncateTable() - dbName := tqry.GetDatabase() - tblName := tqry.GetTable() - oldId := tqry.GetTableId() - + var newTblId uint64 dbSource, err = c.e.Database(c.ctx, dbName, c.proc.TxnOperator) if err != nil { return err } - if rel, err = dbSource.Relation(c.ctx, tblName, nil); err != nil { + if rel, err = dbSource.Relation(c.ctx, tableName, nil); err != nil { var e error // avoid contamination of error messages dbSource, e = c.e.Database(c.ctx, defines.TEMPORARY_DBNAME, c.proc.TxnOperator) if e != nil { return err } - rel, e = dbSource.Relation(c.ctx, engine.GetTempTableName(dbName, tblName), nil) + rel, e = dbSource.Relation(c.ctx, engine.GetTempTableName(dbName, tableName), nil) if e != nil { return err } @@ -1271,7 +1273,7 @@ func (s *Scope) TruncateTable(c *Compile) error { if !isTemp && c.proc.TxnOperator.Txn().IsPessimistic() { var err error - if e := lockMoTable(c, dbName, tblName, lock.LockMode_Shared); e != nil { + if e := lockMoTable(c, dbName, tableName, lock.LockMode_Shared); e != nil { if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { return e @@ -1279,7 +1281,7 @@ func (s *Scope) TruncateTable(c *Compile) error { err = e } // before dropping table, lock it. - if e := lockTable(c.ctx, c.e, c.proc, rel, dbName, tqry.PartitionTableNames, false); e != nil { + if e := lockTable(c.ctx, c.e, c.proc, rel, dbName, partitionTableNames, false); e != nil { if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { return e @@ -1292,11 +1294,11 @@ func (s *Scope) TruncateTable(c *Compile) error { } if isTemp { - // memoryengine truncate always return 0, so for temporary table, just use origin tableId as newId - _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, tblName)) - newId = rel.GetTableID(c.ctx) + // memory engine truncate always return 0, so for temporary table, just use origin tableId as newTblId + _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, tableName)) + newTblId = rel.GetTableID(c.ctx) } else { - newId, err = dbSource.Truncate(c.ctx, tblName) + newTblId, err = dbSource.Truncate(c.ctx, tableName) } if err != nil { @@ -1304,7 +1306,7 @@ func (s *Scope) TruncateTable(c *Compile) error { } // Truncate Index Tables if needed - for _, name := range tqry.IndexTableNames { + for _, name := range indexTableNames { var err error if isTemp { _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) @@ -1316,11 +1318,11 @@ func (s *Scope) TruncateTable(c *Compile) error { } } - //Truncate Partition subtable if needed - for _, name := range tqry.PartitionTableNames { + //Truncate Partition subTable if needed + for _, name := range partitionTableNames { var err error if isTemp { - dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) + _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) } else { _, err = dbSource.Truncate(c.ctx, name) } @@ -1330,8 +1332,8 @@ func (s *Scope) TruncateTable(c *Compile) error { } // update tableDef of foreign key's table with new table id - for _, ftblId := range tqry.ForeignTbl { - _, _, fkRelation, err := c.e.GetRelationById(c.ctx, c.proc.TxnOperator, ftblId) + for _, fTblId := range foreignTbl { + _, _, fkRelation, err := c.e.GetRelationById(c.ctx, c.proc.TxnOperator, fTblId) if err != nil { return err } @@ -1349,8 +1351,8 @@ func (s *Scope) TruncateTable(c *Compile) error { for _, ct := range oldCt.Cts { if def, ok := ct.(*engine.RefChildTableDef); ok { for idx, refTable := range def.Tables { - if refTable == oldId { - def.Tables[idx] = newId + if refTable == tableId { + def.Tables[idx] = newTblId break } } @@ -1368,20 +1370,20 @@ func (s *Scope) TruncateTable(c *Compile) error { } if isTemp { - oldId = rel.GetTableID(c.ctx) + tableId = rel.GetTableID(c.ctx) } err = incrservice.GetAutoIncrementService(c.ctx).Reset( c.ctx, - oldId, - newId, - false, + tableId, + newTblId, + keepAutoIncrement, c.proc.TxnOperator) if err != nil { return err } // update index information in mo_catalog.mo_indexes - updateSql := fmt.Sprintf(updateMoIndexesTruncateTableFormat, newId, oldId) + updateSql := fmt.Sprintf(updateMoIndexesTruncateTableFormat, newTblId, tableId) err = c.runSql(updateSql) if err != nil { return err @@ -1389,6 +1391,20 @@ func (s *Scope) TruncateTable(c *Compile) error { return nil } +// TruncateTable Truncation operations cannot be performed if the session holds an active table lock. +func (s *Scope) TruncateTable(c *Compile) error { + truncateTbl := s.Plan.GetDdl().GetTruncateTable() + return doTruncateTable( + c, + truncateTbl.GetDatabase(), + truncateTbl.GetTable(), + truncateTbl.GetTableId(), + truncateTbl.PartitionTableNames, + truncateTbl.IndexTableNames, + truncateTbl.ForeignTbl, + false) +} + func (s *Scope) DropSequence(c *Compile) error { qry := s.Plan.GetDdl().GetDropSequence() dbName := qry.GetDatabase() From 05f9f2f145e338ab4e01566eec4f1a49331c128d Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Fri, 1 Dec 2023 17:02:15 +0800 Subject: [PATCH 05/22] [refactor] move truncate to deletion operator --- pkg/sql/colexec/deletion/deletion.go | 38 ++- pkg/sql/colexec/deletion/util.go | 208 +++++++++++++++ pkg/sql/colexec/lockop/lock_op.go | 8 +- pkg/sql/colexec/lockop/lock_op_no_txn.go | 2 +- pkg/sql/colexec/lockop/util.go | 187 +++++++++++++ pkg/sql/compile/compile.go | 16 +- pkg/sql/compile/ddl.go | 324 ++--------------------- 7 files changed, 460 insertions(+), 323 deletions(-) create mode 100644 pkg/sql/colexec/deletion/util.go create mode 100644 pkg/sql/colexec/lockop/util.go diff --git a/pkg/sql/colexec/deletion/deletion.go b/pkg/sql/colexec/deletion/deletion.go index 6cb26c2a4e99..1f5e70b3895a 100644 --- a/pkg/sql/colexec/deletion/deletion.go +++ b/pkg/sql/colexec/deletion/deletion.go @@ -16,6 +16,8 @@ package deletion import ( "bytes" + "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/vm/engine" "sync/atomic" "github.com/matrixorigin/matrixone/pkg/catalog" @@ -160,6 +162,41 @@ func (arg *Argument) remote_delete(proc *process.Process) (vm.CallResult, error) } func (arg *Argument) normal_delete(proc *process.Process) (vm.CallResult, error) { + delCtx := arg.DeleteCtx + + if arg.DeleteCtx.CanTruncate { + dbName := delCtx.Ref.SchemaName + tableName := delCtx.Ref.ObjName + tableId := uint64(delCtx.Ref.Obj) + result := vm.NewCallResult() + affectRows, err := delCtx.Source.Rows(proc.Ctx) + eng := proc.Ctx.Value(defines.EngineKey{}).(engine.Engine) + if err != nil { + return result, err + } + + err = TruncateTable( + proc.Ctx, + eng, + proc, + dbName, + tableName, + tableId, + delCtx.PartitionTableNames, + delCtx.IndexTableNames, + delCtx.ForeignTbl, + true, + ) + + if err != nil { + return result, err + } + if delCtx.AddAffectedRows { + atomic.AddUint64(&arg.affectedRows, uint64(affectRows)) + } + return result, nil + } + result, err := arg.children[0].Call(proc) if err != nil { return result, err @@ -170,7 +207,6 @@ func (arg *Argument) normal_delete(proc *process.Process) (vm.CallResult, error) bat := result.Batch var affectedRows uint64 - delCtx := arg.DeleteCtx if len(delCtx.PartitionTableIDs) > 0 { delBatches, err := colexec.GroupByPartitionForDelete(proc, bat, delCtx.RowIdIdx, delCtx.PartitionIndexInBatch, diff --git a/pkg/sql/colexec/deletion/util.go b/pkg/sql/colexec/deletion/util.go new file mode 100644 index 000000000000..c0bc81f2e249 --- /dev/null +++ b/pkg/sql/colexec/deletion/util.go @@ -0,0 +1,208 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deletion + +import ( + "context" + "fmt" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + moruntime "github.com/matrixorigin/matrixone/pkg/common/runtime" + "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/incrservice" + "github.com/matrixorigin/matrixone/pkg/pb/lock" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/lockop" + "github.com/matrixorigin/matrixone/pkg/util/executor" + "github.com/matrixorigin/matrixone/pkg/vm/engine" + "github.com/matrixorigin/matrixone/pkg/vm/process" +) + +func runSql(proc *process.Process, dbName string, sql string) error { + if sql == "" { + return nil + } + v, ok := moruntime.ProcessLevelRuntime().GetGlobalVariables(moruntime.InternalSQLExecutor) + if !ok { + panic("missing lock service") + } + exec := v.(executor.SQLExecutor) + opts := executor.Options{}. + // All runSql and runSqlWithResult is a part of input sql, can not incr statement. + // All these sub-sql's need to be rolled back and retried en masse when they conflict in pessimistic mode + WithDisableIncrStatement(). + WithTxn(proc.TxnOperator). + WithDatabase(dbName). + WithTimeZone(proc.SessionInfo.TimeZone) + res, err := exec.Exec(proc.Ctx, sql, opts) + if err != nil { + return err + } + res.Close() + return nil +} + +func TruncateTable( + ctx context.Context, + eng engine.Engine, + proc *process.Process, + dbName string, + tableName string, + tableId uint64, + partitionTableNames []string, + indexTableNames []string, + foreignTbl []uint64, + keepAutoIncrement bool, +) error { + var dbSource engine.Database + var rel engine.Relation + var err error + var isTemp bool + var newTblId uint64 + dbSource, err = eng.Database(ctx, dbName, proc.TxnOperator) + if err != nil { + return err + } + + if rel, err = dbSource.Relation(ctx, tableName, nil); err != nil { + var e error // avoid contamination of error messages + dbSource, e = eng.Database(ctx, defines.TEMPORARY_DBNAME, proc.TxnOperator) + if e != nil { + return err + } + rel, e = dbSource.Relation(ctx, engine.GetTempTableName(dbName, tableName), nil) + if e != nil { + return err + } + isTemp = true + } + + if !isTemp && proc.TxnOperator.Txn().IsPessimistic() { + var err error + if e := lockop.LockMoTable(ctx, eng, proc, dbName, tableName, lock.LockMode_Shared); e != nil { + if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && + !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { + return e + } + err = e + } + // before dropping table, lock it. + if e := lockop.LockTable(ctx, eng, proc, rel, dbName, partitionTableNames, false); e != nil { + if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && + !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { + return e + } + err = e + } + if err != nil { + return err + } + } + + if isTemp { + // memory engine truncate always return 0, so for temporary table, just use origin tableId as newTblId + _, err = dbSource.Truncate(ctx, engine.GetTempTableName(dbName, tableName)) + newTblId = rel.GetTableID(ctx) + } else { + newTblId, err = dbSource.Truncate(ctx, tableName) + } + + if err != nil { + return err + } + + // Truncate Index Tables if needed + for _, name := range indexTableNames { + var err error + if isTemp { + _, err = dbSource.Truncate(ctx, engine.GetTempTableName(dbName, name)) + } else { + _, err = dbSource.Truncate(ctx, name) + } + if err != nil { + return err + } + } + + //Truncate Partition subTable if needed + for _, name := range partitionTableNames { + var err error + if isTemp { + _, err = dbSource.Truncate(ctx, engine.GetTempTableName(dbName, name)) + } else { + _, err = dbSource.Truncate(ctx, name) + } + if err != nil { + return err + } + } + + // update tableDef of foreign key's table with new table id + for _, fTblId := range foreignTbl { + _, _, fkRelation, err := eng.GetRelationById(ctx, proc.TxnOperator, fTblId) + if err != nil { + return err + } + fkTableDef, err := fkRelation.TableDefs(ctx) + if err != nil { + return err + } + var oldCt *engine.ConstraintDef + for _, def := range fkTableDef { + if ct, ok := def.(*engine.ConstraintDef); ok { + oldCt = ct + break + } + } + for _, ct := range oldCt.Cts { + if def, ok := ct.(*engine.RefChildTableDef); ok { + for idx, refTable := range def.Tables { + if refTable == tableId { + def.Tables[idx] = newTblId + break + } + } + break + } + } + if err != nil { + return err + } + err = fkRelation.UpdateConstraint(ctx, oldCt) + if err != nil { + return err + } + + } + + if isTemp { + tableId = rel.GetTableID(ctx) + } + err = incrservice.GetAutoIncrementService(ctx).Reset( + ctx, + tableId, + newTblId, + keepAutoIncrement, + proc.TxnOperator) + if err != nil { + return err + } + + // update index information in mo_catalog.mo_indexes + updateSql := fmt.Sprintf(`update mo_catalog.mo_indexes set table_id = %v where table_id = %v`, newTblId, tableId) + err = runSql(proc, dbName, updateSql) + if err != nil { + return err + } + return nil +} diff --git a/pkg/sql/colexec/lockop/lock_op.go b/pkg/sql/colexec/lockop/lock_op.go index 65973c6b2dd2..6f92fdfa7f2b 100644 --- a/pkg/sql/colexec/lockop/lock_op.go +++ b/pkg/sql/colexec/lockop/lock_op.go @@ -269,9 +269,9 @@ func performLock( return nil } -// LockTable lock table, all rows in the table will be locked, and wait current txn +// lockTable lock table, all rows in the table will be locked, and wait current txn // closed. -func LockTable( +func lockTable( eng engine.Engine, proc *process.Process, tableID uint64, @@ -619,7 +619,7 @@ func (arg *Argument) AddLockTargetWithMode( return arg } -// LockTable lock all table, used for delete, truncate and drop table +// lockTable lock all table, used for delete, truncate and drop table func (arg *Argument) LockTable( tableID uint64, changeDef bool) *Argument { @@ -629,7 +629,7 @@ func (arg *Argument) LockTable( changeDef) } -// LockTableWithMode is similar to LockTable, but with specify +// LockTableWithMode is similar to lockTable, but with specify // lock mode func (arg *Argument) LockTableWithMode( tableID uint64, diff --git a/pkg/sql/colexec/lockop/lock_op_no_txn.go b/pkg/sql/colexec/lockop/lock_op_no_txn.go index 8667f968f643..f0c429ce6c26 100644 --- a/pkg/sql/colexec/lockop/lock_op_no_txn.go +++ b/pkg/sql/colexec/lockop/lock_op_no_txn.go @@ -35,7 +35,7 @@ var ( internalProcesses = map[string]*process.Process{} ) -// LockTableWithUniqueID is similar to LockTable, but used to lock a table directly based on +// LockTableWithUniqueID is similar to lockTable, but used to lock a table directly based on // a unique identifier, without using an external transaction. func LockTableWithUniqueID( ctx context.Context, diff --git a/pkg/sql/colexec/lockop/util.go b/pkg/sql/colexec/lockop/util.go new file mode 100644 index 000000000000..06bc1d1134e5 --- /dev/null +++ b/pkg/sql/colexec/lockop/util.go @@ -0,0 +1,187 @@ +package lockop + +import ( + "context" + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/pb/lock" + "github.com/matrixorigin/matrixone/pkg/sql/plan/function" + "github.com/matrixorigin/matrixone/pkg/vm/engine" + "github.com/matrixorigin/matrixone/pkg/vm/process" +) + +func getLockVector(proc *process.Process, accountId uint32, names []string) (*vector.Vector, error) { + vecs := make([]*vector.Vector, len(names)+1) + defer func() { + for _, v := range vecs { + if v != nil { + proc.PutVector(v) + } + } + }() + + // append account_id + accountIdVec := proc.GetVector(types.T_uint32.ToType()) + err := vector.AppendFixed(accountIdVec, accountId, false, proc.GetMPool()) + if err != nil { + return nil, err + } + vecs[0] = accountIdVec + // append names + for i, name := range names { + nameVec := proc.GetVector(types.T_varchar.ToType()) + err := vector.AppendBytes(nameVec, []byte(name), false, proc.GetMPool()) + if err != nil { + return nil, err + } + vecs[i+1] = nameVec + } + + vec, err := function.RunFunctionDirectly(proc, function.SerialFunctionEncodeID, vecs, 1) + if err != nil { + return nil, err + } + return vec, nil +} + +func getRelFromMoCatalog( + ctx context.Context, + eng engine.Engine, + proc *process.Process, + tblName string, +) (engine.Relation, error) { + dbSource, err := eng.Database(ctx, catalog.MO_CATALOG, proc.TxnOperator) + if err != nil { + return nil, err + } + + rel, err := dbSource.Relation(ctx, tblName, nil) + if err != nil { + return nil, err + } + + return rel, nil +} + +func lockRows( + eng engine.Engine, + proc *process.Process, + rel engine.Relation, + vec *vector.Vector, + lockMode lock.LockMode) error { + + if vec == nil || vec.Length() == 0 { + panic("lock rows is empty") + } + + id := rel.GetTableID(proc.Ctx) + + err := LockRows( + eng, + proc, + id, + vec, + *vec.GetType(), + lockMode) + return err +} + +func doLockTable( + eng engine.Engine, + proc *process.Process, + rel engine.Relation, + defChanged bool) error { + id := rel.GetTableID(proc.Ctx) + defs, err := rel.GetPrimaryKeys(proc.Ctx) + if err != nil { + return err + } + + if len(defs) != 1 { + panic("invalid primary keys") + } + + err = lockTable( + eng, + proc, + id, + defs[0].Type, + defChanged) + + return err +} + +func LockTable( + ctx context.Context, + eng engine.Engine, + proc *process.Process, + rel engine.Relation, + dbName string, + partitionTableNames []string, + defChanged bool) error { + + if len(partitionTableNames) == 0 { + return doLockTable(eng, proc, rel, defChanged) + } + + dbSource, err := eng.Database(ctx, dbName, proc.TxnOperator) + if err != nil { + return err + } + + for _, tableName := range partitionTableNames { + pRel, pErr := dbSource.Relation(ctx, tableName, nil) + if pErr != nil { + return pErr + } + err = doLockTable(eng, proc, pRel, defChanged) + if err != nil { + return err + } + } + return nil +} + +func LockMoTable( + ctx context.Context, + eng engine.Engine, + proc *process.Process, + dbName string, + tblName string, + lockMode lock.LockMode, +) error { + dbRel, err := getRelFromMoCatalog(ctx, eng, proc, catalog.MO_TABLES) + if err != nil { + return err + } + vec, err := getLockVector(proc, proc.SessionInfo.AccountId, []string{dbName, tblName}) + if err != nil { + return err + } + defer vec.Free(proc.Mp()) + if err := lockRows(eng, proc, dbRel, vec, lockMode); err != nil { + return err + } + return nil +} + +func LockMoDatabase( + ctx context.Context, + eng engine.Engine, + proc *process.Process, + dbName string, +) error { + dbRel, err := getRelFromMoCatalog(ctx, eng, proc, catalog.MO_DATABASE) + if err != nil { + return err + } + vec, err := getLockVector(proc, proc.SessionInfo.AccountId, []string{dbName}) + if err != nil { + return err + } + if err := lockRows(eng, proc, dbRel, vec, lock.LockMode_Exclusive); err != nil { + return err + } + return nil +} diff --git a/pkg/sql/compile/compile.go b/pkg/sql/compile/compile.go index 056f99387a03..b6396c9c9e28 100644 --- a/pkg/sql/compile/compile.go +++ b/pkg/sql/compile/compile.go @@ -1132,20 +1132,16 @@ func (c *Compile) compilePlanScope(ctx context.Context, step int32, curNodeIdx i case plan.Node_DELETE: c.appendMetaTables(n.DeleteCtx.Ref) if n.DeleteCtx.CanTruncate { + rs := c.newMergeScope(nil) arg, err := constructDeletion(n, c.e, c.proc) if err != nil { return nil, err } - ss := []*Scope{{ - Magic: Deletion, - Plan: c.pn, - Instructions: vm.Instructions{ - { - Op: vm.Deletion, - Arg: arg, - }, - }, - }} + rs.Instructions = append(rs.Instructions, vm.Instruction{ + Op: vm.Deletion, + Arg: arg, + }) + ss := []*Scope{rs} return ss, nil } curr := c.anal.curr diff --git a/pkg/sql/compile/ddl.go b/pkg/sql/compile/ddl.go index e5fb8878e429..4a1f9848d0ab 100644 --- a/pkg/sql/compile/ddl.go +++ b/pkg/sql/compile/ddl.go @@ -17,6 +17,7 @@ package compile import ( "context" "fmt" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/deletion" "math" "strings" @@ -35,7 +36,6 @@ import ( "github.com/matrixorigin/matrixone/pkg/sql/parsers/dialect" "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" plan2 "github.com/matrixorigin/matrixone/pkg/sql/plan" - "github.com/matrixorigin/matrixone/pkg/sql/plan/function" "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/util/trace" "github.com/matrixorigin/matrixone/pkg/vm/engine" @@ -1238,164 +1238,13 @@ func (s *Scope) removeRefChildTbl(c *Compile, fkTblId uint64, tblId uint64) erro return fkRelation.UpdateConstraint(c.ctx, oldCt) } -func doTruncateTable( - c *Compile, - dbName string, - tableName string, - tableId uint64, - partitionTableNames []string, - indexTableNames []string, - foreignTbl []uint64, - keepAutoIncrement bool, -) error { - var dbSource engine.Database - var rel engine.Relation - var err error - var isTemp bool - var newTblId uint64 - dbSource, err = c.e.Database(c.ctx, dbName, c.proc.TxnOperator) - if err != nil { - return err - } - - if rel, err = dbSource.Relation(c.ctx, tableName, nil); err != nil { - var e error // avoid contamination of error messages - dbSource, e = c.e.Database(c.ctx, defines.TEMPORARY_DBNAME, c.proc.TxnOperator) - if e != nil { - return err - } - rel, e = dbSource.Relation(c.ctx, engine.GetTempTableName(dbName, tableName), nil) - if e != nil { - return err - } - isTemp = true - } - - if !isTemp && c.proc.TxnOperator.Txn().IsPessimistic() { - var err error - if e := lockMoTable(c, dbName, tableName, lock.LockMode_Shared); e != nil { - if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && - !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { - return e - } - err = e - } - // before dropping table, lock it. - if e := lockTable(c.ctx, c.e, c.proc, rel, dbName, partitionTableNames, false); e != nil { - if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && - !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { - return e - } - err = e - } - if err != nil { - return err - } - } - - if isTemp { - // memory engine truncate always return 0, so for temporary table, just use origin tableId as newTblId - _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, tableName)) - newTblId = rel.GetTableID(c.ctx) - } else { - newTblId, err = dbSource.Truncate(c.ctx, tableName) - } - - if err != nil { - return err - } - - // Truncate Index Tables if needed - for _, name := range indexTableNames { - var err error - if isTemp { - _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) - } else { - _, err = dbSource.Truncate(c.ctx, name) - } - if err != nil { - return err - } - } - - //Truncate Partition subTable if needed - for _, name := range partitionTableNames { - var err error - if isTemp { - _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) - } else { - _, err = dbSource.Truncate(c.ctx, name) - } - if err != nil { - return err - } - } - - // update tableDef of foreign key's table with new table id - for _, fTblId := range foreignTbl { - _, _, fkRelation, err := c.e.GetRelationById(c.ctx, c.proc.TxnOperator, fTblId) - if err != nil { - return err - } - fkTableDef, err := fkRelation.TableDefs(c.ctx) - if err != nil { - return err - } - var oldCt *engine.ConstraintDef - for _, def := range fkTableDef { - if ct, ok := def.(*engine.ConstraintDef); ok { - oldCt = ct - break - } - } - for _, ct := range oldCt.Cts { - if def, ok := ct.(*engine.RefChildTableDef); ok { - for idx, refTable := range def.Tables { - if refTable == tableId { - def.Tables[idx] = newTblId - break - } - } - break - } - } - if err != nil { - return err - } - err = fkRelation.UpdateConstraint(c.ctx, oldCt) - if err != nil { - return err - } - - } - - if isTemp { - tableId = rel.GetTableID(c.ctx) - } - err = incrservice.GetAutoIncrementService(c.ctx).Reset( - c.ctx, - tableId, - newTblId, - keepAutoIncrement, - c.proc.TxnOperator) - if err != nil { - return err - } - - // update index information in mo_catalog.mo_indexes - updateSql := fmt.Sprintf(updateMoIndexesTruncateTableFormat, newTblId, tableId) - err = c.runSql(updateSql) - if err != nil { - return err - } - return nil -} - // TruncateTable Truncation operations cannot be performed if the session holds an active table lock. func (s *Scope) TruncateTable(c *Compile) error { truncateTbl := s.Plan.GetDdl().GetTruncateTable() - return doTruncateTable( - c, + return deletion.TruncateTable( + c.ctx, + c.e, + c.proc, truncateTbl.GetDatabase(), truncateTbl.GetTable(), truncateTbl.GetTableId(), @@ -2345,85 +2194,6 @@ func getInterfaceValue[T constraints.Integer](val interface{}) T { return 0 } -func doLockTable( - eng engine.Engine, - proc *process.Process, - rel engine.Relation, - defChanged bool) error { - id := rel.GetTableID(proc.Ctx) - defs, err := rel.GetPrimaryKeys(proc.Ctx) - if err != nil { - return err - } - - if len(defs) != 1 { - panic("invalid primary keys") - } - - err = lockop.LockTable( - eng, - proc, - id, - defs[0].Type, - defChanged) - - return err -} - -func lockTable( - ctx context.Context, - eng engine.Engine, - proc *process.Process, - rel engine.Relation, - dbName string, - partitionTableNames []string, - defChanged bool) error { - - if len(partitionTableNames) == 0 { - return doLockTable(eng, proc, rel, defChanged) - } - - dbSource, err := eng.Database(ctx, dbName, proc.TxnOperator) - if err != nil { - return err - } - - for _, tableName := range partitionTableNames { - pRel, pErr := dbSource.Relation(ctx, tableName, nil) - if pErr != nil { - return pErr - } - err = doLockTable(eng, proc, pRel, defChanged) - if err != nil { - return err - } - } - return nil -} - -func lockRows( - eng engine.Engine, - proc *process.Process, - rel engine.Relation, - vec *vector.Vector, - lockMode lock.LockMode) error { - - if vec == nil || vec.Length() == 0 { - panic("lock rows is empty") - } - - id := rel.GetTableID(proc.Ctx) - - err := lockop.LockRows( - eng, - proc, - id, - vec, - *vec.GetType(), - lockMode) - return err -} - func maybeCreateAutoIncrement( ctx context.Context, db engine.Database, @@ -2452,81 +2222,21 @@ func maybeCreateAutoIncrement( txnOp) } -func getRelFromMoCatalog(c *Compile, tblName string) (engine.Relation, error) { - dbSource, err := c.e.Database(c.ctx, catalog.MO_CATALOG, c.proc.TxnOperator) - if err != nil { - return nil, err - } - - rel, err := dbSource.Relation(c.ctx, tblName, nil) - if err != nil { - return nil, err - } - - return rel, nil -} - -func getLockVector(proc *process.Process, accountId uint32, names []string) (*vector.Vector, error) { - vecs := make([]*vector.Vector, len(names)+1) - defer func() { - for _, v := range vecs { - if v != nil { - proc.PutVector(v) - } - } - }() - - // append account_id - accountIdVec := proc.GetVector(types.T_uint32.ToType()) - err := vector.AppendFixed(accountIdVec, accountId, false, proc.GetMPool()) - if err != nil { - return nil, err - } - vecs[0] = accountIdVec - // append names - for i, name := range names { - nameVec := proc.GetVector(types.T_varchar.ToType()) - err := vector.AppendBytes(nameVec, []byte(name), false, proc.GetMPool()) - if err != nil { - return nil, err - } - vecs[i+1] = nameVec - } - - vec, err := function.RunFunctionDirectly(proc, function.SerialFunctionEncodeID, vecs, 1) - if err != nil { - return nil, err - } - return vec, nil +func lockMoDatabase(c *Compile, dbName string) error { + return lockop.LockMoDatabase(c.ctx, c.e, c.proc, dbName) } -func lockMoDatabase(c *Compile, dbName string) error { - dbRel, err := getRelFromMoCatalog(c, catalog.MO_DATABASE) - if err != nil { - return err - } - vec, err := getLockVector(c.proc, c.proc.SessionInfo.AccountId, []string{dbName}) - if err != nil { - return err - } - if err := lockRows(c.e, c.proc, dbRel, vec, lock.LockMode_Exclusive); err != nil { - return err - } - return nil +func lockTable( + ctx context.Context, + eng engine.Engine, + proc *process.Process, + rel engine.Relation, + dbName string, + partitionTableNames []string, + defChanged bool) error { + return lockop.LockTable(ctx, eng, proc, rel, dbName, partitionTableNames, defChanged) } func lockMoTable(c *Compile, dbName string, tblName string, lockMode lock.LockMode) error { - dbRel, err := getRelFromMoCatalog(c, catalog.MO_TABLES) - if err != nil { - return err - } - vec, err := getLockVector(c.proc, c.proc.SessionInfo.AccountId, []string{dbName, tblName}) - if err != nil { - return err - } - defer vec.Free(c.proc.Mp()) - if err := lockRows(c.e, c.proc, dbRel, vec, lockMode); err != nil { - return err - } - return nil + return lockop.LockMoTable(c.ctx, c.e, c.proc, dbName, tblName, lockMode) } From 9e13b11af6f048be93c5343deba7e7c069923127 Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Fri, 1 Dec 2023 17:05:40 +0800 Subject: [PATCH 06/22] [mod] revert dml.go --- pkg/sql/compile/dml.go | 144 ++++------------------------------------- 1 file changed, 13 insertions(+), 131 deletions(-) diff --git a/pkg/sql/compile/dml.go b/pkg/sql/compile/dml.go index d276f31be5b8..190329f5dc20 100644 --- a/pkg/sql/compile/dml.go +++ b/pkg/sql/compile/dml.go @@ -15,14 +15,9 @@ package compile import ( - "fmt" - "github.com/matrixorigin/matrixone/pkg/common/moerr" - "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/incrservice" - "github.com/matrixorigin/matrixone/pkg/pb/lock" "github.com/matrixorigin/matrixone/pkg/sql/colexec/deletion" "github.com/matrixorigin/matrixone/pkg/sql/colexec/insert" - "github.com/matrixorigin/matrixone/pkg/vm/engine" ) func (s *Scope) Delete(c *Compile) (uint64, error) { @@ -30,142 +25,35 @@ func (s *Scope) Delete(c *Compile) (uint64, error) { arg := s.Instructions[len(s.Instructions)-1].Arg.(*deletion.Argument) if arg.DeleteCtx.CanTruncate { - var dbSource engine.Database - var rel engine.Relation var err error - var isTemp bool - var newId uint64 - + var affectRows int64 delCtx := arg.DeleteCtx - dbName := delCtx.Ref.SchemaName - tblName := delCtx.Ref.ObjName - oldId := uint64(delCtx.Ref.Obj) - affectRows, err := delCtx.Source.Rows(c.ctx) - + err = delCtx.Source.UpdateObjectInfos(c.ctx) if err != nil { return 0, err } - - dbSource, err = c.e.Database(c.ctx, dbName, c.proc.TxnOperator) + affectRow, err := delCtx.Source.Rows(s.Proc.Ctx) if err != nil { return 0, err } + affectRows = affectRows + affectRow - if rel, err = dbSource.Relation(c.ctx, tblName, nil); err != nil { - var e error // avoid contamination of error messages - dbSource, e = c.e.Database(c.ctx, defines.TEMPORARY_DBNAME, c.proc.TxnOperator) - if e != nil { - return 0, err - } - rel, e = dbSource.Relation(c.ctx, engine.GetTempTableName(dbName, tblName), nil) - if e != nil { - return 0, err - } - isTemp = true - } - - if !isTemp && c.proc.TxnOperator.Txn().IsPessimistic() { - var err error - if e := lockMoTable(c, dbName, tblName, lock.LockMode_Shared); e != nil { - if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && - !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { - return 0, e - } - err = e - } - // before dropping table, lock it. - if e := lockTable(c.ctx, c.e, c.proc, rel, dbName, delCtx.PartitionTableNames, false); e != nil { - if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && - !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { - return 0, e - } - err = e - } - if err != nil { - return 0, err - } - } - - if isTemp { - // memoryengine truncate always return 0, so for temporary table, just use origin tableId as newId - _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, tblName)) - newId = rel.GetTableID(c.ctx) - } else { - newId, err = dbSource.Truncate(c.ctx, tblName) - } - + dbName := delCtx.Ref.SchemaName + tblName := delCtx.Ref.ObjName + oldId := uint64(delCtx.Ref.Obj) + dbSource, err := c.e.Database(c.ctx, dbName, c.proc.TxnOperator) if err != nil { return 0, err } - // Truncate Index Tables if needed - for _, name := range delCtx.IndexTableNames { - var err error - if isTemp { - _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) - } else { - _, err = dbSource.Truncate(c.ctx, name) - } - if err != nil { - return 0, err - } - } - - //Truncate Partition subtable if needed - for _, name := range delCtx.PartitionTableNames { - var err error - if isTemp { - dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) - } else { - _, err = dbSource.Truncate(c.ctx, name) - } - if err != nil { - return 0, err - } - } - - // update tableDef of foreign key's table with new table id - for _, fTblId := range delCtx.ForeignTbl { - _, _, fkRelation, err := c.e.GetRelationById(c.ctx, c.proc.TxnOperator, fTblId) - if err != nil { - return 0, err - } - fkTableDef, err := fkRelation.TableDefs(c.ctx) - if err != nil { - return 0, err - } - var oldCt *engine.ConstraintDef - for _, def := range fkTableDef { - if ct, ok := def.(*engine.ConstraintDef); ok { - oldCt = ct - break - } - } - for _, ct := range oldCt.Cts { - if def, ok := ct.(*engine.RefChildTableDef); ok { - for idx, refTable := range def.Tables { - if refTable == oldId { - def.Tables[idx] = newId - break - } - } - break - } - } - if err != nil { - return 0, err - } - err = fkRelation.UpdateConstraint(c.ctx, oldCt) - if err != nil { - return 0, err - } - + // truncate origin table + newId, err := dbSource.Truncate(c.ctx, tblName) + if err != nil { + return 0, err } - if isTemp { - oldId = rel.GetTableID(c.ctx) - } + // keep old offset. err = incrservice.GetAutoIncrementService(c.ctx).Reset( c.ctx, oldId, @@ -176,12 +64,6 @@ func (s *Scope) Delete(c *Compile) (uint64, error) { return 0, err } - // update index information in mo_catalog.mo_indexes - updateSql := fmt.Sprintf(updateMoIndexesTruncateTableFormat, newId, oldId) - err = c.runSql(updateSql) - if err != nil { - return 0, err - } return uint64(affectRows), nil } From 9fcc82ad3c72e6b6768e1b86a52d382833cc5793 Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Fri, 1 Dec 2023 17:20:12 +0800 Subject: [PATCH 07/22] [mod] add License --- pkg/sql/colexec/lockop/util.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/sql/colexec/lockop/util.go b/pkg/sql/colexec/lockop/util.go index 06bc1d1134e5..79aff3e38ff0 100644 --- a/pkg/sql/colexec/lockop/util.go +++ b/pkg/sql/colexec/lockop/util.go @@ -1,3 +1,17 @@ +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package lockop import ( From 4096cda2328ca50bbc7fb52fc632145068be9241 Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Sun, 3 Dec 2023 20:31:21 +0800 Subject: [PATCH 08/22] [mod] remove useless code --- pkg/sql/compile/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sql/compile/util.go b/pkg/sql/compile/util.go index dabaf15ded07..04502fe94256 100644 --- a/pkg/sql/compile/util.go +++ b/pkg/sql/compile/util.go @@ -73,7 +73,7 @@ var ( deleteMoIndexesWithTableIdFormat = `delete from mo_catalog.mo_indexes where table_id = %v;` deleteMoIndexesWithTableIdAndIndexNameFormat = `delete from mo_catalog.mo_indexes where table_id = %v and name = '%s';` updateMoIndexesVisibleFormat = `update mo_catalog.mo_indexes set is_visible = %v where table_id = %v and name = '%s';` - updateMoIndexesTruncateTableFormat = `update mo_catalog.mo_indexes set table_id = %v where table_id = %v` + //updateMoIndexesTruncateTableFormat = `update mo_catalog.mo_indexes set table_id = %v where table_id = %v` ) var ( From f761fc403fa95b2634aa86dfe2587429c3cc1aaa Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Tue, 5 Dec 2023 11:27:23 +0800 Subject: [PATCH 09/22] Revert "[mod] remove useless code" This reverts commit 4096cda2328ca50bbc7fb52fc632145068be9241. --- pkg/sql/compile/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sql/compile/util.go b/pkg/sql/compile/util.go index 3006b87581a4..f1ead45638d4 100644 --- a/pkg/sql/compile/util.go +++ b/pkg/sql/compile/util.go @@ -74,7 +74,7 @@ var ( deleteMoIndexesWithTableIdFormat = `delete from mo_catalog.mo_indexes where table_id = %v;` deleteMoIndexesWithTableIdAndIndexNameFormat = `delete from mo_catalog.mo_indexes where table_id = %v and name = '%s';` updateMoIndexesVisibleFormat = `update mo_catalog.mo_indexes set is_visible = %v where table_id = %v and name = '%s';` - //updateMoIndexesTruncateTableFormat = `update mo_catalog.mo_indexes set table_id = %v where table_id = %v` + updateMoIndexesTruncateTableFormat = `update mo_catalog.mo_indexes set table_id = %v where table_id = %v` ) var ( From 8d70fb7f05b50fce12df2037a5ff101769187631 Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Tue, 5 Dec 2023 11:27:28 +0800 Subject: [PATCH 10/22] Revert "[mod] add License" This reverts commit 9fcc82ad3c72e6b6768e1b86a52d382833cc5793. --- pkg/sql/colexec/lockop/util.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/pkg/sql/colexec/lockop/util.go b/pkg/sql/colexec/lockop/util.go index 79aff3e38ff0..06bc1d1134e5 100644 --- a/pkg/sql/colexec/lockop/util.go +++ b/pkg/sql/colexec/lockop/util.go @@ -1,17 +1,3 @@ -// Copyright 2023 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package lockop import ( From 873ffdf4f4a2832000a404a4b73b296960dc61c3 Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Tue, 5 Dec 2023 11:27:32 +0800 Subject: [PATCH 11/22] Revert "[mod] revert dml.go" This reverts commit 9e13b11af6f048be93c5343deba7e7c069923127. --- pkg/sql/compile/dml.go | 144 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 131 insertions(+), 13 deletions(-) diff --git a/pkg/sql/compile/dml.go b/pkg/sql/compile/dml.go index 190329f5dc20..d276f31be5b8 100644 --- a/pkg/sql/compile/dml.go +++ b/pkg/sql/compile/dml.go @@ -15,9 +15,14 @@ package compile import ( + "fmt" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/incrservice" + "github.com/matrixorigin/matrixone/pkg/pb/lock" "github.com/matrixorigin/matrixone/pkg/sql/colexec/deletion" "github.com/matrixorigin/matrixone/pkg/sql/colexec/insert" + "github.com/matrixorigin/matrixone/pkg/vm/engine" ) func (s *Scope) Delete(c *Compile) (uint64, error) { @@ -25,35 +30,142 @@ func (s *Scope) Delete(c *Compile) (uint64, error) { arg := s.Instructions[len(s.Instructions)-1].Arg.(*deletion.Argument) if arg.DeleteCtx.CanTruncate { + var dbSource engine.Database + var rel engine.Relation var err error - var affectRows int64 + var isTemp bool + var newId uint64 + delCtx := arg.DeleteCtx - err = delCtx.Source.UpdateObjectInfos(c.ctx) + dbName := delCtx.Ref.SchemaName + tblName := delCtx.Ref.ObjName + oldId := uint64(delCtx.Ref.Obj) + affectRows, err := delCtx.Source.Rows(c.ctx) + if err != nil { return 0, err } - affectRow, err := delCtx.Source.Rows(s.Proc.Ctx) + + dbSource, err = c.e.Database(c.ctx, dbName, c.proc.TxnOperator) if err != nil { return 0, err } - affectRows = affectRows + affectRow - dbName := delCtx.Ref.SchemaName - tblName := delCtx.Ref.ObjName - oldId := uint64(delCtx.Ref.Obj) - dbSource, err := c.e.Database(c.ctx, dbName, c.proc.TxnOperator) - if err != nil { - return 0, err + if rel, err = dbSource.Relation(c.ctx, tblName, nil); err != nil { + var e error // avoid contamination of error messages + dbSource, e = c.e.Database(c.ctx, defines.TEMPORARY_DBNAME, c.proc.TxnOperator) + if e != nil { + return 0, err + } + rel, e = dbSource.Relation(c.ctx, engine.GetTempTableName(dbName, tblName), nil) + if e != nil { + return 0, err + } + isTemp = true + } + + if !isTemp && c.proc.TxnOperator.Txn().IsPessimistic() { + var err error + if e := lockMoTable(c, dbName, tblName, lock.LockMode_Shared); e != nil { + if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && + !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { + return 0, e + } + err = e + } + // before dropping table, lock it. + if e := lockTable(c.ctx, c.e, c.proc, rel, dbName, delCtx.PartitionTableNames, false); e != nil { + if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && + !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { + return 0, e + } + err = e + } + if err != nil { + return 0, err + } + } + + if isTemp { + // memoryengine truncate always return 0, so for temporary table, just use origin tableId as newId + _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, tblName)) + newId = rel.GetTableID(c.ctx) + } else { + newId, err = dbSource.Truncate(c.ctx, tblName) } - // truncate origin table - newId, err := dbSource.Truncate(c.ctx, tblName) if err != nil { return 0, err } - // keep old offset. + // Truncate Index Tables if needed + for _, name := range delCtx.IndexTableNames { + var err error + if isTemp { + _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) + } else { + _, err = dbSource.Truncate(c.ctx, name) + } + if err != nil { + return 0, err + } + } + + //Truncate Partition subtable if needed + for _, name := range delCtx.PartitionTableNames { + var err error + if isTemp { + dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) + } else { + _, err = dbSource.Truncate(c.ctx, name) + } + if err != nil { + return 0, err + } + } + + // update tableDef of foreign key's table with new table id + for _, fTblId := range delCtx.ForeignTbl { + _, _, fkRelation, err := c.e.GetRelationById(c.ctx, c.proc.TxnOperator, fTblId) + if err != nil { + return 0, err + } + fkTableDef, err := fkRelation.TableDefs(c.ctx) + if err != nil { + return 0, err + } + var oldCt *engine.ConstraintDef + for _, def := range fkTableDef { + if ct, ok := def.(*engine.ConstraintDef); ok { + oldCt = ct + break + } + } + for _, ct := range oldCt.Cts { + if def, ok := ct.(*engine.RefChildTableDef); ok { + for idx, refTable := range def.Tables { + if refTable == oldId { + def.Tables[idx] = newId + break + } + } + break + } + } + if err != nil { + return 0, err + } + err = fkRelation.UpdateConstraint(c.ctx, oldCt) + if err != nil { + return 0, err + } + + } + + if isTemp { + oldId = rel.GetTableID(c.ctx) + } err = incrservice.GetAutoIncrementService(c.ctx).Reset( c.ctx, oldId, @@ -64,6 +176,12 @@ func (s *Scope) Delete(c *Compile) (uint64, error) { return 0, err } + // update index information in mo_catalog.mo_indexes + updateSql := fmt.Sprintf(updateMoIndexesTruncateTableFormat, newId, oldId) + err = c.runSql(updateSql) + if err != nil { + return 0, err + } return uint64(affectRows), nil } From 5038a6c4c7a4ed9526526c2c5341c31400caa5fb Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Tue, 5 Dec 2023 11:27:36 +0800 Subject: [PATCH 12/22] Revert "[refactor] move truncate to deletion operator" This reverts commit 05f9f2f145e338ab4e01566eec4f1a49331c128d. --- pkg/sql/colexec/deletion/deletion.go | 38 +-- pkg/sql/colexec/deletion/util.go | 208 --------------- pkg/sql/colexec/lockop/lock_op.go | 8 +- pkg/sql/colexec/lockop/lock_op_no_txn.go | 2 +- pkg/sql/colexec/lockop/util.go | 187 ------------- pkg/sql/compile/compile.go | 16 +- pkg/sql/compile/ddl.go | 324 +++++++++++++++++++++-- 7 files changed, 323 insertions(+), 460 deletions(-) delete mode 100644 pkg/sql/colexec/deletion/util.go delete mode 100644 pkg/sql/colexec/lockop/util.go diff --git a/pkg/sql/colexec/deletion/deletion.go b/pkg/sql/colexec/deletion/deletion.go index 23b0d5e537ab..e2805b69b4b2 100644 --- a/pkg/sql/colexec/deletion/deletion.go +++ b/pkg/sql/colexec/deletion/deletion.go @@ -16,8 +16,6 @@ package deletion import ( "bytes" - "github.com/matrixorigin/matrixone/pkg/defines" - "github.com/matrixorigin/matrixone/pkg/vm/engine" "sync/atomic" "github.com/matrixorigin/matrixone/pkg/catalog" @@ -162,41 +160,6 @@ func (arg *Argument) remote_delete(proc *process.Process) (vm.CallResult, error) } func (arg *Argument) normal_delete(proc *process.Process) (vm.CallResult, error) { - delCtx := arg.DeleteCtx - - if arg.DeleteCtx.CanTruncate { - dbName := delCtx.Ref.SchemaName - tableName := delCtx.Ref.ObjName - tableId := uint64(delCtx.Ref.Obj) - result := vm.NewCallResult() - affectRows, err := delCtx.Source.Rows(proc.Ctx) - eng := proc.Ctx.Value(defines.EngineKey{}).(engine.Engine) - if err != nil { - return result, err - } - - err = TruncateTable( - proc.Ctx, - eng, - proc, - dbName, - tableName, - tableId, - delCtx.PartitionTableNames, - delCtx.IndexTableNames, - delCtx.ForeignTbl, - true, - ) - - if err != nil { - return result, err - } - if delCtx.AddAffectedRows { - atomic.AddUint64(&arg.affectedRows, uint64(affectRows)) - } - return result, nil - } - result, err := arg.children[0].Call(proc) if err != nil { return result, err @@ -207,6 +170,7 @@ func (arg *Argument) normal_delete(proc *process.Process) (vm.CallResult, error) bat := result.Batch var affectedRows uint64 + delCtx := arg.DeleteCtx if len(delCtx.PartitionTableIDs) > 0 { delBatches, err := colexec.GroupByPartitionForDelete(proc, bat, delCtx.RowIdIdx, delCtx.PartitionIndexInBatch, diff --git a/pkg/sql/colexec/deletion/util.go b/pkg/sql/colexec/deletion/util.go deleted file mode 100644 index c0bc81f2e249..000000000000 --- a/pkg/sql/colexec/deletion/util.go +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright 2021 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package deletion - -import ( - "context" - "fmt" - "github.com/matrixorigin/matrixone/pkg/common/moerr" - moruntime "github.com/matrixorigin/matrixone/pkg/common/runtime" - "github.com/matrixorigin/matrixone/pkg/defines" - "github.com/matrixorigin/matrixone/pkg/incrservice" - "github.com/matrixorigin/matrixone/pkg/pb/lock" - "github.com/matrixorigin/matrixone/pkg/sql/colexec/lockop" - "github.com/matrixorigin/matrixone/pkg/util/executor" - "github.com/matrixorigin/matrixone/pkg/vm/engine" - "github.com/matrixorigin/matrixone/pkg/vm/process" -) - -func runSql(proc *process.Process, dbName string, sql string) error { - if sql == "" { - return nil - } - v, ok := moruntime.ProcessLevelRuntime().GetGlobalVariables(moruntime.InternalSQLExecutor) - if !ok { - panic("missing lock service") - } - exec := v.(executor.SQLExecutor) - opts := executor.Options{}. - // All runSql and runSqlWithResult is a part of input sql, can not incr statement. - // All these sub-sql's need to be rolled back and retried en masse when they conflict in pessimistic mode - WithDisableIncrStatement(). - WithTxn(proc.TxnOperator). - WithDatabase(dbName). - WithTimeZone(proc.SessionInfo.TimeZone) - res, err := exec.Exec(proc.Ctx, sql, opts) - if err != nil { - return err - } - res.Close() - return nil -} - -func TruncateTable( - ctx context.Context, - eng engine.Engine, - proc *process.Process, - dbName string, - tableName string, - tableId uint64, - partitionTableNames []string, - indexTableNames []string, - foreignTbl []uint64, - keepAutoIncrement bool, -) error { - var dbSource engine.Database - var rel engine.Relation - var err error - var isTemp bool - var newTblId uint64 - dbSource, err = eng.Database(ctx, dbName, proc.TxnOperator) - if err != nil { - return err - } - - if rel, err = dbSource.Relation(ctx, tableName, nil); err != nil { - var e error // avoid contamination of error messages - dbSource, e = eng.Database(ctx, defines.TEMPORARY_DBNAME, proc.TxnOperator) - if e != nil { - return err - } - rel, e = dbSource.Relation(ctx, engine.GetTempTableName(dbName, tableName), nil) - if e != nil { - return err - } - isTemp = true - } - - if !isTemp && proc.TxnOperator.Txn().IsPessimistic() { - var err error - if e := lockop.LockMoTable(ctx, eng, proc, dbName, tableName, lock.LockMode_Shared); e != nil { - if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && - !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { - return e - } - err = e - } - // before dropping table, lock it. - if e := lockop.LockTable(ctx, eng, proc, rel, dbName, partitionTableNames, false); e != nil { - if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && - !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { - return e - } - err = e - } - if err != nil { - return err - } - } - - if isTemp { - // memory engine truncate always return 0, so for temporary table, just use origin tableId as newTblId - _, err = dbSource.Truncate(ctx, engine.GetTempTableName(dbName, tableName)) - newTblId = rel.GetTableID(ctx) - } else { - newTblId, err = dbSource.Truncate(ctx, tableName) - } - - if err != nil { - return err - } - - // Truncate Index Tables if needed - for _, name := range indexTableNames { - var err error - if isTemp { - _, err = dbSource.Truncate(ctx, engine.GetTempTableName(dbName, name)) - } else { - _, err = dbSource.Truncate(ctx, name) - } - if err != nil { - return err - } - } - - //Truncate Partition subTable if needed - for _, name := range partitionTableNames { - var err error - if isTemp { - _, err = dbSource.Truncate(ctx, engine.GetTempTableName(dbName, name)) - } else { - _, err = dbSource.Truncate(ctx, name) - } - if err != nil { - return err - } - } - - // update tableDef of foreign key's table with new table id - for _, fTblId := range foreignTbl { - _, _, fkRelation, err := eng.GetRelationById(ctx, proc.TxnOperator, fTblId) - if err != nil { - return err - } - fkTableDef, err := fkRelation.TableDefs(ctx) - if err != nil { - return err - } - var oldCt *engine.ConstraintDef - for _, def := range fkTableDef { - if ct, ok := def.(*engine.ConstraintDef); ok { - oldCt = ct - break - } - } - for _, ct := range oldCt.Cts { - if def, ok := ct.(*engine.RefChildTableDef); ok { - for idx, refTable := range def.Tables { - if refTable == tableId { - def.Tables[idx] = newTblId - break - } - } - break - } - } - if err != nil { - return err - } - err = fkRelation.UpdateConstraint(ctx, oldCt) - if err != nil { - return err - } - - } - - if isTemp { - tableId = rel.GetTableID(ctx) - } - err = incrservice.GetAutoIncrementService(ctx).Reset( - ctx, - tableId, - newTblId, - keepAutoIncrement, - proc.TxnOperator) - if err != nil { - return err - } - - // update index information in mo_catalog.mo_indexes - updateSql := fmt.Sprintf(`update mo_catalog.mo_indexes set table_id = %v where table_id = %v`, newTblId, tableId) - err = runSql(proc, dbName, updateSql) - if err != nil { - return err - } - return nil -} diff --git a/pkg/sql/colexec/lockop/lock_op.go b/pkg/sql/colexec/lockop/lock_op.go index e11aa9477ca2..52323ba71750 100644 --- a/pkg/sql/colexec/lockop/lock_op.go +++ b/pkg/sql/colexec/lockop/lock_op.go @@ -269,9 +269,9 @@ func performLock( return nil } -// lockTable lock table, all rows in the table will be locked, and wait current txn +// LockTable lock table, all rows in the table will be locked, and wait current txn // closed. -func lockTable( +func LockTable( eng engine.Engine, proc *process.Process, tableID uint64, @@ -629,7 +629,7 @@ func (arg *Argument) AddLockTargetWithMode( return arg } -// lockTable lock all table, used for delete, truncate and drop table +// LockTable lock all table, used for delete, truncate and drop table func (arg *Argument) LockTable( tableID uint64, changeDef bool) *Argument { @@ -639,7 +639,7 @@ func (arg *Argument) LockTable( changeDef) } -// LockTableWithMode is similar to lockTable, but with specify +// LockTableWithMode is similar to LockTable, but with specify // lock mode func (arg *Argument) LockTableWithMode( tableID uint64, diff --git a/pkg/sql/colexec/lockop/lock_op_no_txn.go b/pkg/sql/colexec/lockop/lock_op_no_txn.go index f0c429ce6c26..8667f968f643 100644 --- a/pkg/sql/colexec/lockop/lock_op_no_txn.go +++ b/pkg/sql/colexec/lockop/lock_op_no_txn.go @@ -35,7 +35,7 @@ var ( internalProcesses = map[string]*process.Process{} ) -// LockTableWithUniqueID is similar to lockTable, but used to lock a table directly based on +// LockTableWithUniqueID is similar to LockTable, but used to lock a table directly based on // a unique identifier, without using an external transaction. func LockTableWithUniqueID( ctx context.Context, diff --git a/pkg/sql/colexec/lockop/util.go b/pkg/sql/colexec/lockop/util.go deleted file mode 100644 index 06bc1d1134e5..000000000000 --- a/pkg/sql/colexec/lockop/util.go +++ /dev/null @@ -1,187 +0,0 @@ -package lockop - -import ( - "context" - "github.com/matrixorigin/matrixone/pkg/catalog" - "github.com/matrixorigin/matrixone/pkg/container/types" - "github.com/matrixorigin/matrixone/pkg/container/vector" - "github.com/matrixorigin/matrixone/pkg/pb/lock" - "github.com/matrixorigin/matrixone/pkg/sql/plan/function" - "github.com/matrixorigin/matrixone/pkg/vm/engine" - "github.com/matrixorigin/matrixone/pkg/vm/process" -) - -func getLockVector(proc *process.Process, accountId uint32, names []string) (*vector.Vector, error) { - vecs := make([]*vector.Vector, len(names)+1) - defer func() { - for _, v := range vecs { - if v != nil { - proc.PutVector(v) - } - } - }() - - // append account_id - accountIdVec := proc.GetVector(types.T_uint32.ToType()) - err := vector.AppendFixed(accountIdVec, accountId, false, proc.GetMPool()) - if err != nil { - return nil, err - } - vecs[0] = accountIdVec - // append names - for i, name := range names { - nameVec := proc.GetVector(types.T_varchar.ToType()) - err := vector.AppendBytes(nameVec, []byte(name), false, proc.GetMPool()) - if err != nil { - return nil, err - } - vecs[i+1] = nameVec - } - - vec, err := function.RunFunctionDirectly(proc, function.SerialFunctionEncodeID, vecs, 1) - if err != nil { - return nil, err - } - return vec, nil -} - -func getRelFromMoCatalog( - ctx context.Context, - eng engine.Engine, - proc *process.Process, - tblName string, -) (engine.Relation, error) { - dbSource, err := eng.Database(ctx, catalog.MO_CATALOG, proc.TxnOperator) - if err != nil { - return nil, err - } - - rel, err := dbSource.Relation(ctx, tblName, nil) - if err != nil { - return nil, err - } - - return rel, nil -} - -func lockRows( - eng engine.Engine, - proc *process.Process, - rel engine.Relation, - vec *vector.Vector, - lockMode lock.LockMode) error { - - if vec == nil || vec.Length() == 0 { - panic("lock rows is empty") - } - - id := rel.GetTableID(proc.Ctx) - - err := LockRows( - eng, - proc, - id, - vec, - *vec.GetType(), - lockMode) - return err -} - -func doLockTable( - eng engine.Engine, - proc *process.Process, - rel engine.Relation, - defChanged bool) error { - id := rel.GetTableID(proc.Ctx) - defs, err := rel.GetPrimaryKeys(proc.Ctx) - if err != nil { - return err - } - - if len(defs) != 1 { - panic("invalid primary keys") - } - - err = lockTable( - eng, - proc, - id, - defs[0].Type, - defChanged) - - return err -} - -func LockTable( - ctx context.Context, - eng engine.Engine, - proc *process.Process, - rel engine.Relation, - dbName string, - partitionTableNames []string, - defChanged bool) error { - - if len(partitionTableNames) == 0 { - return doLockTable(eng, proc, rel, defChanged) - } - - dbSource, err := eng.Database(ctx, dbName, proc.TxnOperator) - if err != nil { - return err - } - - for _, tableName := range partitionTableNames { - pRel, pErr := dbSource.Relation(ctx, tableName, nil) - if pErr != nil { - return pErr - } - err = doLockTable(eng, proc, pRel, defChanged) - if err != nil { - return err - } - } - return nil -} - -func LockMoTable( - ctx context.Context, - eng engine.Engine, - proc *process.Process, - dbName string, - tblName string, - lockMode lock.LockMode, -) error { - dbRel, err := getRelFromMoCatalog(ctx, eng, proc, catalog.MO_TABLES) - if err != nil { - return err - } - vec, err := getLockVector(proc, proc.SessionInfo.AccountId, []string{dbName, tblName}) - if err != nil { - return err - } - defer vec.Free(proc.Mp()) - if err := lockRows(eng, proc, dbRel, vec, lockMode); err != nil { - return err - } - return nil -} - -func LockMoDatabase( - ctx context.Context, - eng engine.Engine, - proc *process.Process, - dbName string, -) error { - dbRel, err := getRelFromMoCatalog(ctx, eng, proc, catalog.MO_DATABASE) - if err != nil { - return err - } - vec, err := getLockVector(proc, proc.SessionInfo.AccountId, []string{dbName}) - if err != nil { - return err - } - if err := lockRows(eng, proc, dbRel, vec, lock.LockMode_Exclusive); err != nil { - return err - } - return nil -} diff --git a/pkg/sql/compile/compile.go b/pkg/sql/compile/compile.go index 0602299230e1..fcb044910629 100644 --- a/pkg/sql/compile/compile.go +++ b/pkg/sql/compile/compile.go @@ -1133,16 +1133,20 @@ func (c *Compile) compilePlanScope(ctx context.Context, step int32, curNodeIdx i case plan.Node_DELETE: c.appendMetaTables(n.DeleteCtx.Ref) if n.DeleteCtx.CanTruncate { - rs := c.newMergeScope(nil) arg, err := constructDeletion(n, c.e, c.proc) if err != nil { return nil, err } - rs.Instructions = append(rs.Instructions, vm.Instruction{ - Op: vm.Deletion, - Arg: arg, - }) - ss := []*Scope{rs} + ss := []*Scope{{ + Magic: Deletion, + Plan: c.pn, + Instructions: vm.Instructions{ + { + Op: vm.Deletion, + Arg: arg, + }, + }, + }} return ss, nil } curr := c.anal.curr diff --git a/pkg/sql/compile/ddl.go b/pkg/sql/compile/ddl.go index 4a1f9848d0ab..e5fb8878e429 100644 --- a/pkg/sql/compile/ddl.go +++ b/pkg/sql/compile/ddl.go @@ -17,7 +17,6 @@ package compile import ( "context" "fmt" - "github.com/matrixorigin/matrixone/pkg/sql/colexec/deletion" "math" "strings" @@ -36,6 +35,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/sql/parsers/dialect" "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" plan2 "github.com/matrixorigin/matrixone/pkg/sql/plan" + "github.com/matrixorigin/matrixone/pkg/sql/plan/function" "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/util/trace" "github.com/matrixorigin/matrixone/pkg/vm/engine" @@ -1238,13 +1238,164 @@ func (s *Scope) removeRefChildTbl(c *Compile, fkTblId uint64, tblId uint64) erro return fkRelation.UpdateConstraint(c.ctx, oldCt) } +func doTruncateTable( + c *Compile, + dbName string, + tableName string, + tableId uint64, + partitionTableNames []string, + indexTableNames []string, + foreignTbl []uint64, + keepAutoIncrement bool, +) error { + var dbSource engine.Database + var rel engine.Relation + var err error + var isTemp bool + var newTblId uint64 + dbSource, err = c.e.Database(c.ctx, dbName, c.proc.TxnOperator) + if err != nil { + return err + } + + if rel, err = dbSource.Relation(c.ctx, tableName, nil); err != nil { + var e error // avoid contamination of error messages + dbSource, e = c.e.Database(c.ctx, defines.TEMPORARY_DBNAME, c.proc.TxnOperator) + if e != nil { + return err + } + rel, e = dbSource.Relation(c.ctx, engine.GetTempTableName(dbName, tableName), nil) + if e != nil { + return err + } + isTemp = true + } + + if !isTemp && c.proc.TxnOperator.Txn().IsPessimistic() { + var err error + if e := lockMoTable(c, dbName, tableName, lock.LockMode_Shared); e != nil { + if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && + !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { + return e + } + err = e + } + // before dropping table, lock it. + if e := lockTable(c.ctx, c.e, c.proc, rel, dbName, partitionTableNames, false); e != nil { + if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && + !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { + return e + } + err = e + } + if err != nil { + return err + } + } + + if isTemp { + // memory engine truncate always return 0, so for temporary table, just use origin tableId as newTblId + _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, tableName)) + newTblId = rel.GetTableID(c.ctx) + } else { + newTblId, err = dbSource.Truncate(c.ctx, tableName) + } + + if err != nil { + return err + } + + // Truncate Index Tables if needed + for _, name := range indexTableNames { + var err error + if isTemp { + _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) + } else { + _, err = dbSource.Truncate(c.ctx, name) + } + if err != nil { + return err + } + } + + //Truncate Partition subTable if needed + for _, name := range partitionTableNames { + var err error + if isTemp { + _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) + } else { + _, err = dbSource.Truncate(c.ctx, name) + } + if err != nil { + return err + } + } + + // update tableDef of foreign key's table with new table id + for _, fTblId := range foreignTbl { + _, _, fkRelation, err := c.e.GetRelationById(c.ctx, c.proc.TxnOperator, fTblId) + if err != nil { + return err + } + fkTableDef, err := fkRelation.TableDefs(c.ctx) + if err != nil { + return err + } + var oldCt *engine.ConstraintDef + for _, def := range fkTableDef { + if ct, ok := def.(*engine.ConstraintDef); ok { + oldCt = ct + break + } + } + for _, ct := range oldCt.Cts { + if def, ok := ct.(*engine.RefChildTableDef); ok { + for idx, refTable := range def.Tables { + if refTable == tableId { + def.Tables[idx] = newTblId + break + } + } + break + } + } + if err != nil { + return err + } + err = fkRelation.UpdateConstraint(c.ctx, oldCt) + if err != nil { + return err + } + + } + + if isTemp { + tableId = rel.GetTableID(c.ctx) + } + err = incrservice.GetAutoIncrementService(c.ctx).Reset( + c.ctx, + tableId, + newTblId, + keepAutoIncrement, + c.proc.TxnOperator) + if err != nil { + return err + } + + // update index information in mo_catalog.mo_indexes + updateSql := fmt.Sprintf(updateMoIndexesTruncateTableFormat, newTblId, tableId) + err = c.runSql(updateSql) + if err != nil { + return err + } + return nil +} + // TruncateTable Truncation operations cannot be performed if the session holds an active table lock. func (s *Scope) TruncateTable(c *Compile) error { truncateTbl := s.Plan.GetDdl().GetTruncateTable() - return deletion.TruncateTable( - c.ctx, - c.e, - c.proc, + return doTruncateTable( + c, truncateTbl.GetDatabase(), truncateTbl.GetTable(), truncateTbl.GetTableId(), @@ -2194,6 +2345,85 @@ func getInterfaceValue[T constraints.Integer](val interface{}) T { return 0 } +func doLockTable( + eng engine.Engine, + proc *process.Process, + rel engine.Relation, + defChanged bool) error { + id := rel.GetTableID(proc.Ctx) + defs, err := rel.GetPrimaryKeys(proc.Ctx) + if err != nil { + return err + } + + if len(defs) != 1 { + panic("invalid primary keys") + } + + err = lockop.LockTable( + eng, + proc, + id, + defs[0].Type, + defChanged) + + return err +} + +func lockTable( + ctx context.Context, + eng engine.Engine, + proc *process.Process, + rel engine.Relation, + dbName string, + partitionTableNames []string, + defChanged bool) error { + + if len(partitionTableNames) == 0 { + return doLockTable(eng, proc, rel, defChanged) + } + + dbSource, err := eng.Database(ctx, dbName, proc.TxnOperator) + if err != nil { + return err + } + + for _, tableName := range partitionTableNames { + pRel, pErr := dbSource.Relation(ctx, tableName, nil) + if pErr != nil { + return pErr + } + err = doLockTable(eng, proc, pRel, defChanged) + if err != nil { + return err + } + } + return nil +} + +func lockRows( + eng engine.Engine, + proc *process.Process, + rel engine.Relation, + vec *vector.Vector, + lockMode lock.LockMode) error { + + if vec == nil || vec.Length() == 0 { + panic("lock rows is empty") + } + + id := rel.GetTableID(proc.Ctx) + + err := lockop.LockRows( + eng, + proc, + id, + vec, + *vec.GetType(), + lockMode) + return err +} + func maybeCreateAutoIncrement( ctx context.Context, db engine.Database, @@ -2222,21 +2452,81 @@ func maybeCreateAutoIncrement( txnOp) } -func lockMoDatabase(c *Compile, dbName string) error { - return lockop.LockMoDatabase(c.ctx, c.e, c.proc, dbName) +func getRelFromMoCatalog(c *Compile, tblName string) (engine.Relation, error) { + dbSource, err := c.e.Database(c.ctx, catalog.MO_CATALOG, c.proc.TxnOperator) + if err != nil { + return nil, err + } + + rel, err := dbSource.Relation(c.ctx, tblName, nil) + if err != nil { + return nil, err + } + + return rel, nil } -func lockTable( - ctx context.Context, - eng engine.Engine, - proc *process.Process, - rel engine.Relation, - dbName string, - partitionTableNames []string, - defChanged bool) error { - return lockop.LockTable(ctx, eng, proc, rel, dbName, partitionTableNames, defChanged) +func getLockVector(proc *process.Process, accountId uint32, names []string) (*vector.Vector, error) { + vecs := make([]*vector.Vector, len(names)+1) + defer func() { + for _, v := range vecs { + if v != nil { + proc.PutVector(v) + } + } + }() + + // append account_id + accountIdVec := proc.GetVector(types.T_uint32.ToType()) + err := vector.AppendFixed(accountIdVec, accountId, false, proc.GetMPool()) + if err != nil { + return nil, err + } + vecs[0] = accountIdVec + // append names + for i, name := range names { + nameVec := proc.GetVector(types.T_varchar.ToType()) + err := vector.AppendBytes(nameVec, []byte(name), false, proc.GetMPool()) + if err != nil { + return nil, err + } + vecs[i+1] = nameVec + } + + vec, err := function.RunFunctionDirectly(proc, function.SerialFunctionEncodeID, vecs, 1) + if err != nil { + return nil, err + } + return vec, nil +} + +func lockMoDatabase(c *Compile, dbName string) error { + dbRel, err := getRelFromMoCatalog(c, catalog.MO_DATABASE) + if err != nil { + return err + } + vec, err := getLockVector(c.proc, c.proc.SessionInfo.AccountId, []string{dbName}) + if err != nil { + return err + } + if err := lockRows(c.e, c.proc, dbRel, vec, lock.LockMode_Exclusive); err != nil { + return err + } + return nil } func lockMoTable(c *Compile, dbName string, tblName string, lockMode lock.LockMode) error { - return lockop.LockMoTable(c.ctx, c.e, c.proc, dbName, tblName, lockMode) + dbRel, err := getRelFromMoCatalog(c, catalog.MO_TABLES) + if err != nil { + return err + } + vec, err := getLockVector(c.proc, c.proc.SessionInfo.AccountId, []string{dbName, tblName}) + if err != nil { + return err + } + defer vec.Free(c.proc.Mp()) + if err := lockRows(c.e, c.proc, dbRel, vec, lockMode); err != nil { + return err + } + return nil } From 5751f4bce327b83e161fee5cd0a7343d640020b2 Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Tue, 5 Dec 2023 11:27:41 +0800 Subject: [PATCH 13/22] Revert "[refactor] truncate table operation" This reverts commit 3855d65e5cefd90c44d29385dddd7aa046e19dd2. --- pkg/sql/compile/ddl.go | 76 +++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 46 deletions(-) diff --git a/pkg/sql/compile/ddl.go b/pkg/sql/compile/ddl.go index e5fb8878e429..711ae9c266bc 100644 --- a/pkg/sql/compile/ddl.go +++ b/pkg/sql/compile/ddl.go @@ -1238,33 +1238,31 @@ func (s *Scope) removeRefChildTbl(c *Compile, fkTblId uint64, tblId uint64) erro return fkRelation.UpdateConstraint(c.ctx, oldCt) } -func doTruncateTable( - c *Compile, - dbName string, - tableName string, - tableId uint64, - partitionTableNames []string, - indexTableNames []string, - foreignTbl []uint64, - keepAutoIncrement bool, -) error { +// Truncation operations cannot be performed if the session holds an active table lock. +func (s *Scope) TruncateTable(c *Compile) error { var dbSource engine.Database var rel engine.Relation var err error var isTemp bool - var newTblId uint64 + var newId uint64 + + tqry := s.Plan.GetDdl().GetTruncateTable() + dbName := tqry.GetDatabase() + tblName := tqry.GetTable() + oldId := tqry.GetTableId() + dbSource, err = c.e.Database(c.ctx, dbName, c.proc.TxnOperator) if err != nil { return err } - if rel, err = dbSource.Relation(c.ctx, tableName, nil); err != nil { + if rel, err = dbSource.Relation(c.ctx, tblName, nil); err != nil { var e error // avoid contamination of error messages dbSource, e = c.e.Database(c.ctx, defines.TEMPORARY_DBNAME, c.proc.TxnOperator) if e != nil { return err } - rel, e = dbSource.Relation(c.ctx, engine.GetTempTableName(dbName, tableName), nil) + rel, e = dbSource.Relation(c.ctx, engine.GetTempTableName(dbName, tblName), nil) if e != nil { return err } @@ -1273,7 +1271,7 @@ func doTruncateTable( if !isTemp && c.proc.TxnOperator.Txn().IsPessimistic() { var err error - if e := lockMoTable(c, dbName, tableName, lock.LockMode_Shared); e != nil { + if e := lockMoTable(c, dbName, tblName, lock.LockMode_Shared); e != nil { if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { return e @@ -1281,7 +1279,7 @@ func doTruncateTable( err = e } // before dropping table, lock it. - if e := lockTable(c.ctx, c.e, c.proc, rel, dbName, partitionTableNames, false); e != nil { + if e := lockTable(c.ctx, c.e, c.proc, rel, dbName, tqry.PartitionTableNames, false); e != nil { if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { return e @@ -1294,11 +1292,11 @@ func doTruncateTable( } if isTemp { - // memory engine truncate always return 0, so for temporary table, just use origin tableId as newTblId - _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, tableName)) - newTblId = rel.GetTableID(c.ctx) + // memoryengine truncate always return 0, so for temporary table, just use origin tableId as newId + _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, tblName)) + newId = rel.GetTableID(c.ctx) } else { - newTblId, err = dbSource.Truncate(c.ctx, tableName) + newId, err = dbSource.Truncate(c.ctx, tblName) } if err != nil { @@ -1306,7 +1304,7 @@ func doTruncateTable( } // Truncate Index Tables if needed - for _, name := range indexTableNames { + for _, name := range tqry.IndexTableNames { var err error if isTemp { _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) @@ -1318,11 +1316,11 @@ func doTruncateTable( } } - //Truncate Partition subTable if needed - for _, name := range partitionTableNames { + //Truncate Partition subtable if needed + for _, name := range tqry.PartitionTableNames { var err error if isTemp { - _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) + dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) } else { _, err = dbSource.Truncate(c.ctx, name) } @@ -1332,8 +1330,8 @@ func doTruncateTable( } // update tableDef of foreign key's table with new table id - for _, fTblId := range foreignTbl { - _, _, fkRelation, err := c.e.GetRelationById(c.ctx, c.proc.TxnOperator, fTblId) + for _, ftblId := range tqry.ForeignTbl { + _, _, fkRelation, err := c.e.GetRelationById(c.ctx, c.proc.TxnOperator, ftblId) if err != nil { return err } @@ -1351,8 +1349,8 @@ func doTruncateTable( for _, ct := range oldCt.Cts { if def, ok := ct.(*engine.RefChildTableDef); ok { for idx, refTable := range def.Tables { - if refTable == tableId { - def.Tables[idx] = newTblId + if refTable == oldId { + def.Tables[idx] = newId break } } @@ -1370,20 +1368,20 @@ func doTruncateTable( } if isTemp { - tableId = rel.GetTableID(c.ctx) + oldId = rel.GetTableID(c.ctx) } err = incrservice.GetAutoIncrementService(c.ctx).Reset( c.ctx, - tableId, - newTblId, - keepAutoIncrement, + oldId, + newId, + false, c.proc.TxnOperator) if err != nil { return err } // update index information in mo_catalog.mo_indexes - updateSql := fmt.Sprintf(updateMoIndexesTruncateTableFormat, newTblId, tableId) + updateSql := fmt.Sprintf(updateMoIndexesTruncateTableFormat, newId, oldId) err = c.runSql(updateSql) if err != nil { return err @@ -1391,20 +1389,6 @@ func doTruncateTable( return nil } -// TruncateTable Truncation operations cannot be performed if the session holds an active table lock. -func (s *Scope) TruncateTable(c *Compile) error { - truncateTbl := s.Plan.GetDdl().GetTruncateTable() - return doTruncateTable( - c, - truncateTbl.GetDatabase(), - truncateTbl.GetTable(), - truncateTbl.GetTableId(), - truncateTbl.PartitionTableNames, - truncateTbl.IndexTableNames, - truncateTbl.ForeignTbl, - false) -} - func (s *Scope) DropSequence(c *Compile) error { qry := s.Plan.GetDdl().GetDropSequence() dbName := qry.GetDatabase() From 6937cec932229fbbb28f737c4d1ef8c60e9203df Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Tue, 5 Dec 2023 11:27:45 +0800 Subject: [PATCH 14/22] Revert "[mod] remove useless codes" This reverts commit 682d31fefdf311406c3ae4120982a46a5d1cb539. --- pkg/sql/compile/compile.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/sql/compile/compile.go b/pkg/sql/compile/compile.go index fcb044910629..4922ad401d24 100644 --- a/pkg/sql/compile/compile.go +++ b/pkg/sql/compile/compile.go @@ -2812,6 +2812,19 @@ func (c *Compile) newDeleteMergeScope(arg *deletion.Argument, ss []*Scope) *Scop return c.newMergeScope(rs) } +func (c *Compile) newDeleteScope(arg *deletion.Argument, ss []*Scope) []*Scope { + deleteIns := &vm.Instruction{ + Op: vm.Deletion, + Arg: arg, + } + for i := range ss { + ss[i].Instructions = append( + ss[i].Instructions, + dupInstruction(deleteIns, nil, 0)) + } + return ss +} + func (c *Compile) newMergeScope(ss []*Scope) *Scope { rs := &Scope{ PreScopes: ss, From bd06f249b2a8a67d9e0828c23ab7c0c4120c2582 Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Tue, 5 Dec 2023 11:27:51 +0800 Subject: [PATCH 15/22] Revert "[fix] the issue of being unable to delete special tables" This reverts commit 688bee7f571f74e4ffa2a06a297e4065585724c3. --- pkg/sql/plan/build_dml_util.go | 42 ++++++++++------------------------ 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/pkg/sql/plan/build_dml_util.go b/pkg/sql/plan/build_dml_util.go index df609bb08b82..8dd81b7ea17a 100644 --- a/pkg/sql/plan/build_dml_util.go +++ b/pkg/sql/plan/build_dml_util.go @@ -268,17 +268,8 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC // both UK and SK. To handle SK case, we will have flags to indicate if it's UK or SK. hasUniqueKey := haveUniqueKey(delCtx.tableDef) hasSecondaryKey := haveSecondaryKey(delCtx.tableDef) - - canTruncate := delCtx.isDeleteWithoutFilters - - if len(delCtx.tableDef.RefChildTbls) > 0 || - delCtx.tableDef.ViewSql != nil || - (util.TableIsClusterTable(delCtx.tableDef.GetTableType()) && ctx.GetAccountId() != catalog.System_Account) || - delCtx.objRef.PubInfo != nil { - canTruncate = false - } - - if (hasUniqueKey || hasSecondaryKey) && !canTruncate { + if (hasUniqueKey || hasSecondaryKey) && !delCtx.isDeleteWithoutFilters { + uniqueDeleteIdx := len(delCtx.tableDef.Cols) + delCtx.updateColLength typMap := make(map[string]*plan.Type) posMap := make(map[string]int) for idx, col := range delCtx.tableDef.Cols { @@ -311,26 +302,17 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC if uniqueTableDef == nil { return moerr.NewNoSuchTable(builder.GetContext(), delCtx.objRef.SchemaName, indexdef.IndexTableName) } - var lastNodeId int32 - var err error - var uniqueDeleteIdx int - var uniqueTblPkPos int - var uniqueTblPkTyp *Type - - if delCtx.isDeleteWithoutFilters { - lastNodeId, err = appendDeleteUniqueTablePlanWithoutFilters(builder, bindCtx, uniqueObjRef, uniqueTableDef) - uniqueDeleteIdx = getRowIdPos(uniqueTableDef) - uniqueTblPkPos, uniqueTblPkTyp = getPkPos(uniqueTableDef, false) - } else { - lastNodeId = appendSinkScanNode(builder, bindCtx, delCtx.sourceStep) - lastNodeId, err = appendDeleteUniqueTablePlan(builder, bindCtx, uniqueObjRef, uniqueTableDef, indexdef, typMap, posMap, lastNodeId, isUk) - uniqueDeleteIdx = len(delCtx.tableDef.Cols) + delCtx.updateColLength - uniqueTblPkPos = uniqueDeleteIdx + 1 - uniqueTblPkTyp = uniqueTableDef.Cols[0].Typ - } + lastNodeId := appendSinkScanNode(builder, bindCtx, delCtx.sourceStep) + + lastNodeId, err := appendDeleteUniqueTablePlan(builder, bindCtx, uniqueObjRef, uniqueTableDef, indexdef, typMap, posMap, lastNodeId, isUk) + if err != nil { return err } + + uniqueTblPkPos := uniqueDeleteIdx + 1 + uniqueTblPkTyp := uniqueTableDef.Cols[0].Typ + if isUpdate { // do it like simple update lastNodeId = appendSinkNode(builder, bindCtx, lastNodeId) @@ -390,7 +372,7 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC } else { // it's more simple for delete hidden unique table .so we append nodes after the plan. not recursive call buildDeletePlans delNodeInfo := makeDeleteNodeInfo(builder.compCtx, uniqueObjRef, uniqueTableDef, uniqueDeleteIdx, -1, false, uniqueTblPkPos, uniqueTblPkTyp, delCtx.lockTable, delCtx.partitionInfos) - lastNodeId, err = makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, isUk, isSK, false) + lastNodeId, err = makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, isUk, isSK, delCtx.isDeleteWithoutFilters) putDeleteNodeInfo(delNodeInfo) if err != nil { return err @@ -410,7 +392,7 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC } pkPos, pkTyp := getPkPos(delCtx.tableDef, false) delNodeInfo := makeDeleteNodeInfo(ctx, delCtx.objRef, delCtx.tableDef, delCtx.rowIdPos, partExprIdx, true, pkPos, pkTyp, delCtx.lockTable, delCtx.partitionInfos) - lastNodeId, err := makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, false, false, canTruncate) + lastNodeId, err := makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, false, false, delCtx.isDeleteWithoutFilters) putDeleteNodeInfo(delNodeInfo) if err != nil { return err From 46b673170bded88cc2c369685696cf5ba2f8b744 Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Tue, 5 Dec 2023 11:27:58 +0800 Subject: [PATCH 16/22] Revert "[perf] optimizing unconditional delete through truncate" This reverts commit 0ba1555e40057453533f8977ae419b2cb63a3d1f. --- pkg/sql/colexec/deletion/types.go | 8 +- pkg/sql/compile/compile.go | 30 ------- pkg/sql/compile/dml.go | 144 +++--------------------------- pkg/sql/compile/operator.go | 17 ---- pkg/sql/plan/build_dml_util.go | 39 ++++---- 5 files changed, 39 insertions(+), 199 deletions(-) diff --git a/pkg/sql/colexec/deletion/types.go b/pkg/sql/colexec/deletion/types.go index 773e2a9493b6..3286154b5831 100644 --- a/pkg/sql/colexec/deletion/types.go +++ b/pkg/sql/colexec/deletion/types.go @@ -98,11 +98,9 @@ func (arg *Argument) AppendChild(child vm.Operator) { type DeleteCtx struct { CanTruncate bool - RowIdIdx int // The array index position of the rowid column - PartitionTableIDs []uint64 // Align array index with the partition number - PartitionTableNames []string // Align array index with the partition number - IndexTableNames []string - ForeignTbl []uint64 + RowIdIdx int // The array index position of the rowid column + PartitionTableIDs []uint64 // Align array index with the partition number + PartitionTableNames []string // Align array index with the partition number PartitionIndexInBatch int // The array index position of the partition expression column PartitionSources []engine.Relation // Align array index with the partition number Source engine.Relation diff --git a/pkg/sql/compile/compile.go b/pkg/sql/compile/compile.go index 4922ad401d24..e2c14a623d11 100644 --- a/pkg/sql/compile/compile.go +++ b/pkg/sql/compile/compile.go @@ -1132,23 +1132,6 @@ func (c *Compile) compilePlanScope(ctx context.Context, step int32, curNodeIdx i return c.compileSort(n, c.compileUnionAll(left, right)), nil case plan.Node_DELETE: c.appendMetaTables(n.DeleteCtx.Ref) - if n.DeleteCtx.CanTruncate { - arg, err := constructDeletion(n, c.e, c.proc) - if err != nil { - return nil, err - } - ss := []*Scope{{ - Magic: Deletion, - Plan: c.pn, - Instructions: vm.Instructions{ - { - Op: vm.Deletion, - Arg: arg, - }, - }, - }} - return ss, nil - } curr := c.anal.curr c.setAnalyzeCurrent(nil, int(n.Children[0])) @@ -2812,19 +2795,6 @@ func (c *Compile) newDeleteMergeScope(arg *deletion.Argument, ss []*Scope) *Scop return c.newMergeScope(rs) } -func (c *Compile) newDeleteScope(arg *deletion.Argument, ss []*Scope) []*Scope { - deleteIns := &vm.Instruction{ - Op: vm.Deletion, - Arg: arg, - } - for i := range ss { - ss[i].Instructions = append( - ss[i].Instructions, - dupInstruction(deleteIns, nil, 0)) - } - return ss -} - func (c *Compile) newMergeScope(ss []*Scope) *Scope { rs := &Scope{ PreScopes: ss, diff --git a/pkg/sql/compile/dml.go b/pkg/sql/compile/dml.go index d276f31be5b8..190329f5dc20 100644 --- a/pkg/sql/compile/dml.go +++ b/pkg/sql/compile/dml.go @@ -15,14 +15,9 @@ package compile import ( - "fmt" - "github.com/matrixorigin/matrixone/pkg/common/moerr" - "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/incrservice" - "github.com/matrixorigin/matrixone/pkg/pb/lock" "github.com/matrixorigin/matrixone/pkg/sql/colexec/deletion" "github.com/matrixorigin/matrixone/pkg/sql/colexec/insert" - "github.com/matrixorigin/matrixone/pkg/vm/engine" ) func (s *Scope) Delete(c *Compile) (uint64, error) { @@ -30,142 +25,35 @@ func (s *Scope) Delete(c *Compile) (uint64, error) { arg := s.Instructions[len(s.Instructions)-1].Arg.(*deletion.Argument) if arg.DeleteCtx.CanTruncate { - var dbSource engine.Database - var rel engine.Relation var err error - var isTemp bool - var newId uint64 - + var affectRows int64 delCtx := arg.DeleteCtx - dbName := delCtx.Ref.SchemaName - tblName := delCtx.Ref.ObjName - oldId := uint64(delCtx.Ref.Obj) - affectRows, err := delCtx.Source.Rows(c.ctx) - + err = delCtx.Source.UpdateObjectInfos(c.ctx) if err != nil { return 0, err } - - dbSource, err = c.e.Database(c.ctx, dbName, c.proc.TxnOperator) + affectRow, err := delCtx.Source.Rows(s.Proc.Ctx) if err != nil { return 0, err } + affectRows = affectRows + affectRow - if rel, err = dbSource.Relation(c.ctx, tblName, nil); err != nil { - var e error // avoid contamination of error messages - dbSource, e = c.e.Database(c.ctx, defines.TEMPORARY_DBNAME, c.proc.TxnOperator) - if e != nil { - return 0, err - } - rel, e = dbSource.Relation(c.ctx, engine.GetTempTableName(dbName, tblName), nil) - if e != nil { - return 0, err - } - isTemp = true - } - - if !isTemp && c.proc.TxnOperator.Txn().IsPessimistic() { - var err error - if e := lockMoTable(c, dbName, tblName, lock.LockMode_Shared); e != nil { - if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && - !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { - return 0, e - } - err = e - } - // before dropping table, lock it. - if e := lockTable(c.ctx, c.e, c.proc, rel, dbName, delCtx.PartitionTableNames, false); e != nil { - if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && - !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { - return 0, e - } - err = e - } - if err != nil { - return 0, err - } - } - - if isTemp { - // memoryengine truncate always return 0, so for temporary table, just use origin tableId as newId - _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, tblName)) - newId = rel.GetTableID(c.ctx) - } else { - newId, err = dbSource.Truncate(c.ctx, tblName) - } - + dbName := delCtx.Ref.SchemaName + tblName := delCtx.Ref.ObjName + oldId := uint64(delCtx.Ref.Obj) + dbSource, err := c.e.Database(c.ctx, dbName, c.proc.TxnOperator) if err != nil { return 0, err } - // Truncate Index Tables if needed - for _, name := range delCtx.IndexTableNames { - var err error - if isTemp { - _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) - } else { - _, err = dbSource.Truncate(c.ctx, name) - } - if err != nil { - return 0, err - } - } - - //Truncate Partition subtable if needed - for _, name := range delCtx.PartitionTableNames { - var err error - if isTemp { - dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) - } else { - _, err = dbSource.Truncate(c.ctx, name) - } - if err != nil { - return 0, err - } - } - - // update tableDef of foreign key's table with new table id - for _, fTblId := range delCtx.ForeignTbl { - _, _, fkRelation, err := c.e.GetRelationById(c.ctx, c.proc.TxnOperator, fTblId) - if err != nil { - return 0, err - } - fkTableDef, err := fkRelation.TableDefs(c.ctx) - if err != nil { - return 0, err - } - var oldCt *engine.ConstraintDef - for _, def := range fkTableDef { - if ct, ok := def.(*engine.ConstraintDef); ok { - oldCt = ct - break - } - } - for _, ct := range oldCt.Cts { - if def, ok := ct.(*engine.RefChildTableDef); ok { - for idx, refTable := range def.Tables { - if refTable == oldId { - def.Tables[idx] = newId - break - } - } - break - } - } - if err != nil { - return 0, err - } - err = fkRelation.UpdateConstraint(c.ctx, oldCt) - if err != nil { - return 0, err - } - + // truncate origin table + newId, err := dbSource.Truncate(c.ctx, tblName) + if err != nil { + return 0, err } - if isTemp { - oldId = rel.GetTableID(c.ctx) - } + // keep old offset. err = incrservice.GetAutoIncrementService(c.ctx).Reset( c.ctx, oldId, @@ -176,12 +64,6 @@ func (s *Scope) Delete(c *Compile) (uint64, error) { return 0, err } - // update index information in mo_catalog.mo_indexes - updateSql := fmt.Sprintf(updateMoIndexesTruncateTableFormat, newId, oldId) - err = c.runSql(updateSql) - if err != nil { - return 0, err - } return uint64(affectRows), nil } diff --git a/pkg/sql/compile/operator.go b/pkg/sql/compile/operator.go index 70850122178f..bf0af3dd7e28 100644 --- a/pkg/sql/compile/operator.go +++ b/pkg/sql/compile/operator.go @@ -540,23 +540,6 @@ func constructDeletion(n *plan.Node, eg engine.Engine, proc *process.Process) (* } } - if delCtx.CanTruncate { - tableDef := delCtx.Source.GetTableDef(proc.Ctx) - delCtx.IndexTableNames = make([]string, 0) - if tableDef.Indexes != nil { - for _, indexDef := range tableDef.Indexes { - if indexDef.TableExist { - delCtx.IndexTableNames = append(delCtx.IndexTableNames, indexDef.IndexTableName) - } - } - } - if tableDef.Fkeys != nil { - for _, fk := range tableDef.Fkeys { - delCtx.ForeignTbl = append(delCtx.ForeignTbl, fk.ForeignTbl) - } - } - } - return &deletion.Argument{ DeleteCtx: delCtx, }, nil diff --git a/pkg/sql/plan/build_dml_util.go b/pkg/sql/plan/build_dml_util.go index 8dd81b7ea17a..5b971d4f7f46 100644 --- a/pkg/sql/plan/build_dml_util.go +++ b/pkg/sql/plan/build_dml_util.go @@ -268,8 +268,7 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC // both UK and SK. To handle SK case, we will have flags to indicate if it's UK or SK. hasUniqueKey := haveUniqueKey(delCtx.tableDef) hasSecondaryKey := haveSecondaryKey(delCtx.tableDef) - if (hasUniqueKey || hasSecondaryKey) && !delCtx.isDeleteWithoutFilters { - uniqueDeleteIdx := len(delCtx.tableDef.Cols) + delCtx.updateColLength + if hasUniqueKey || hasSecondaryKey { typMap := make(map[string]*plan.Type) posMap := make(map[string]int) for idx, col := range delCtx.tableDef.Cols { @@ -302,17 +301,26 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC if uniqueTableDef == nil { return moerr.NewNoSuchTable(builder.GetContext(), delCtx.objRef.SchemaName, indexdef.IndexTableName) } - lastNodeId := appendSinkScanNode(builder, bindCtx, delCtx.sourceStep) - - lastNodeId, err := appendDeleteUniqueTablePlan(builder, bindCtx, uniqueObjRef, uniqueTableDef, indexdef, typMap, posMap, lastNodeId, isUk) - + var lastNodeId int32 + var err error + var uniqueDeleteIdx int + var uniqueTblPkPos int + var uniqueTblPkTyp *Type + + if delCtx.isDeleteWithoutFilters { + lastNodeId, err = appendDeleteUniqueTablePlanWithoutFilters(builder, bindCtx, uniqueObjRef, uniqueTableDef) + uniqueDeleteIdx = getRowIdPos(uniqueTableDef) + uniqueTblPkPos, uniqueTblPkTyp = getPkPos(uniqueTableDef, false) + } else { + lastNodeId = appendSinkScanNode(builder, bindCtx, delCtx.sourceStep) + lastNodeId, err = appendDeleteUniqueTablePlan(builder, bindCtx, uniqueObjRef, uniqueTableDef, indexdef, typMap, posMap, lastNodeId, isUk) + uniqueDeleteIdx = len(delCtx.tableDef.Cols) + delCtx.updateColLength + uniqueTblPkPos = uniqueDeleteIdx + 1 + uniqueTblPkTyp = uniqueTableDef.Cols[0].Typ + } if err != nil { return err } - - uniqueTblPkPos := uniqueDeleteIdx + 1 - uniqueTblPkTyp := uniqueTableDef.Cols[0].Typ - if isUpdate { // do it like simple update lastNodeId = appendSinkNode(builder, bindCtx, lastNodeId) @@ -322,7 +330,7 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC //sink_scan -> lock -> delete lastNodeId = appendSinkScanNode(builder, bindCtx, newSourceStep) delNodeInfo := makeDeleteNodeInfo(builder.compCtx, uniqueObjRef, uniqueTableDef, uniqueDeleteIdx, -1, false, uniqueTblPkPos, uniqueTblPkTyp, delCtx.lockTable, delCtx.partitionInfos) - lastNodeId, err = makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, isUk, isSK, false) + lastNodeId, err = makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, isUk, isSK) putDeleteNodeInfo(delNodeInfo) if err != nil { return err @@ -372,7 +380,7 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC } else { // it's more simple for delete hidden unique table .so we append nodes after the plan. not recursive call buildDeletePlans delNodeInfo := makeDeleteNodeInfo(builder.compCtx, uniqueObjRef, uniqueTableDef, uniqueDeleteIdx, -1, false, uniqueTblPkPos, uniqueTblPkTyp, delCtx.lockTable, delCtx.partitionInfos) - lastNodeId, err = makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, isUk, isSK, delCtx.isDeleteWithoutFilters) + lastNodeId, err = makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, isUk, isSK) putDeleteNodeInfo(delNodeInfo) if err != nil { return err @@ -392,7 +400,7 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC } pkPos, pkTyp := getPkPos(delCtx.tableDef, false) delNodeInfo := makeDeleteNodeInfo(ctx, delCtx.objRef, delCtx.tableDef, delCtx.rowIdPos, partExprIdx, true, pkPos, pkTyp, delCtx.lockTable, delCtx.partitionInfos) - lastNodeId, err := makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, false, false, delCtx.isDeleteWithoutFilters) + lastNodeId, err := makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, false, false) putDeleteNodeInfo(delNodeInfo) if err != nil { return err @@ -1518,9 +1526,8 @@ func makeOneDeletePlan( bindCtx *BindContext, lastNodeId int32, delNodeInfo *deleteNodeInfo, - isUK bool, + isUK bool, // is delete unique key hidden table isSK bool, - canTruncate bool, ) (int32, error) { if isUK || isSK { // append lock @@ -1552,7 +1559,7 @@ func makeOneDeletePlan( DeleteCtx: &plan.DeleteCtx{ RowIdIdx: int32(delNodeInfo.deleteIndex), Ref: delNodeInfo.objRef, - CanTruncate: canTruncate, + CanTruncate: false, AddAffectedRows: delNodeInfo.addAffectedRows, IsClusterTable: delNodeInfo.IsClusterTable, PartitionTableIds: delNodeInfo.partTableIDs, From 42eae4a5c694958e73b2c0e16eddca21e902ce05 Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Tue, 5 Dec 2023 11:26:59 +0800 Subject: [PATCH 17/22] [perf] optimizing unconditional delete through truncate --- pkg/pb/plan/plan.pb.go | 97 ++++++++++++++++--------------- pkg/sql/colexec/deletion/types.go | 2 + pkg/sql/compile/compile.go | 14 +++++ pkg/sql/compile/ddl.go | 51 ++++++++++++---- pkg/sql/compile/operator.go | 17 +++++- pkg/sql/plan/build_dml_util.go | 28 +++++++-- proto/plan.proto | 2 + 7 files changed, 147 insertions(+), 64 deletions(-) diff --git a/pkg/pb/plan/plan.pb.go b/pkg/pb/plan/plan.pb.go index cbc2d5904aaf..1f5004a1e81c 100644 --- a/pkg/pb/plan/plan.pb.go +++ b/pkg/pb/plan/plan.pb.go @@ -518,9 +518,10 @@ const ( Node_PRE_INSERT_UK Node_NodeType = 62 Node_PRE_INSERT_SK Node_NodeType = 63 // - Node_TIME_WINDOW Node_NodeType = 64 - Node_Fill Node_NodeType = 65 - Node_PARTITION Node_NodeType = 66 + Node_TIME_WINDOW Node_NodeType = 64 + Node_Fill Node_NodeType = 65 + Node_PARTITION Node_NodeType = 66 + Node_DELETE_BY_TRUNCATE Node_NodeType = 67 ) var Node_NodeType_name = map[int32]string{ @@ -568,53 +569,55 @@ var Node_NodeType_name = map[int32]string{ 64: "TIME_WINDOW", 65: "Fill", 66: "PARTITION", + 67: "DELETE_BY_TRUNCATE", } var Node_NodeType_value = map[string]int32{ - "UNKNOWN": 0, - "VALUE_SCAN": 1, - "TABLE_SCAN": 2, - "FUNCTION_SCAN": 3, - "EXTERNAL_SCAN": 4, - "MATERIAL_SCAN": 5, - "SOURCE_SCAN": 6, - "PROJECT": 10, - "EXTERNAL_FUNCTION": 11, - "MATERIAL": 20, - "RECURSIVE_CTE": 21, - "SINK": 22, - "SINK_SCAN": 23, - "RECURSIVE_SCAN": 24, - "AGG": 30, - "DISTINCT": 31, - "FILTER": 32, - "JOIN": 33, - "SAMPLE": 34, - "SORT": 35, - "UNION": 36, - "UNION_ALL": 37, - "UNIQUE": 38, - "WINDOW": 39, - "BROADCAST": 40, - "SPLIT": 41, - "GATHER": 42, - "ASSERT": 50, - "INSERT": 51, - "DELETE": 52, - "REPLACE": 53, - "LOCK_OP": 54, - "INTERSECT": 55, - "INTERSECT_ALL": 56, - "MINUS": 57, - "MINUS_ALL": 58, - "ON_DUPLICATE_KEY": 59, - "PRE_INSERT": 60, - "PRE_DELETE": 61, - "PRE_INSERT_UK": 62, - "PRE_INSERT_SK": 63, - "TIME_WINDOW": 64, - "Fill": 65, - "PARTITION": 66, + "UNKNOWN": 0, + "VALUE_SCAN": 1, + "TABLE_SCAN": 2, + "FUNCTION_SCAN": 3, + "EXTERNAL_SCAN": 4, + "MATERIAL_SCAN": 5, + "SOURCE_SCAN": 6, + "PROJECT": 10, + "EXTERNAL_FUNCTION": 11, + "MATERIAL": 20, + "RECURSIVE_CTE": 21, + "SINK": 22, + "SINK_SCAN": 23, + "RECURSIVE_SCAN": 24, + "AGG": 30, + "DISTINCT": 31, + "FILTER": 32, + "JOIN": 33, + "SAMPLE": 34, + "SORT": 35, + "UNION": 36, + "UNION_ALL": 37, + "UNIQUE": 38, + "WINDOW": 39, + "BROADCAST": 40, + "SPLIT": 41, + "GATHER": 42, + "ASSERT": 50, + "INSERT": 51, + "DELETE": 52, + "REPLACE": 53, + "LOCK_OP": 54, + "INTERSECT": 55, + "INTERSECT_ALL": 56, + "MINUS": 57, + "MINUS_ALL": 58, + "ON_DUPLICATE_KEY": 59, + "PRE_INSERT": 60, + "PRE_DELETE": 61, + "PRE_INSERT_UK": 62, + "PRE_INSERT_SK": 63, + "TIME_WINDOW": 64, + "Fill": 65, + "PARTITION": 66, + "DELETE_BY_TRUNCATE": 67, } func (x Node_NodeType) String() string { diff --git a/pkg/sql/colexec/deletion/types.go b/pkg/sql/colexec/deletion/types.go index 3286154b5831..5fbd348fcf3b 100644 --- a/pkg/sql/colexec/deletion/types.go +++ b/pkg/sql/colexec/deletion/types.go @@ -103,6 +103,8 @@ type DeleteCtx struct { PartitionTableNames []string // Align array index with the partition number PartitionIndexInBatch int // The array index position of the partition expression column PartitionSources []engine.Relation // Align array index with the partition number + IndexTableNames []string + ForeignTbl []uint64 Source engine.Relation Ref *plan.ObjectRef AddAffectedRows bool diff --git a/pkg/sql/compile/compile.go b/pkg/sql/compile/compile.go index e2c14a623d11..cfce1da9a1ec 100644 --- a/pkg/sql/compile/compile.go +++ b/pkg/sql/compile/compile.go @@ -1176,6 +1176,20 @@ func (c *Compile) compilePlanScope(ctx context.Context, step int32, curNodeIdx i ss = []*Scope{rs} c.setAnalyzeCurrent(ss, curr) return ss, nil + case plan.Node_DELETE_BY_TRUNCATE: + arg, err := constructDeletion(n, c.e, c.proc) + if err != nil { + return nil, err + } + return []*Scope{{ + Magic: TruncateTable, + Instructions: vm.Instructions{ + vm.Instruction{ + Op: vm.Deletion, + Arg: arg, + }, + }, + }}, nil case plan.Node_ON_DUPLICATE_KEY: curr := c.anal.curr c.setAnalyzeCurrent(nil, int(n.Children[0])) diff --git a/pkg/sql/compile/ddl.go b/pkg/sql/compile/ddl.go index 711ae9c266bc..453c5b7631ad 100644 --- a/pkg/sql/compile/ddl.go +++ b/pkg/sql/compile/ddl.go @@ -17,6 +17,7 @@ package compile import ( "context" "fmt" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/deletion" "math" "strings" @@ -1245,12 +1246,41 @@ func (s *Scope) TruncateTable(c *Compile) error { var err error var isTemp bool var newId uint64 - + var dbName string + var tblName string + var oldId uint64 + var partitionTableNames []string + var indexTableNames []string + var foreignTbl []uint64 + keepAutoIncrement := false + affectRows := int64(0) tqry := s.Plan.GetDdl().GetTruncateTable() - dbName := tqry.GetDatabase() - tblName := tqry.GetTable() - oldId := tqry.GetTableId() - + if tqry != nil { + dbName = tqry.GetDatabase() + tblName = tqry.GetTable() + oldId = tqry.GetTableId() + partitionTableNames = tqry.GetPartitionTableNames() + indexTableNames = tqry.GetIndexTableNames() + foreignTbl = tqry.GetForeignTbl() + } else { + switch arg := s.Instructions[0].Arg.(type) { + case *deletion.Argument: + delCtx := arg.DeleteCtx + dbName = delCtx.Ref.SchemaName + tblName = delCtx.Ref.ObjName + oldId = uint64(delCtx.Ref.Obj) + partitionTableNames = delCtx.PartitionTableNames + indexTableNames = delCtx.IndexTableNames + foreignTbl = delCtx.ForeignTbl + keepAutoIncrement = true + affectRows, err = delCtx.Source.Rows(c.ctx) + if err != nil { + return err + } + default: + return moerr.NewNYI(c.ctx, fmt.Sprintf("truncate do not support instruction op '%v'", s.Instructions[0].Op)) + } + } dbSource, err = c.e.Database(c.ctx, dbName, c.proc.TxnOperator) if err != nil { return err @@ -1279,7 +1309,7 @@ func (s *Scope) TruncateTable(c *Compile) error { err = e } // before dropping table, lock it. - if e := lockTable(c.ctx, c.e, c.proc, rel, dbName, tqry.PartitionTableNames, false); e != nil { + if e := lockTable(c.ctx, c.e, c.proc, rel, dbName, partitionTableNames, false); e != nil { if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { return e @@ -1304,7 +1334,7 @@ func (s *Scope) TruncateTable(c *Compile) error { } // Truncate Index Tables if needed - for _, name := range tqry.IndexTableNames { + for _, name := range indexTableNames { var err error if isTemp { _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) @@ -1317,7 +1347,7 @@ func (s *Scope) TruncateTable(c *Compile) error { } //Truncate Partition subtable if needed - for _, name := range tqry.PartitionTableNames { + for _, name := range partitionTableNames { var err error if isTemp { dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) @@ -1330,7 +1360,7 @@ func (s *Scope) TruncateTable(c *Compile) error { } // update tableDef of foreign key's table with new table id - for _, ftblId := range tqry.ForeignTbl { + for _, ftblId := range foreignTbl { _, _, fkRelation, err := c.e.GetRelationById(c.ctx, c.proc.TxnOperator, ftblId) if err != nil { return err @@ -1374,7 +1404,7 @@ func (s *Scope) TruncateTable(c *Compile) error { c.ctx, oldId, newId, - false, + keepAutoIncrement, c.proc.TxnOperator) if err != nil { return err @@ -1386,6 +1416,7 @@ func (s *Scope) TruncateTable(c *Compile) error { if err != nil { return err } + c.setAffectedRows(uint64(affectRows)) return nil } diff --git a/pkg/sql/compile/operator.go b/pkg/sql/compile/operator.go index bf0af3dd7e28..608363126042 100644 --- a/pkg/sql/compile/operator.go +++ b/pkg/sql/compile/operator.go @@ -539,7 +539,22 @@ func constructDeletion(n *plan.Node, eg engine.Engine, proc *process.Process) (* delCtx.PartitionSources[i] = pRel } } - + if delCtx.CanTruncate { + tableDef := delCtx.Source.GetTableDef(proc.Ctx) + delCtx.IndexTableNames = make([]string, 0) + if tableDef.Indexes != nil { + for _, indexDef := range tableDef.Indexes { + if indexDef.TableExist { + delCtx.IndexTableNames = append(delCtx.IndexTableNames, indexDef.IndexTableName) + } + } + } + if tableDef.Fkeys != nil { + for _, fk := range tableDef.Fkeys { + delCtx.ForeignTbl = append(delCtx.ForeignTbl, fk.ForeignTbl) + } + } + } return &deletion.Argument{ DeleteCtx: delCtx, }, nil diff --git a/pkg/sql/plan/build_dml_util.go b/pkg/sql/plan/build_dml_util.go index 5b971d4f7f46..cddfd7ea32c6 100644 --- a/pkg/sql/plan/build_dml_util.go +++ b/pkg/sql/plan/build_dml_util.go @@ -268,7 +268,16 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC // both UK and SK. To handle SK case, we will have flags to indicate if it's UK or SK. hasUniqueKey := haveUniqueKey(delCtx.tableDef) hasSecondaryKey := haveSecondaryKey(delCtx.tableDef) - if hasUniqueKey || hasSecondaryKey { + canTruncate := delCtx.isDeleteWithoutFilters + + if len(delCtx.tableDef.RefChildTbls) > 0 || + delCtx.tableDef.ViewSql != nil || + (util.TableIsClusterTable(delCtx.tableDef.GetTableType()) && ctx.GetAccountId() != catalog.System_Account) || + delCtx.objRef.PubInfo != nil { + canTruncate = false + } + + if (hasUniqueKey || hasSecondaryKey) && !canTruncate { typMap := make(map[string]*plan.Type) posMap := make(map[string]int) for idx, col := range delCtx.tableDef.Cols { @@ -330,7 +339,7 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC //sink_scan -> lock -> delete lastNodeId = appendSinkScanNode(builder, bindCtx, newSourceStep) delNodeInfo := makeDeleteNodeInfo(builder.compCtx, uniqueObjRef, uniqueTableDef, uniqueDeleteIdx, -1, false, uniqueTblPkPos, uniqueTblPkTyp, delCtx.lockTable, delCtx.partitionInfos) - lastNodeId, err = makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, isUk, isSK) + lastNodeId, err = makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, isUk, isSK, false) putDeleteNodeInfo(delNodeInfo) if err != nil { return err @@ -380,7 +389,7 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC } else { // it's more simple for delete hidden unique table .so we append nodes after the plan. not recursive call buildDeletePlans delNodeInfo := makeDeleteNodeInfo(builder.compCtx, uniqueObjRef, uniqueTableDef, uniqueDeleteIdx, -1, false, uniqueTblPkPos, uniqueTblPkTyp, delCtx.lockTable, delCtx.partitionInfos) - lastNodeId, err = makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, isUk, isSK) + lastNodeId, err = makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, isUk, isSK, false) putDeleteNodeInfo(delNodeInfo) if err != nil { return err @@ -400,7 +409,7 @@ func buildDeletePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC } pkPos, pkTyp := getPkPos(delCtx.tableDef, false) delNodeInfo := makeDeleteNodeInfo(ctx, delCtx.objRef, delCtx.tableDef, delCtx.rowIdPos, partExprIdx, true, pkPos, pkTyp, delCtx.lockTable, delCtx.partitionInfos) - lastNodeId, err := makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, false, false) + lastNodeId, err := makeOneDeletePlan(builder, bindCtx, lastNodeId, delNodeInfo, false, false, canTruncate) putDeleteNodeInfo(delNodeInfo) if err != nil { return err @@ -1528,6 +1537,7 @@ func makeOneDeletePlan( delNodeInfo *deleteNodeInfo, isUK bool, // is delete unique key hidden table isSK bool, + canTruncate bool, ) (int32, error) { if isUK || isSK { // append lock @@ -1551,15 +1561,21 @@ func makeOneDeletePlan( lastNodeId = builder.appendNode(lockNode, bindCtx) } + nodeType := plan.Node_DELETE + + if canTruncate { + nodeType = plan.Node_DELETE_BY_TRUNCATE + } + // append delete node deleteNode := &Node{ - NodeType: plan.Node_DELETE, + NodeType: nodeType, Children: []int32{lastNodeId}, // ProjectList: getProjectionByLastNode(builder, lastNodeId), DeleteCtx: &plan.DeleteCtx{ RowIdIdx: int32(delNodeInfo.deleteIndex), Ref: delNodeInfo.objRef, - CanTruncate: false, + CanTruncate: canTruncate, AddAffectedRows: delNodeInfo.addAffectedRows, IsClusterTable: delNodeInfo.IsClusterTable, PartitionTableIds: delNodeInfo.partTableIDs, diff --git a/proto/plan.proto b/proto/plan.proto index ef1c67d98cc3..7d233941dd7b 100644 --- a/proto/plan.proto +++ b/proto/plan.proto @@ -689,6 +689,8 @@ message Node { TIME_WINDOW = 64; Fill = 65; PARTITION = 66; + + DELETE_BY_TRUNCATE = 67; } enum JoinType { From 8a2348cec1bb1b5818e4dc0af835a5fb0a7602bd Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Tue, 5 Dec 2023 16:38:33 +0800 Subject: [PATCH 18/22] [fix] explain error case by delete by truncate node --- pkg/sql/plan/explain/explain_node.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/sql/plan/explain/explain_node.go b/pkg/sql/plan/explain/explain_node.go index 4bafab096307..41c25e8acc82 100644 --- a/pkg/sql/plan/explain/explain_node.go +++ b/pkg/sql/plan/explain/explain_node.go @@ -114,6 +114,8 @@ func (ndesc *NodeDescribeImpl) GetNodeBasicInfo(ctx context.Context, options *Ex pname = "Insert" case plan.Node_DELETE: pname = "Delete" + case plan.Node_DELETE_BY_TRUNCATE: + pname = "Delete by truncate" case plan.Node_INTERSECT: pname = "Intersect" case plan.Node_INTERSECT_ALL: From bb6e9c616fbc27221f1b2f906204279d8862fa9c Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Thu, 7 Dec 2023 11:29:47 +0800 Subject: [PATCH 19/22] [mod] remove delete by truncate node --- pkg/pb/plan/plan.pb.go | 97 ++++++++++++++-------------- pkg/sql/compile/compile.go | 29 +++++---- pkg/sql/plan/build_dml_util.go | 8 +-- pkg/sql/plan/explain/explain_node.go | 2 - proto/plan.proto | 2 - 5 files changed, 63 insertions(+), 75 deletions(-) diff --git a/pkg/pb/plan/plan.pb.go b/pkg/pb/plan/plan.pb.go index 1f5004a1e81c..cbc2d5904aaf 100644 --- a/pkg/pb/plan/plan.pb.go +++ b/pkg/pb/plan/plan.pb.go @@ -518,10 +518,9 @@ const ( Node_PRE_INSERT_UK Node_NodeType = 62 Node_PRE_INSERT_SK Node_NodeType = 63 // - Node_TIME_WINDOW Node_NodeType = 64 - Node_Fill Node_NodeType = 65 - Node_PARTITION Node_NodeType = 66 - Node_DELETE_BY_TRUNCATE Node_NodeType = 67 + Node_TIME_WINDOW Node_NodeType = 64 + Node_Fill Node_NodeType = 65 + Node_PARTITION Node_NodeType = 66 ) var Node_NodeType_name = map[int32]string{ @@ -569,55 +568,53 @@ var Node_NodeType_name = map[int32]string{ 64: "TIME_WINDOW", 65: "Fill", 66: "PARTITION", - 67: "DELETE_BY_TRUNCATE", } var Node_NodeType_value = map[string]int32{ - "UNKNOWN": 0, - "VALUE_SCAN": 1, - "TABLE_SCAN": 2, - "FUNCTION_SCAN": 3, - "EXTERNAL_SCAN": 4, - "MATERIAL_SCAN": 5, - "SOURCE_SCAN": 6, - "PROJECT": 10, - "EXTERNAL_FUNCTION": 11, - "MATERIAL": 20, - "RECURSIVE_CTE": 21, - "SINK": 22, - "SINK_SCAN": 23, - "RECURSIVE_SCAN": 24, - "AGG": 30, - "DISTINCT": 31, - "FILTER": 32, - "JOIN": 33, - "SAMPLE": 34, - "SORT": 35, - "UNION": 36, - "UNION_ALL": 37, - "UNIQUE": 38, - "WINDOW": 39, - "BROADCAST": 40, - "SPLIT": 41, - "GATHER": 42, - "ASSERT": 50, - "INSERT": 51, - "DELETE": 52, - "REPLACE": 53, - "LOCK_OP": 54, - "INTERSECT": 55, - "INTERSECT_ALL": 56, - "MINUS": 57, - "MINUS_ALL": 58, - "ON_DUPLICATE_KEY": 59, - "PRE_INSERT": 60, - "PRE_DELETE": 61, - "PRE_INSERT_UK": 62, - "PRE_INSERT_SK": 63, - "TIME_WINDOW": 64, - "Fill": 65, - "PARTITION": 66, - "DELETE_BY_TRUNCATE": 67, + "UNKNOWN": 0, + "VALUE_SCAN": 1, + "TABLE_SCAN": 2, + "FUNCTION_SCAN": 3, + "EXTERNAL_SCAN": 4, + "MATERIAL_SCAN": 5, + "SOURCE_SCAN": 6, + "PROJECT": 10, + "EXTERNAL_FUNCTION": 11, + "MATERIAL": 20, + "RECURSIVE_CTE": 21, + "SINK": 22, + "SINK_SCAN": 23, + "RECURSIVE_SCAN": 24, + "AGG": 30, + "DISTINCT": 31, + "FILTER": 32, + "JOIN": 33, + "SAMPLE": 34, + "SORT": 35, + "UNION": 36, + "UNION_ALL": 37, + "UNIQUE": 38, + "WINDOW": 39, + "BROADCAST": 40, + "SPLIT": 41, + "GATHER": 42, + "ASSERT": 50, + "INSERT": 51, + "DELETE": 52, + "REPLACE": 53, + "LOCK_OP": 54, + "INTERSECT": 55, + "INTERSECT_ALL": 56, + "MINUS": 57, + "MINUS_ALL": 58, + "ON_DUPLICATE_KEY": 59, + "PRE_INSERT": 60, + "PRE_DELETE": 61, + "PRE_INSERT_UK": 62, + "PRE_INSERT_SK": 63, + "TIME_WINDOW": 64, + "Fill": 65, + "PARTITION": 66, } func (x Node_NodeType) String() string { diff --git a/pkg/sql/compile/compile.go b/pkg/sql/compile/compile.go index cfce1da9a1ec..7cea565b3f28 100644 --- a/pkg/sql/compile/compile.go +++ b/pkg/sql/compile/compile.go @@ -1131,6 +1131,21 @@ func (c *Compile) compilePlanScope(ctx context.Context, step int32, curNodeIdx i c.setAnalyzeCurrent(right, curr) return c.compileSort(n, c.compileUnionAll(left, right)), nil case plan.Node_DELETE: + if n.DeleteCtx.CanTruncate { + arg, err := constructDeletion(n, c.e, c.proc) + if err != nil { + return nil, err + } + return []*Scope{{ + Magic: TruncateTable, + Instructions: vm.Instructions{ + vm.Instruction{ + Op: vm.Deletion, + Arg: arg, + }, + }, + }}, nil + } c.appendMetaTables(n.DeleteCtx.Ref) curr := c.anal.curr c.setAnalyzeCurrent(nil, int(n.Children[0])) @@ -1176,20 +1191,6 @@ func (c *Compile) compilePlanScope(ctx context.Context, step int32, curNodeIdx i ss = []*Scope{rs} c.setAnalyzeCurrent(ss, curr) return ss, nil - case plan.Node_DELETE_BY_TRUNCATE: - arg, err := constructDeletion(n, c.e, c.proc) - if err != nil { - return nil, err - } - return []*Scope{{ - Magic: TruncateTable, - Instructions: vm.Instructions{ - vm.Instruction{ - Op: vm.Deletion, - Arg: arg, - }, - }, - }}, nil case plan.Node_ON_DUPLICATE_KEY: curr := c.anal.curr c.setAnalyzeCurrent(nil, int(n.Children[0])) diff --git a/pkg/sql/plan/build_dml_util.go b/pkg/sql/plan/build_dml_util.go index cddfd7ea32c6..44abcebe77fe 100644 --- a/pkg/sql/plan/build_dml_util.go +++ b/pkg/sql/plan/build_dml_util.go @@ -1561,15 +1561,9 @@ func makeOneDeletePlan( lastNodeId = builder.appendNode(lockNode, bindCtx) } - nodeType := plan.Node_DELETE - - if canTruncate { - nodeType = plan.Node_DELETE_BY_TRUNCATE - } - // append delete node deleteNode := &Node{ - NodeType: nodeType, + NodeType: plan.Node_DELETE, Children: []int32{lastNodeId}, // ProjectList: getProjectionByLastNode(builder, lastNodeId), DeleteCtx: &plan.DeleteCtx{ diff --git a/pkg/sql/plan/explain/explain_node.go b/pkg/sql/plan/explain/explain_node.go index 41c25e8acc82..4bafab096307 100644 --- a/pkg/sql/plan/explain/explain_node.go +++ b/pkg/sql/plan/explain/explain_node.go @@ -114,8 +114,6 @@ func (ndesc *NodeDescribeImpl) GetNodeBasicInfo(ctx context.Context, options *Ex pname = "Insert" case plan.Node_DELETE: pname = "Delete" - case plan.Node_DELETE_BY_TRUNCATE: - pname = "Delete by truncate" case plan.Node_INTERSECT: pname = "Intersect" case plan.Node_INTERSECT_ALL: diff --git a/proto/plan.proto b/proto/plan.proto index 7d233941dd7b..ef1c67d98cc3 100644 --- a/proto/plan.proto +++ b/proto/plan.proto @@ -689,8 +689,6 @@ message Node { TIME_WINDOW = 64; Fill = 65; PARTITION = 66; - - DELETE_BY_TRUNCATE = 67; } enum JoinType { From 5199899077214d8fbf4e14247c8d63af5ae3a3bb Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Thu, 7 Dec 2023 14:12:49 +0800 Subject: [PATCH 20/22] [mod] add truncate table for deleteCtx --- pkg/pb/plan/plan.pb.go | 10 ++++--- pkg/sql/compile/compile.go | 20 +++++++------ pkg/sql/compile/ddl.go | 51 +++++++--------------------------- pkg/sql/compile/operator.go | 17 +----------- pkg/sql/plan/build_dml_util.go | 29 ++++++++++++++++++- proto/plan.proto | 1 + 6 files changed, 57 insertions(+), 71 deletions(-) diff --git a/pkg/pb/plan/plan.pb.go b/pkg/pb/plan/plan.pb.go index cbc2d5904aaf..a8375d3d1e61 100644 --- a/pkg/pb/plan/plan.pb.go +++ b/pkg/pb/plan/plan.pb.go @@ -6742,10 +6742,12 @@ type DeleteCtx struct { AddAffectedRows bool `protobuf:"varint,4,opt,name=add_affected_rows,json=addAffectedRows,proto3" json:"add_affected_rows,omitempty"` IsClusterTable bool `protobuf:"varint,5,opt,name=is_cluster_table,json=isClusterTable,proto3" json:"is_cluster_table,omitempty"` // Align array index with the partition number - PartitionTableIds []uint64 `protobuf:"varint,6,rep,packed,name=partition_table_ids,json=partitionTableIds,proto3" json:"partition_table_ids,omitempty"` - PartitionTableNames []string `protobuf:"bytes,7,rep,name=partition_table_names,json=partitionTableNames,proto3" json:"partition_table_names,omitempty"` - PartitionIdx int32 `protobuf:"varint,8,opt,name=partition_idx,json=partitionIdx,proto3" json:"partition_idx,omitempty"` - PrimaryKeyIdx int32 `protobuf:"varint,9,opt,name=primary_key_idx,json=primaryKeyIdx,proto3" json:"primary_key_idx,omitempty"` + PartitionTableIds []uint64 `protobuf:"varint,6,rep,packed,name=partition_table_ids,json=partitionTableIds,proto3" json:"partition_table_ids,omitempty"` + PartitionTableNames []string `protobuf:"bytes,7,rep,name=partition_table_names,json=partitionTableNames,proto3" json:"partition_table_names,omitempty"` + PartitionIdx int32 `protobuf:"varint,8,opt,name=partition_idx,json=partitionIdx,proto3" json:"partition_idx,omitempty"` + PrimaryKeyIdx int32 `protobuf:"varint,9,opt,name=primary_key_idx,json=primaryKeyIdx,proto3" json:"primary_key_idx,omitempty"` + TruncateTable *TruncateTable `protobuf:"bytes,12,opt,name=truncate_table,json=truncateTable,proto3,oneof" json:"truncate_table,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` diff --git a/pkg/sql/compile/compile.go b/pkg/sql/compile/compile.go index 7cea565b3f28..36ca8e63f239 100644 --- a/pkg/sql/compile/compile.go +++ b/pkg/sql/compile/compile.go @@ -569,7 +569,9 @@ func (c *Compile) compileScope(ctx context.Context, pn *plan.Plan) ([]*Scope, er return nil, err } for _, s := range scopes { - s.Plan = pn + if s.Plan == nil { + s.Plan = pn + } } return scopes, nil case *plan.Plan_Ddl: @@ -1132,16 +1134,16 @@ func (c *Compile) compilePlanScope(ctx context.Context, step int32, curNodeIdx i return c.compileSort(n, c.compileUnionAll(left, right)), nil case plan.Node_DELETE: if n.DeleteCtx.CanTruncate { - arg, err := constructDeletion(n, c.e, c.proc) - if err != nil { - return nil, err - } return []*Scope{{ Magic: TruncateTable, - Instructions: vm.Instructions{ - vm.Instruction{ - Op: vm.Deletion, - Arg: arg, + Plan: &plan.Plan{ + Plan: &plan.Plan_Ddl{ + Ddl: &plan.DataDefinition{ + DdlType: plan.DataDefinition_TRUNCATE_TABLE, + Definition: &plan.DataDefinition_TruncateTable{ + TruncateTable: n.DeleteCtx.TruncateTable, + }, + }, }, }, }}, nil diff --git a/pkg/sql/compile/ddl.go b/pkg/sql/compile/ddl.go index 453c5b7631ad..711ae9c266bc 100644 --- a/pkg/sql/compile/ddl.go +++ b/pkg/sql/compile/ddl.go @@ -17,7 +17,6 @@ package compile import ( "context" "fmt" - "github.com/matrixorigin/matrixone/pkg/sql/colexec/deletion" "math" "strings" @@ -1246,41 +1245,12 @@ func (s *Scope) TruncateTable(c *Compile) error { var err error var isTemp bool var newId uint64 - var dbName string - var tblName string - var oldId uint64 - var partitionTableNames []string - var indexTableNames []string - var foreignTbl []uint64 - keepAutoIncrement := false - affectRows := int64(0) + tqry := s.Plan.GetDdl().GetTruncateTable() - if tqry != nil { - dbName = tqry.GetDatabase() - tblName = tqry.GetTable() - oldId = tqry.GetTableId() - partitionTableNames = tqry.GetPartitionTableNames() - indexTableNames = tqry.GetIndexTableNames() - foreignTbl = tqry.GetForeignTbl() - } else { - switch arg := s.Instructions[0].Arg.(type) { - case *deletion.Argument: - delCtx := arg.DeleteCtx - dbName = delCtx.Ref.SchemaName - tblName = delCtx.Ref.ObjName - oldId = uint64(delCtx.Ref.Obj) - partitionTableNames = delCtx.PartitionTableNames - indexTableNames = delCtx.IndexTableNames - foreignTbl = delCtx.ForeignTbl - keepAutoIncrement = true - affectRows, err = delCtx.Source.Rows(c.ctx) - if err != nil { - return err - } - default: - return moerr.NewNYI(c.ctx, fmt.Sprintf("truncate do not support instruction op '%v'", s.Instructions[0].Op)) - } - } + dbName := tqry.GetDatabase() + tblName := tqry.GetTable() + oldId := tqry.GetTableId() + dbSource, err = c.e.Database(c.ctx, dbName, c.proc.TxnOperator) if err != nil { return err @@ -1309,7 +1279,7 @@ func (s *Scope) TruncateTable(c *Compile) error { err = e } // before dropping table, lock it. - if e := lockTable(c.ctx, c.e, c.proc, rel, dbName, partitionTableNames, false); e != nil { + if e := lockTable(c.ctx, c.e, c.proc, rel, dbName, tqry.PartitionTableNames, false); e != nil { if !moerr.IsMoErrCode(e, moerr.ErrTxnNeedRetry) && !moerr.IsMoErrCode(err, moerr.ErrTxnNeedRetryWithDefChanged) { return e @@ -1334,7 +1304,7 @@ func (s *Scope) TruncateTable(c *Compile) error { } // Truncate Index Tables if needed - for _, name := range indexTableNames { + for _, name := range tqry.IndexTableNames { var err error if isTemp { _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) @@ -1347,7 +1317,7 @@ func (s *Scope) TruncateTable(c *Compile) error { } //Truncate Partition subtable if needed - for _, name := range partitionTableNames { + for _, name := range tqry.PartitionTableNames { var err error if isTemp { dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, name)) @@ -1360,7 +1330,7 @@ func (s *Scope) TruncateTable(c *Compile) error { } // update tableDef of foreign key's table with new table id - for _, ftblId := range foreignTbl { + for _, ftblId := range tqry.ForeignTbl { _, _, fkRelation, err := c.e.GetRelationById(c.ctx, c.proc.TxnOperator, ftblId) if err != nil { return err @@ -1404,7 +1374,7 @@ func (s *Scope) TruncateTable(c *Compile) error { c.ctx, oldId, newId, - keepAutoIncrement, + false, c.proc.TxnOperator) if err != nil { return err @@ -1416,7 +1386,6 @@ func (s *Scope) TruncateTable(c *Compile) error { if err != nil { return err } - c.setAffectedRows(uint64(affectRows)) return nil } diff --git a/pkg/sql/compile/operator.go b/pkg/sql/compile/operator.go index 608363126042..bf0af3dd7e28 100644 --- a/pkg/sql/compile/operator.go +++ b/pkg/sql/compile/operator.go @@ -539,22 +539,7 @@ func constructDeletion(n *plan.Node, eg engine.Engine, proc *process.Process) (* delCtx.PartitionSources[i] = pRel } } - if delCtx.CanTruncate { - tableDef := delCtx.Source.GetTableDef(proc.Ctx) - delCtx.IndexTableNames = make([]string, 0) - if tableDef.Indexes != nil { - for _, indexDef := range tableDef.Indexes { - if indexDef.TableExist { - delCtx.IndexTableNames = append(delCtx.IndexTableNames, indexDef.IndexTableName) - } - } - } - if tableDef.Fkeys != nil { - for _, fk := range tableDef.Fkeys { - delCtx.ForeignTbl = append(delCtx.ForeignTbl, fk.ForeignTbl) - } - } - } + return &deletion.Argument{ DeleteCtx: delCtx, }, nil diff --git a/pkg/sql/plan/build_dml_util.go b/pkg/sql/plan/build_dml_util.go index 44abcebe77fe..575c4a823706 100644 --- a/pkg/sql/plan/build_dml_util.go +++ b/pkg/sql/plan/build_dml_util.go @@ -97,6 +97,8 @@ type deleteNodeInfo struct { partTableIDs []uint64 // Align array index with the partition number partTableNames []string // Align array index with the partition number partitionIdx int // The array index position of the partition expression column + indexTableNames []string + foreignTbl []uint64 addAffectedRows bool pkPos int pkTyp *plan.Type @@ -1560,7 +1562,19 @@ func makeOneDeletePlan( } lastNodeId = builder.appendNode(lockNode, bindCtx) } - + truncateTable := &plan.TruncateTable{} + if canTruncate { + tableDef := delNodeInfo.tableDef + truncateTable.Table = tableDef.Name + truncateTable.TableId = tableDef.TblId + truncateTable.Database = delNodeInfo.objRef.SchemaName + truncateTable.IndexTableNames = delNodeInfo.indexTableNames + truncateTable.PartitionTableNames = delNodeInfo.partTableNames + truncateTable.ForeignTbl = delNodeInfo.foreignTbl + truncateTable.ClusterTable = &plan.ClusterTable{ + IsClusterTable: util.TableIsClusterTable(tableDef.GetTableType()), + } + } // append delete node deleteNode := &Node{ NodeType: plan.Node_DELETE, @@ -1576,6 +1590,7 @@ func makeOneDeletePlan( PartitionTableNames: delNodeInfo.partTableNames, PartitionIdx: int32(delNodeInfo.partitionIdx), PrimaryKeyIdx: int32(delNodeInfo.pkPos), + TruncateTable: truncateTable, }, } lastNodeId = builder.appendNode(deleteNode, bindCtx) @@ -1688,6 +1703,18 @@ func makeDeleteNodeInfo(ctx CompilerContext, objRef *ObjectRef, tableDef *TableD } } + if tableDef.Fkeys != nil { + for _, fk := range tableDef.Fkeys { + delNodeInfo.foreignTbl = append(delNodeInfo.foreignTbl, fk.ForeignTbl) + } + } + if tableDef.Indexes != nil { + for _, indexdef := range tableDef.Indexes { + if indexdef.TableExist { + delNodeInfo.indexTableNames = append(delNodeInfo.indexTableNames, indexdef.IndexTableName) + } + } + } return delNodeInfo } diff --git a/proto/plan.proto b/proto/plan.proto index ef1c67d98cc3..2b2679ed22d6 100644 --- a/proto/plan.proto +++ b/proto/plan.proto @@ -872,6 +872,7 @@ message DeleteCtx { repeated string partition_table_names = 7; int32 partition_idx = 8; int32 primary_key_idx = 9; + TruncateTable truncate_table = 10; } message Query { From 988f3240d1629f5174611dad9dd779a24e6c2e72 Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Thu, 7 Dec 2023 14:16:13 +0800 Subject: [PATCH 21/22] [mod] remove useless codes --- pkg/sql/colexec/deletion/types.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/sql/colexec/deletion/types.go b/pkg/sql/colexec/deletion/types.go index 5fbd348fcf3b..3286154b5831 100644 --- a/pkg/sql/colexec/deletion/types.go +++ b/pkg/sql/colexec/deletion/types.go @@ -103,8 +103,6 @@ type DeleteCtx struct { PartitionTableNames []string // Align array index with the partition number PartitionIndexInBatch int // The array index position of the partition expression column PartitionSources []engine.Relation // Align array index with the partition number - IndexTableNames []string - ForeignTbl []uint64 Source engine.Relation Ref *plan.ObjectRef AddAffectedRows bool From 50b6a6b07c9ae560d8985ad5bd8fa20b91c57423 Mon Sep 17 00:00:00 2001 From: noorall <863485501@qq.com> Date: Thu, 7 Dec 2023 14:56:31 +0800 Subject: [PATCH 22/22] [mod] add judgments for truncate type --- pkg/pb/plan/plan.pb.go | 1 + pkg/sql/compile/ddl.go | 13 ++++++++++++- pkg/sql/plan/build_dml_util.go | 1 + proto/plan.proto | 1 + 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pkg/pb/plan/plan.pb.go b/pkg/pb/plan/plan.pb.go index a8375d3d1e61..5b51e3b5fc02 100644 --- a/pkg/pb/plan/plan.pb.go +++ b/pkg/pb/plan/plan.pb.go @@ -9674,6 +9674,7 @@ type TruncateTable struct { ClusterTable *ClusterTable `protobuf:"bytes,5,opt,name=cluster_table,json=clusterTable,proto3" json:"cluster_table,omitempty"` TableId uint64 `protobuf:"varint,6,opt,name=table_id,json=tableId,proto3" json:"table_id,omitempty"` ForeignTbl []uint64 `protobuf:"varint,7,rep,packed,name=foreign_tbl,json=foreignTbl,proto3" json:"foreign_tbl,omitempty"` + IsDelete bool `protobuf:"varint,6,opt,name=is_delete,json=isDelete,proto3" json:"is_delete,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` diff --git a/pkg/sql/compile/ddl.go b/pkg/sql/compile/ddl.go index 711ae9c266bc..6ae450b32a2a 100644 --- a/pkg/sql/compile/ddl.go +++ b/pkg/sql/compile/ddl.go @@ -1250,6 +1250,8 @@ func (s *Scope) TruncateTable(c *Compile) error { dbName := tqry.GetDatabase() tblName := tqry.GetTable() oldId := tqry.GetTableId() + keepAutoIncrement := false + affectedRows := int64(0) dbSource, err = c.e.Database(c.ctx, dbName, c.proc.TxnOperator) if err != nil { @@ -1291,6 +1293,14 @@ func (s *Scope) TruncateTable(c *Compile) error { } } + if tqry.IsDelete { + keepAutoIncrement = true + affectedRows, err = rel.Rows(c.ctx) + if err != nil { + return err + } + } + if isTemp { // memoryengine truncate always return 0, so for temporary table, just use origin tableId as newId _, err = dbSource.Truncate(c.ctx, engine.GetTempTableName(dbName, tblName)) @@ -1374,7 +1384,7 @@ func (s *Scope) TruncateTable(c *Compile) error { c.ctx, oldId, newId, - false, + keepAutoIncrement, c.proc.TxnOperator) if err != nil { return err @@ -1386,6 +1396,7 @@ func (s *Scope) TruncateTable(c *Compile) error { if err != nil { return err } + c.addAffectedRows(uint64(affectedRows)) return nil } diff --git a/pkg/sql/plan/build_dml_util.go b/pkg/sql/plan/build_dml_util.go index 575c4a823706..9dcdcb0d9102 100644 --- a/pkg/sql/plan/build_dml_util.go +++ b/pkg/sql/plan/build_dml_util.go @@ -1574,6 +1574,7 @@ func makeOneDeletePlan( truncateTable.ClusterTable = &plan.ClusterTable{ IsClusterTable: util.TableIsClusterTable(tableDef.GetTableType()), } + truncateTable.IsDelete = true } // append delete node deleteNode := &Node{ diff --git a/proto/plan.proto b/proto/plan.proto index 2b2679ed22d6..594033d64d5f 100644 --- a/proto/plan.proto +++ b/proto/plan.proto @@ -1254,6 +1254,7 @@ message TruncateTable { ClusterTable cluster_table = 5; uint64 table_id = 6; repeated uint64 foreign_tbl = 7; + bool is_delete = 8; } message ClusterTable{