ddl: make local sort generate only one subtask for each instance (#50925

) ref #48795
pingcap · Feb 5, 2024 · 613d999 · 613d999
1 parent 1621e0f
commit 613d999
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 13 deletions.
diff --git a/pkg/ddl/BUILD.bazel b/pkg/ddl/BUILD.bazel
@@ -315,7 +315,6 @@ go_test(
         "//pkg/util/sem",
         "//pkg/util/sqlexec",
         "//pkg/util/timeutil",
-        "@com_github_docker_go_units//:go-units",
         "@com_github_ngaut_pools//:pools",
         "@com_github_pingcap_errors//:errors",
         "@com_github_pingcap_failpoint//:failpoint",

diff --git a/pkg/ddl/backfilling_dist_scheduler.go b/pkg/ddl/backfilling_dist_scheduler.go
@@ -35,7 +35,6 @@ import (
 	"github.com/pingcap/tidb/pkg/kv"
 	"github.com/pingcap/tidb/pkg/meta"
 	"github.com/pingcap/tidb/pkg/parser/model"
-	"github.com/pingcap/tidb/pkg/sessionctx/variable"
 	"github.com/pingcap/tidb/pkg/store/helper"
 	"github.com/pingcap/tidb/pkg/table"
 	"github.com/pingcap/tidb/pkg/util/logutil"
@@ -320,12 +319,11 @@ func generateNonPartitionPlan(
 
 func calculateRegionBatch(totalRegionCnt int, instanceCnt int, useLocalDisk bool) int {
 	var regionBatch int
-	avgTasksPerInstance := totalRegionCnt / instanceCnt
+	avgTasksPerInstance := (totalRegionCnt + instanceCnt - 1) / instanceCnt // ceiling
 	if useLocalDisk {
-		// Make subtask large enough to reduce the overhead of local/global flush.
-		avgTasksPerDisk := int(int64(variable.DDLDiskQuota.Load()) / int64(config.SplitRegionSize))
-		regionBatch = min(avgTasksPerDisk, avgTasksPerInstance)
+		regionBatch = avgTasksPerInstance
 	} else {
+		// For cloud storage, each subtask should contain no more than 100 regions.
 		regionBatch = min(100, avgTasksPerInstance)
 	}
 	regionBatch = max(regionBatch, 1)

diff --git a/pkg/ddl/backfilling_dist_scheduler_test.go b/pkg/ddl/backfilling_dist_scheduler_test.go
@@ -20,7 +20,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/docker/go-units"
 	"github.com/ngaut/pools"
 	"github.com/pingcap/failpoint"
 	"github.com/pingcap/tidb/br/pkg/lightning/backend/external"
@@ -32,7 +31,6 @@ import (
 	"github.com/pingcap/tidb/pkg/meta"
 	"github.com/pingcap/tidb/pkg/parser/model"
 	"github.com/pingcap/tidb/pkg/parser/mysql"
-	"github.com/pingcap/tidb/pkg/sessionctx/variable"
 	"github.com/pingcap/tidb/pkg/testkit"
 	"github.com/stretchr/testify/require"
 	"github.com/tikv/client-go/v2/util"
@@ -115,21 +113,19 @@ func TestBackfillingSchedulerLocalMode(t *testing.T) {
 func TestCalculateRegionBatch(t *testing.T) {
 	// Test calculate in cloud storage.
 	batchCnt := ddl.CalculateRegionBatchForTest(100, 8, false)
-	require.Equal(t, 12, batchCnt)
+	require.Equal(t, 13, batchCnt)
 	batchCnt = ddl.CalculateRegionBatchForTest(2, 8, false)
 	require.Equal(t, 1, batchCnt)
 	batchCnt = ddl.CalculateRegionBatchForTest(8, 8, false)
 	require.Equal(t, 1, batchCnt)
 
 	// Test calculate in local storage.
-	variable.DDLDiskQuota.Store(96 * units.MiB * 1000)
 	batchCnt = ddl.CalculateRegionBatchForTest(100, 8, true)
-	require.Equal(t, 12, batchCnt)
+	require.Equal(t, 13, batchCnt)
 	batchCnt = ddl.CalculateRegionBatchForTest(2, 8, true)
 	require.Equal(t, 1, batchCnt)
-	variable.DDLDiskQuota.Store(96 * units.MiB * 2)
 	batchCnt = ddl.CalculateRegionBatchForTest(24, 8, true)
-	require.Equal(t, 2, batchCnt)
+	require.Equal(t, 3, batchCnt)
 }
 
 func TestBackfillingSchedulerGlobalSortMode(t *testing.T) {