Revert "external: LoadIngestData download all needed data and sort all (

pingcap#48871)" This reverts commit 4309c0f.
lance6716 · Dec 4, 2023 · b35f796 · b35f796
1 parent 8286fd3
commit b35f796
Show file tree

Hide file tree

Showing 10 changed files with 161 additions and 318 deletions.
diff --git a/br/pkg/lightning/backend/external/BUILD.bazel b/br/pkg/lightning/backend/external/BUILD.bazel
@@ -30,13 +30,13 @@ go_library(
         "//br/pkg/storage",
         "//pkg/kv",
         "//pkg/metrics",
+        "//pkg/util",
         "//pkg/util/hack",
         "//pkg/util/logutil",
         "//pkg/util/size",
         "@com_github_cockroachdb_pebble//:pebble",
         "@com_github_docker_go_units//:go-units",
         "@com_github_google_uuid//:uuid",
-        "@com_github_jfcg_sorty_v2//:sorty",
         "@com_github_pingcap_errors//:errors",
         "@com_github_pingcap_failpoint//:failpoint",
         "@org_golang_x_sync//errgroup",

diff --git a/br/pkg/lightning/backend/external/bench_test.go b/br/pkg/lightning/backend/external/bench_test.go
@@ -380,7 +380,7 @@ func readMergeIter(s *readTestSuite) {
 
 	readBufSize := s.memoryLimit / len(files)
 	zeroOffsets := make([]uint64, len(files))
-	iter, err := NewMergeKVIter(ctx, files, zeroOffsets, s.store, readBufSize, false, 0)
+	iter, err := NewMergeKVIter(ctx, files, zeroOffsets, s.store, readBufSize, false)
 	intest.AssertNoError(err)
 
 	kvCnt := 0

diff --git a/br/pkg/lightning/backend/external/byte_reader.go b/br/pkg/lightning/backend/external/byte_reader.go
@@ -30,6 +30,8 @@ var (
 	// ConcurrentReaderBufferSizePerConc is the buffer size for concurrent reader per
 	// concurrency.
 	ConcurrentReaderBufferSizePerConc = int(4 * size.MB)
+	// ConcurrentReaderConcurrency is the concurrency for concurrent reader.
+	ConcurrentReaderConcurrency = 8
 )
 
 // byteReader provides structured reading on a byte stream of external storage.