Merge branch 'master' into fix-test20

Makefile

@@ -227,7 +227,8 @@ failpoint-disable: install-tools
 
 ut: pd-ut
 	@$(FAILPOINT_ENABLE)
-	./bin/pd-ut run --race
+	# only run unit tests
+	./bin/pd-ut run --ignore tests --race
 	@$(CLEAN_UT_BINARY)
 	@$(FAILPOINT_DISABLE)
 

pkg/core/region.go

@@ -929,7 +929,7 @@ func NewRegionsInfo() *RegionsInfo {
 		learners:     make(map[uint64]*regionTree),
 		witnesses:    make(map[uint64]*regionTree),
 		pendingPeers: make(map[uint64]*regionTree),
-		overlapTree:  newRegionTree(),
+		overlapTree:  newRegionTreeWithCountRef(),
 	}
 }
 
@@ -1131,42 +1131,31 @@ func (r *RegionsInfo) updateSubTreeLocked(rangeChanged bool, overlaps []*RegionI
 	item := &regionItem{region}
 	r.subRegions[region.GetID()] = item
 	r.overlapTree.update(item, false)
-	setPeer := func(peersMap map[uint64]*regionTree, storeID uint64, item *regionItem, countRef bool) {
+	// Add leaders and followers.
+	setPeer := func(peersMap map[uint64]*regionTree, storeID uint64) {
 		store, ok := peersMap[storeID]
 		if !ok {
-			if !countRef {
-				store = newRegionTree()
-			} else {
-				store = newRegionTreeWithCountRef()
-			}
+			store = newRegionTree()
 			peersMap[storeID] = store
 		}
 		store.update(item, false)
 	}
-
-	// Add to leaders and followers.
 	for _, peer := range region.GetVoters() {
 		storeID := peer.GetStoreId()
 		if peer.GetId() == region.leader.GetId() {
-			// Add leader peer to leaders.
-			setPeer(r.leaders, storeID, item, true)
+			setPeer(r.leaders, storeID)
 		} else {
-			// Add follower peer to followers.
-			setPeer(r.followers, storeID, item, false)
+			setPeer(r.followers, storeID)
 		}
 	}
-
+	// Add other peers.
 	setPeers := func(peersMap map[uint64]*regionTree, peers []*metapb.Peer) {
 		for _, peer := range peers {
-			storeID := peer.GetStoreId()
-			setPeer(peersMap, storeID, item, false)
+			setPeer(peersMap, peer.GetStoreId())
 		}
 	}
-	// Add to learners.
 	setPeers(r.learners, region.GetLearners())
-	// Add to witnesses.
 	setPeers(r.witnesses, region.GetWitnesses())
-	// Add to PendingPeers
 	setPeers(r.pendingPeers, region.GetPendingPeers())
 }
 
@@ -1335,7 +1324,7 @@ func (r *RegionsInfo) RemoveRegion(region *RegionInfo) {
 // ResetRegionCache resets the regions info.
 func (r *RegionsInfo) ResetRegionCache() {
 	r.t.Lock()
-	r.tree = newRegionTree()
+	r.tree = newRegionTreeWithCountRef()
 	r.regions = make(map[uint64]*regionItem)
 	r.t.Unlock()
 	r.st.Lock()
@@ -1345,7 +1334,7 @@ func (r *RegionsInfo) ResetRegionCache() {
 	r.learners = make(map[uint64]*regionTree)
 	r.witnesses = make(map[uint64]*regionTree)
 	r.pendingPeers = make(map[uint64]*regionTree)
-	r.overlapTree = newRegionTree()
+	r.overlapTree = newRegionTreeWithCountRef()
 }
 
 // RemoveRegionFromSubTree removes RegionInfo from regionSubTrees

pkg/core/region_test.go

@@ -1081,3 +1081,18 @@ func TestCheckAndPutSubTree(t *testing.T) {
 	// should failed to put because the root tree is missing
 	re.Equal(0, regions.tree.length())
 }
+
+func TestCntRefAfterResetRegionCache(t *testing.T) {
+	re := require.New(t)
+	regions := NewRegionsInfo()
+	// Put the region first.
+	region := NewTestRegionInfo(1, 1, []byte("a"), []byte("b"))
+	regions.CheckAndPutRegion(region)
+	re.Equal(int32(2), region.GetRef())
+	regions.ResetRegionCache()
+	// Put the region after reset.
+	region = NewTestRegionInfo(1, 1, []byte("a"), []byte("b"))
+	re.Zero(region.GetRef())
+	regions.CheckAndPutRegion(region)
+	re.Equal(int32(2), region.GetRef())
+}

scripts/ci-subtask.sh

@@ -9,27 +9,28 @@ integrations_dir=$(pwd)/tests/integrations
 case $1 in
     1)
         # unit tests ignore `tests`
-        ./bin/pd-ut run --race --ignore tests --coverprofile $ROOT_PATH_COV  || exit 1
+        ./bin/pd-ut run --race --ignore tests --coverprofile $ROOT_PATH_COV || exit 1
         ;;
     2)
         # unit tests only in `tests`
-        ./bin/pd-ut run tests --race --coverprofile $ROOT_PATH_COV  || exit 1
+        ./bin/pd-ut run tests --race --coverprofile $ROOT_PATH_COV || exit 1
         ;;
     3)
         # tools tests
         cd ./tools && make ci-test-job && cat covprofile >> $ROOT_PATH_COV || exit 1
         ;;
     4)
         # integration test client
-        cd ./client && make ci-test-job && cat covprofile >> $ROOT_PATH_COV  || exit 1
-        cd $integrations_dir && make ci-test-job test_name=client && cat ./client/covprofile >> $ROOT_PATH_COV || exit 1
+        ./bin/pd-ut it run client --race --coverprofile $ROOT_PATH_COV || exit 1
+        # client tests
+        cd ./client && make ci-test-job && cat covprofile >> $ROOT_PATH_COV || exit 1
         ;;
     5)
         # integration test tso
-        cd $integrations_dir && make ci-test-job test_name=tso && cat ./tso/covprofile >> $ROOT_PATH_COV || exit 1
+        ./bin/pd-ut it run tso --race --coverprofile $ROOT_PATH_COV || exit 1
         ;;
     6)
         # integration test mcs
-        cd $integrations_dir && make ci-test-job test_name=mcs && cat ./mcs/covprofile >> $ROOT_PATH_COV || exit 1
+        ./bin/pd-ut it run mcs --race --coverprofile $ROOT_PATH_COV || exit 1
         ;;
 esac

tests/integrations/mcs/keyspace/tso_keyspace_group_test.go

@@ -47,7 +47,6 @@ type keyspaceGroupTestSuite struct {
 	cluster          *tests.TestCluster
 	server           *tests.TestServer
 	backendEndpoints string
-	dialClient       *http.Client
 }
 
 func TestKeyspaceGroupTestSuite(t *testing.T) {
@@ -67,11 +66,6 @@ func (suite *keyspaceGroupTestSuite) SetupTest() {
 	suite.server = cluster.GetLeaderServer()
 	re.NoError(suite.server.BootstrapCluster())
 	suite.backendEndpoints = suite.server.GetAddr()
-	suite.dialClient = &http.Client{
-		Transport: &http.Transport{
-			DisableKeepAlives: true,
-		},
-	}
 	suite.cleanupFunc = func() {
 		cancel()
 	}
@@ -81,7 +75,6 @@ func (suite *keyspaceGroupTestSuite) TearDownTest() {
 	re := suite.Require()
 	suite.cleanupFunc()
 	suite.cluster.Destroy()
-	suite.dialClient.CloseIdleConnections()
 	re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes"))
 }
 
@@ -347,7 +340,7 @@ func (suite *keyspaceGroupTestSuite) tryAllocNodesForKeyspaceGroup(re *require.A
 	re.NoError(err)
 	httpReq, err := http.NewRequest(http.MethodPost, suite.server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d/alloc", id), bytes.NewBuffer(data))
 	re.NoError(err)
-	resp, err := suite.dialClient.Do(httpReq)
+	resp, err := tests.TestDialClient.Do(httpReq)
 	re.NoError(err)
 	defer resp.Body.Close()
 	nodes := make([]endpoint.KeyspaceGroupMember, 0)
@@ -364,7 +357,7 @@ func (suite *keyspaceGroupTestSuite) tryCreateKeyspaceGroup(re *require.Assertio
 	re.NoError(err)
 	httpReq, err := http.NewRequest(http.MethodPost, suite.server.GetAddr()+keyspaceGroupsPrefix, bytes.NewBuffer(data))
 	re.NoError(err)
-	resp, err := suite.dialClient.Do(httpReq)
+	resp, err := tests.TestDialClient.Do(httpReq)
 	re.NoError(err)
 	defer resp.Body.Close()
 	return resp.StatusCode
@@ -373,7 +366,7 @@ func (suite *keyspaceGroupTestSuite) tryCreateKeyspaceGroup(re *require.Assertio
 func (suite *keyspaceGroupTestSuite) tryGetKeyspaceGroup(re *require.Assertions, id uint32) (*endpoint.KeyspaceGroup, int) {
 	httpReq, err := http.NewRequest(http.MethodGet, suite.server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d", id), http.NoBody)
 	re.NoError(err)
-	resp, err := suite.dialClient.Do(httpReq)
+	resp, err := tests.TestDialClient.Do(httpReq)
 	re.NoError(err)
 	defer resp.Body.Close()
 	kg := &endpoint.KeyspaceGroup{}
@@ -390,7 +383,7 @@ func (suite *keyspaceGroupTestSuite) trySetNodesForKeyspaceGroup(re *require.Ass
 	re.NoError(err)
 	httpReq, err := http.NewRequest(http.MethodPatch, suite.server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d", id), bytes.NewBuffer(data))
 	re.NoError(err)
-	resp, err := suite.dialClient.Do(httpReq)
+	resp, err := tests.TestDialClient.Do(httpReq)
 	re.NoError(err)
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {

tests/integrations/mcs/members/member_test.go

@@ -34,7 +34,7 @@ type memberTestSuite struct {
 	cluster          *tests.TestCluster
 	server           *tests.TestServer
 	backendEndpoints string
-	dialClient       pdClient.Client
+	pdClient         pdClient.Client
 }
 
 func TestMemberTestSuite(t *testing.T) {
@@ -53,7 +53,7 @@ func (suite *memberTestSuite) SetupTest() {
 	suite.server = cluster.GetLeaderServer()
 	re.NoError(suite.server.BootstrapCluster())
 	suite.backendEndpoints = suite.server.GetAddr()
-	suite.dialClient = pdClient.NewClient("mcs-member-test", []string{suite.server.GetAddr()})
+	suite.pdClient = pdClient.NewClient("mcs-member-test", []string{suite.server.GetAddr()})
 
 	// TSO
 	nodes := make(map[string]bs.Server)
@@ -86,30 +86,30 @@ func (suite *memberTestSuite) TearDownTest() {
 	for _, cleanup := range suite.cleanupFunc {
 		cleanup()
 	}
-	if suite.dialClient != nil {
-		suite.dialClient.Close()
+	if suite.pdClient != nil {
+		suite.pdClient.Close()
 	}
 	suite.cluster.Destroy()
 }
 
 func (suite *memberTestSuite) TestMembers() {
 	re := suite.Require()
-	members, err := suite.dialClient.GetMicroServiceMembers(suite.ctx, "tso")
+	members, err := suite.pdClient.GetMicroServiceMembers(suite.ctx, "tso")
 	re.NoError(err)
 	re.Len(members, utils.DefaultKeyspaceGroupReplicaCount)
 
-	members, err = suite.dialClient.GetMicroServiceMembers(suite.ctx, "scheduling")
+	members, err = suite.pdClient.GetMicroServiceMembers(suite.ctx, "scheduling")
 	re.NoError(err)
 	re.Len(members, 3)
 }
 
 func (suite *memberTestSuite) TestPrimary() {
 	re := suite.Require()
-	primary, err := suite.dialClient.GetMicroServicePrimary(suite.ctx, "tso")
+	primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, "tso")
 	re.NoError(err)
 	re.NotEmpty(primary)
 
-	primary, err = suite.dialClient.GetMicroServicePrimary(suite.ctx, "scheduling")
+	primary, err = suite.pdClient.GetMicroServicePrimary(suite.ctx, "scheduling")
 	re.NoError(err)
 	re.NotEmpty(primary)
 }

tests/integrations/mcs/resourcemanager/resource_manager_test.go

@@ -957,7 +957,7 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() {
 		}
 		createJSON, err := json.Marshal(group)
 		re.NoError(err)
-		resp, err := http.Post(getAddr(i)+"/resource-manager/api/v1/config/group", "application/json", strings.NewReader(string(createJSON)))
+		resp, err := tests.TestDialClient.Post(getAddr(i)+"/resource-manager/api/v1/config/group", "application/json", strings.NewReader(string(createJSON)))
 		re.NoError(err)
 		resp.Body.Close()
 		re.Equal(http.StatusOK, resp.StatusCode)
@@ -982,7 +982,7 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() {
 		}
 
 		// Get Resource Group
-		resp, err = http.Get(getAddr(i) + "/resource-manager/api/v1/config/group/" + tcase.name)
+		resp, err = tests.TestDialClient.Get(getAddr(i) + "/resource-manager/api/v1/config/group/" + tcase.name)
 		re.NoError(err)
 		re.Equal(http.StatusOK, resp.StatusCode)
 		respString, err := io.ReadAll(resp.Body)
@@ -995,7 +995,7 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() {
 
 		// Last one, Check list and delete all resource groups
 		if i == len(testCasesSet1)-1 {
-			resp, err := http.Get(getAddr(i) + "/resource-manager/api/v1/config/groups")
+			resp, err := tests.TestDialClient.Get(getAddr(i) + "/resource-manager/api/v1/config/groups")
 			re.NoError(err)
 			re.Equal(http.StatusOK, resp.StatusCode)
 			respString, err := io.ReadAll(resp.Body)
@@ -1023,7 +1023,7 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() {
 			}
 
 			// verify again
-			resp1, err := http.Get(getAddr(i) + "/resource-manager/api/v1/config/groups")
+			resp1, err := tests.TestDialClient.Get(getAddr(i) + "/resource-manager/api/v1/config/groups")
 			re.NoError(err)
 			re.Equal(http.StatusOK, resp1.StatusCode)
 			respString1, err := io.ReadAll(resp1.Body)

tests/integrations/mcs/resourcemanager/server_test.go

@@ -63,7 +63,7 @@ func TestResourceManagerServer(t *testing.T) {
 	// Test registered REST HTTP Handler
 	url := addr + "/resource-manager/api/v1/config"
 	{
-		resp, err := http.Get(url + "/groups")
+		resp, err := tests.TestDialClient.Get(url + "/groups")
 		re.NoError(err)
 		defer resp.Body.Close()
 		re.Equal(http.StatusOK, resp.StatusCode)
@@ -78,13 +78,13 @@ func TestResourceManagerServer(t *testing.T) {
 		}
 		createJSON, err := json.Marshal(group)
 		re.NoError(err)
-		resp, err := http.Post(url+"/group", "application/json", strings.NewReader(string(createJSON)))
+		resp, err := tests.TestDialClient.Post(url+"/group", "application/json", strings.NewReader(string(createJSON)))
 		re.NoError(err)
 		defer resp.Body.Close()
 		re.Equal(http.StatusOK, resp.StatusCode)
 	}
 	{
-		resp, err := http.Get(url + "/group/pingcap")
+		resp, err := tests.TestDialClient.Get(url + "/group/pingcap")
 		re.NoError(err)
 		defer resp.Body.Close()
 		re.Equal(http.StatusOK, resp.StatusCode)
@@ -95,7 +95,7 @@ func TestResourceManagerServer(t *testing.T) {
 
 	// Test metrics handler
 	{
-		resp, err := http.Get(addr + "/metrics")
+		resp, err := tests.TestDialClient.Get(addr + "/metrics")
 		re.NoError(err)
 		defer resp.Body.Close()
 		re.Equal(http.StatusOK, resp.StatusCode)
@@ -106,7 +106,7 @@ func TestResourceManagerServer(t *testing.T) {
 
 	// Test status handler
 	{
-		resp, err := http.Get(addr + "/status")
+		resp, err := tests.TestDialClient.Get(addr + "/status")
 		re.NoError(err)
 		defer resp.Body.Close()
 		re.Equal(http.StatusOK, resp.StatusCode)