Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pkg/sql/catalog/lease/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ go_test(
name = "lease_test",
size = "large",
srcs = [
"count_test.go",
"helpers_test.go",
"ie_writer_test.go",
"kv_writer_test.go",
Expand Down
20 changes: 12 additions & 8 deletions pkg/sql/catalog/lease/count.go
Original file line number Diff line number Diff line change
Expand Up @@ -208,9 +208,13 @@ func countLeasesByRegion(
} else {
err = queryRegionRows(ctx)
}
if err := handleRegionLivenessErrors(ctx, prober, region, err); err != nil {
skipRegion, err := handleRegionLivenessErrors(ctx, prober, region, err)
if err != nil {
return err
}
if skipRegion {
return nil
}
if values == nil {
return errors.New("failed to count leases")
}
Expand All @@ -231,27 +235,27 @@ func getCountLeaseColumns() string {
}

// handleRegionLivenessErrors handles errors that are linked to region liveness
// timeouts.
// timeouts. Return true if the region should be skipped.
func handleRegionLivenessErrors(
ctx context.Context, prober regionliveness.Prober, region string, err error,
) error {
) (bool, error) {
if err != nil {
if regionliveness.IsQueryTimeoutErr(err) {
// Probe and mark the region potentially.
probeErr := prober.ProbeLiveness(ctx, region)
if probeErr != nil {
err = errors.WithSecondaryError(err, probeErr)
return err
return false, err
}
return errors.Wrapf(err, "count-lease timed out reading from a region")
return false, errors.Wrapf(err, "count-lease timed out reading from a region")
} else if regionliveness.IsMissingRegionEnumErr(err) {
// Skip this region because we were unable to find region in
// type descriptor. Since the database regions are cached, they
// may be stale and have dropped regions.
log.Dev.Infof(ctx, "count-lease skipping region %s due to error: %v", region, err)
return nil
return true, nil
}
return err
return false, err
}
return err
return false, err
}
26 changes: 26 additions & 0 deletions pkg/sql/catalog/lease/count_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// Copyright 2025 The Cockroach Authors.
//
// Use of this software is governed by the CockroachDB Software License
// included in the /LICENSE file.

package lease

import (
"context"
"testing"

"github.com/cockroachdb/cockroach/pkg/sql/types"
"github.com/cockroachdb/cockroach/pkg/util/leaktest"
"github.com/cockroachdb/cockroach/pkg/util/log"
"github.com/stretchr/testify/require"
)

func TestHandleRegionLivenessErrorsSkipsMissingRegion(t *testing.T) {
defer leaktest.AfterTest(t)()
defer log.Scope(t).Close(t)

ctx := context.Background()
skip, err := handleRegionLivenessErrors(ctx, nil /* prober */, "us-east2", types.EnumValueNotYetPublicError)
require.NoError(t, err)
require.True(t, skip)
}
20 changes: 17 additions & 3 deletions pkg/sql/catalog/lease/lease.go
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,11 @@ func (m *Manager) WaitForInitialVersion(
sessionIDs, err = getSessionsHoldingDescriptor(ctx, txn, schemaID, region)
}
if err != nil {
return handleRegionLivenessErrors(ctx, prober, region, err)
skipRegion, handledErr := handleRegionLivenessErrors(ctx, prober, region, err)
if skipRegion {
return nil
}
return handledErr
}
sessionsPerRegion[region] = sessionIDs
expectedSessions += len(sessionIDs)
Expand Down Expand Up @@ -469,9 +473,13 @@ func (m *Manager) WaitForInitialVersion(
} else {
regionCount, err = countDescriptorsHeldBySessionIDs(ctx, txn, descIDsForSchema, region, sessionIDs)
}
if err := handleRegionLivenessErrors(ctx, prober, region, err); err != nil {
skipRegion, err := handleRegionLivenessErrors(ctx, prober, region, err)
if err != nil {
return err
}
if skipRegion {
return nil
}
count += regionCount
return nil
})
Expand Down Expand Up @@ -620,7 +628,13 @@ func (m *Manager) WaitForNewVersion(
regionStaleSessionCount, err = countSessionsHoldingStaleDescriptor(ctx, txn, desc, region)
}
if err != nil {
return handleRegionLivenessErrors(ctx, prober, region, err)
skipRegion, handledErr := handleRegionLivenessErrors(ctx, prober, region, err)
if handledErr != nil {
return handledErr
}
if skipRegion {
continue
}
}

staleSessionCount += regionStaleSessionCount
Expand Down