From a35245a4112989d8a4b504e5e4b17a02415b5e77 Mon Sep 17 00:00:00 2001 From: "Yoshiaki Ueda (bootjp)" Date: Thu, 23 Apr 2026 03:36:54 +0900 Subject: [PATCH 1/6] chore(raft): drop hashicorp/raft backend and flip kv/ to raftengine natively - delete internal/raftengine/hashicorp and the etcd/hashicorp FSM adapter - delete internal/raftstore (hashicorp-only Pebble log store) - delete cmd/etcd-raft-migrate and cmd/etcd-raft-rollback (hashicorp migration tools) - drop hashicorp engine branch from the runtime factory and data-dir detection kv/ now implements raftengine.StateMachine directly: kvFSM.Apply takes []byte, Snapshot returns raftengine.Snapshot, and Restore takes io.Reader. Legacy NewCoordinator/NewTransaction/NewLeaderProxy shims that took *raft.Raft are removed in favor of the engine-based constructors. Replace hashicorp-typed identifiers across the tree: raft.ServerAddress and raft.ServerID become plain strings, raft.Server / raft.Voter become raftengine.Server with Suffrage "voter". adapter/ production code, main.go, shard_config.go and the demo server follow suit. Test scaffolding in kv/ and adapter/ that built in-process hashicorp clusters is rewritten on top of the etcd factory with t.TempDir() data dirs. hashicorp-specific tests that exercised AddVoter flow, InmemTransport or the legacy LogStore (adapter/add_voter_join_test.go, adapter/dynamodb_failure_test.go, adapter/redis_list_raft_benchmark_test.go, multiraft_runtime_test.go bootstrap/mixed-artifact cases) are removed; equivalent behaviour is covered by the etcd engine and the bootstrap e2e harness. go.mod is tidied to drop hashicorp/raft, Jille/raft-grpc-transport, hashicorp/go-hclog and transitive hashicorp deps. --- adapter/add_voter_join_test.go | 326 ------------ adapter/distribution_milestone1_e2e_test.go | 59 +-- adapter/distribution_server_test.go | 5 +- adapter/dynamodb.go | 7 +- adapter/dynamodb_failure_test.go | 139 ------ adapter/dynamodb_test.go | 5 +- adapter/internal.go | 6 - adapter/redis.go | 17 +- adapter/redis_info_test.go | 13 +- adapter/redis_keys_pattern_test.go | 5 +- adapter/redis_list_raft_benchmark_test.go | 268 ---------- adapter/redis_retry_test.go | 5 +- adapter/s3.go | 15 +- adapter/s3_test.go | 81 ++- adapter/test_util.go | 215 +++----- cmd/etcd-raft-migrate/main.go | 29 -- cmd/etcd-raft-rollback/main.go | 29 -- cmd/server/demo.go | 162 +++--- go.mod | 13 - go.sum | 333 ------------- .../raftengine/etcd/hashicorp_fsm_adapter.go | 76 --- internal/raftengine/etcd/migrate_test.go | 55 +- internal/raftengine/hashicorp/engine.go | 468 ------------------ internal/raftengine/hashicorp/engine_test.go | 67 --- internal/raftengine/hashicorp/factory.go | 206 -------- .../hashicorp/leadership_err_test.go | 55 -- internal/raftengine/hashicorp/migrate.go | 242 --------- internal/raftengine/hashicorp/migrate_test.go | 198 -------- internal/raftstore/pebble.go | 210 -------- internal/raftstore/pebble_test.go | 47 -- kv/coordinator.go | 14 +- kv/coordinator_dispatch_test.go | 17 +- kv/coordinator_leader_test.go | 2 +- kv/fsm.go | 20 +- kv/fsm_occ_test.go | 7 +- kv/fsm_txn_test.go | 3 +- kv/grpc_conn_cache.go | 15 +- kv/grpc_conn_cache_test.go | 5 +- kv/leader_proxy.go | 7 - kv/leader_proxy_test.go | 95 ++-- kv/leader_routed_store.go | 3 +- kv/leader_routed_store_test.go | 12 +- kv/lease_state_test.go | 22 - kv/lock_resolver_test.go | 9 +- kv/raft_engine.go | 5 +- kv/shard_router_test.go | 118 +---- kv/shard_store_txn_lock_test.go | 13 +- kv/sharded_coordinator.go | 5 +- kv/sharded_coordinator_abort_test.go | 3 +- kv/sharded_coordinator_del_prefix_test.go | 7 +- kv/sharded_coordinator_leader_test.go | 3 +- kv/sharded_integration_test.go | 80 +-- kv/snapshot.go | 30 +- kv/snapshot_test.go | 24 +- kv/transaction.go | 6 - kv/transaction_batch_test.go | 73 +-- main.go | 57 +-- main_bootstrap_e2e_test.go | 9 +- main_bootstrap_test.go | 12 +- main_s3.go | 3 +- multiraft_runtime.go | 45 +- multiraft_runtime_test.go | 61 +-- shard_config.go | 33 +- shard_config_test.go | 16 +- 64 files changed, 487 insertions(+), 3703 deletions(-) delete mode 100644 adapter/add_voter_join_test.go delete mode 100644 adapter/dynamodb_failure_test.go delete mode 100644 adapter/redis_list_raft_benchmark_test.go delete mode 100644 cmd/etcd-raft-migrate/main.go delete mode 100644 cmd/etcd-raft-rollback/main.go delete mode 100644 internal/raftengine/etcd/hashicorp_fsm_adapter.go delete mode 100644 internal/raftengine/hashicorp/engine.go delete mode 100644 internal/raftengine/hashicorp/engine_test.go delete mode 100644 internal/raftengine/hashicorp/factory.go delete mode 100644 internal/raftengine/hashicorp/leadership_err_test.go delete mode 100644 internal/raftengine/hashicorp/migrate.go delete mode 100644 internal/raftengine/hashicorp/migrate_test.go delete mode 100644 internal/raftstore/pebble.go delete mode 100644 internal/raftstore/pebble_test.go diff --git a/adapter/add_voter_join_test.go b/adapter/add_voter_join_test.go deleted file mode 100644 index 21ad4d936..000000000 --- a/adapter/add_voter_join_test.go +++ /dev/null @@ -1,326 +0,0 @@ -package adapter - -import ( - "bytes" - "context" - "errors" - "net" - "strconv" - "sync" - "testing" - "time" - - internalraftadmin "github.com/bootjp/elastickv/internal/raftadmin" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" - "github.com/bootjp/elastickv/kv" - pb "github.com/bootjp/elastickv/proto" - "github.com/bootjp/elastickv/store" - "github.com/hashicorp/raft" - "github.com/stretchr/testify/require" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -func TestAddVoterJoinPath_RegistersMemberAndServesAdapterTraffic(t *testing.T) { - t.Parallel() - - const ( - waitTimeout = 12 * time.Second - waitInterval = 100 * time.Millisecond - rpcTimeout = 2 * time.Second - ) - - ctx := context.Background() - nodes, servers := setupAddVoterJoinPathNodes(t, ctx) - t.Cleanup(func() { - shutdown(nodes) - servers.AwaitNoError(t, waitTimeout) - }) - - waitForNodeListeners(t, ctx, nodes, waitTimeout, waitInterval) - require.Eventually(t, func() bool { - return nodes[0].raft.State() == raft.Leader - }, waitTimeout, waitInterval) - - adminConn, err := grpc.NewClient(nodes[0].grpcAddress, grpc.WithTransportCredentials(insecure.NewCredentials())) - require.NoError(t, err) - t.Cleanup(func() { _ = adminConn.Close() }) - admin := pb.NewRaftAdminClient(adminConn) - - addVotersAndAwait(t, ctx, rpcTimeout, admin, nodes, []int{1, 2}) - - expectedCfg := expectedVoterConfig(nodes) - waitForConfigReplication(t, expectedCfg, nodes, waitTimeout, waitInterval) - waitForRaftReadiness(t, nodes, waitTimeout, waitInterval) - - followerConn, err := grpc.NewClient(nodes[1].grpcAddress, - grpc.WithTransportCredentials(insecure.NewCredentials()), - ) - require.NoError(t, err) - t.Cleanup(func() { _ = followerConn.Close() }) - followerRaw := pb.NewRawKVClient(followerConn) - - leaderConn, err := grpc.NewClient(nodes[0].grpcAddress, - grpc.WithTransportCredentials(insecure.NewCredentials()), - ) - require.NoError(t, err) - t.Cleanup(func() { _ = leaderConn.Close() }) - leaderRaw := pb.NewRawKVClient(leaderConn) - - putAndWaitForRead(t, ctx, rpcTimeout, followerRaw, leaderRaw, []byte("addvoter-key"), []byte("ok"), waitTimeout, waitInterval) - - // Simulate a partition-like failure by isolating node2's raft transport. - require.NoError(t, nodes[2].tm.Close()) - nodes[2].tm = nil - - putAndWaitForRead(t, ctx, rpcTimeout, followerRaw, leaderRaw, []byte("partition-survive-key"), []byte("ok2"), waitTimeout, waitInterval) - - // Force leader change while one node is isolated, then confirm write/read path. - require.NoError(t, nodes[0].raft.LeadershipTransferToServer(raft.ServerID("1"), raft.ServerAddress(nodes[1].raftAddress)).Error()) - require.Eventually(t, func() bool { - return nodes[1].raft.State() == raft.Leader - }, waitTimeout, waitInterval) - - putAndWaitForRead(t, ctx, rpcTimeout, leaderRaw, followerRaw, []byte("leader-transfer-key"), []byte("ok3"), waitTimeout, waitInterval) -} - -func setupAddVoterJoinPathNodes(t *testing.T, ctx context.Context) ([]Node, *serverWorkers) { - t.Helper() - - ports, lis := reserveAddVoterJoinListeners(t, ctx, 3) - - // AddVoter address must point to the node's shared gRPC endpoint where - // raft transport and adapter services are served. - require.Equal(t, ports[1].raftAddress, ports[1].grpcAddress) - require.Equal(t, ports[2].raftAddress, ports[2].grpcAddress) - - leaderRedisMap := map[raft.ServerAddress]string{ - raft.ServerAddress(ports[0].raftAddress): ports[0].redisAddress, - raft.ServerAddress(ports[1].raftAddress): ports[1].redisAddress, - raft.ServerAddress(ports[2].raftAddress): ports[2].redisAddress, - } - bootstrapCfg := raft.Configuration{ - Servers: []raft.Server{ - { - Suffrage: raft.Voter, - ID: raft.ServerID("0"), - Address: raft.ServerAddress(ports[0].raftAddress), - }, - }, - } - - workers := newServerWorkers(len(ports) * 3) - nodes := make([]Node, 0, len(ports)) - for i := range ports { - nodes = append(nodes, startAddVoterJoinNode(t, ctx, workers, i, ports[i], lis[i], bootstrapCfg, leaderRedisMap)) - } - return nodes, workers -} - -func reserveAddVoterJoinListeners(t *testing.T, ctx context.Context, n int) ([]portsAdress, []listeners) { - t.Helper() - - var lc net.ListenConfig - ports := assignPorts(n) - lis := make([]listeners, 0, n) - for i := range ports { - for { - bound, ls, retry, err := bindListeners(ctx, &lc, ports[i]) - require.NoError(t, err) - if !retry { - ports[i] = bound - lis = append(lis, ls) - break - } - ports[i] = assignPorts(1)[0] - } - } - return ports, lis -} - -func startAddVoterJoinNode( - t *testing.T, - ctx context.Context, - workers *serverWorkers, - idx int, - port portsAdress, - lis listeners, - bootstrapCfg raft.Configuration, - leaderRedisMap map[raft.ServerAddress]string, -) Node { - t.Helper() - - st := store.NewMVCCStore() - hlc := kv.NewHLC() - fsm := kv.NewKvFSMWithHLC(st, hlc) - - electionTimeout := leaderElectionTimeout - if idx != 0 { - electionTimeout = followerElectionTimeout - } - - r, tm, err := newRaft(strconv.Itoa(idx), port.raftAddress, fsm, idx == 0, bootstrapCfg, electionTimeout) - require.NoError(t, err) - - s := grpc.NewServer() - trx := kv.NewTransaction(r) - coordinator := kv.NewCoordinator(trx, r, kv.WithHLC(hlc)) - relay := NewRedisPubSubRelay() - routedStore := kv.NewLeaderRoutedStore(st, coordinator) - gs := NewGRPCServer(routedStore, coordinator, WithCloseStore()) - opsCtx, opsCancel := context.WithCancel(ctx) - go coordinator.RunHLCLeaseRenewal(opsCtx) - tm.Register(s) - pb.RegisterRawKVServer(s, gs) - pb.RegisterTransactionalKVServer(s, gs) - pb.RegisterInternalServer(s, NewInternal(trx, r, coordinator.Clock(), relay)) - internalraftadmin.RegisterOperationalServices(opsCtx, s, hashicorpraftengine.New(r), []string{"RawKV"}) - - workers.Go(func() error { - err := s.Serve(lis.grpc) - if errors.Is(err, grpc.ErrServerStopped) || errors.Is(err, net.ErrClosed) { - return nil - } - return err - }) - - rd := NewRedisServer(lis.redis, port.redisAddress, routedStore, coordinator, leaderRedisMap, relay) - workers.Go(func() error { - err := rd.Run() - if errors.Is(err, net.ErrClosed) { - return nil - } - return err - }) - - ds := NewDynamoDBServer(lis.dynamo, routedStore, coordinator) - workers.Go(func() error { - err := ds.Run() - if errors.Is(err, net.ErrClosed) { - return nil - } - return err - }) - - return newNode( - port.grpcAddress, - port.raftAddress, - port.redisAddress, - port.dynamoAddress, - r, - tm, - s, - gs, - rd, - ds, - opsCancel, - ) -} - -func addVotersAndAwait( - t *testing.T, - ctx context.Context, - rpcTimeout time.Duration, - admin pb.RaftAdminClient, - nodes []Node, - targets []int, -) { - t.Helper() - - for _, target := range targets { - addCtx, cancelAdd := context.WithTimeout(ctx, rpcTimeout) - future, err := admin.AddVoter(addCtx, &pb.RaftAdminAddVoterRequest{ - Id: strconv.Itoa(target), - Address: nodes[target].grpcAddress, - PreviousIndex: 0, - }) - cancelAdd() - require.NoError(t, err) - require.Greater(t, future.GetIndex(), uint64(0)) - } -} - -func expectedVoterConfig(nodes []Node) raft.Configuration { - servers := make([]raft.Server, 0, len(nodes)) - for i, n := range nodes { - servers = append(servers, raft.Server{ - Suffrage: raft.Voter, - ID: raft.ServerID(strconv.Itoa(i)), - Address: raft.ServerAddress(n.raftAddress), - }) - } - return raft.Configuration{Servers: servers} -} - -func putAndWaitForRead( - t *testing.T, - ctx context.Context, - rpcTimeout time.Duration, - writer pb.RawKVClient, - reader pb.RawKVClient, - key []byte, - value []byte, - waitTimeout time.Duration, - waitInterval time.Duration, -) { - t.Helper() - - putCtx, cancelPut := context.WithTimeout(ctx, rpcTimeout) - _, err := writer.RawPut(putCtx, &pb.RawPutRequest{Key: key, Value: value}) - cancelPut() - require.NoError(t, err) - - require.Eventually(t, func() bool { - getCtx, cancelGet := context.WithTimeout(ctx, rpcTimeout) - resp, getErr := reader.RawGet(getCtx, &pb.RawGetRequest{Key: key}) - cancelGet() - if getErr != nil { - return false - } - return resp.Exists && bytes.Equal(resp.Value, value) - }, waitTimeout, waitInterval) -} - -type serverWorkers struct { - wg sync.WaitGroup - errCh chan error -} - -func newServerWorkers(buffer int) *serverWorkers { - return &serverWorkers{errCh: make(chan error, buffer)} -} - -func (w *serverWorkers) Go(run func() error) { - if w == nil || run == nil { - return - } - w.wg.Go(func() { - if err := run(); err != nil { - w.errCh <- err - } - }) -} - -func (w *serverWorkers) AwaitNoError(t *testing.T, timeout time.Duration) { - t.Helper() - if w == nil { - return - } - - done := make(chan struct{}) - go func() { - w.wg.Wait() - close(done) - }() - - select { - case <-done: - case <-time.After(timeout): - require.FailNow(t, "server goroutines did not finish in time") - } - - close(w.errCh) - for err := range w.errCh { - require.NoError(t, err) - } -} diff --git a/adapter/distribution_milestone1_e2e_test.go b/adapter/distribution_milestone1_e2e_test.go index 7bc00ce6e..e6af8b151 100644 --- a/adapter/distribution_milestone1_e2e_test.go +++ b/adapter/distribution_milestone1_e2e_test.go @@ -7,12 +7,12 @@ import ( "time" "github.com/bootjp/elastickv/distribution" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" + "github.com/bootjp/elastickv/internal/raftengine" + etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" cerrs "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" "github.com/stretchr/testify/require" ) @@ -25,15 +25,13 @@ func TestMilestone1SplitRange_EndToEndRefreshAndDataPath(t *testing.T) { hlc := kv.NewHLC() group1Store := store.NewMVCCStore() - group1Raft, stopGroup1 := newSingleRaftForDistributionE2E(t, "group-1", kv.NewKvFSMWithHLC(group1Store, hlc)) + group1Engine, stopGroup1 := newSingleRaftForDistributionE2E(t, "group-1", kv.NewKvFSMWithHLC(group1Store, hlc)) defer stopGroup1() group2Store := store.NewMVCCStore() - group2Raft, stopGroup2 := newSingleRaftForDistributionE2E(t, "group-2", kv.NewKvFSMWithHLC(group2Store, hlc)) + group2Engine, stopGroup2 := newSingleRaftForDistributionE2E(t, "group-2", kv.NewKvFSMWithHLC(group2Store, hlc)) defer stopGroup2() - group1Engine := hashicorpraftengine.New(group1Raft) - group2Engine := hashicorpraftengine.New(group2Raft) groups := map[uint64]*kv.ShardGroup{ 1: {Engine: group1Engine, Store: group1Store, Txn: kv.NewLeaderProxyWithEngine(group1Engine)}, 2: {Engine: group2Engine, Store: group2Store, Txn: kv.NewLeaderProxyWithEngine(group2Engine)}, @@ -229,39 +227,34 @@ func requireValueAt(t *testing.T, st store.MVCCStore, key []byte, expected []byt require.Equal(t, expected, actual) } -func newSingleRaftForDistributionE2E(t *testing.T, id string, fsm raft.FSM) (*raft.Raft, func()) { +func newSingleRaftForDistributionE2E(t *testing.T, id string, fsm raftengine.StateMachine) (raftengine.Engine, func()) { t.Helper() - addr, trans := raft.NewInmemTransport(raft.ServerAddress(id)) - cfg := raft.DefaultConfig() - cfg.LocalID = raft.ServerID(id) - cfg.HeartbeatTimeout = 50 * time.Millisecond - cfg.ElectionTimeout = 100 * time.Millisecond - cfg.LeaderLeaseTimeout = 50 * time.Millisecond - - logStore := raft.NewInmemStore() - stableStore := raft.NewInmemStore() - snapshotStore := raft.NewInmemSnapshotStore() - r, err := raft.NewRaft(cfg, fsm, logStore, stableStore, snapshotStore, trans) + factory := etcdraftengine.NewFactory(etcdraftengine.FactoryConfig{ + TickInterval: 10 * time.Millisecond, + HeartbeatTick: 1, + ElectionTick: 10, + MaxSizePerMsg: 1 << 20, + MaxInflightMsg: 256, + }) + result, err := factory.Create(raftengine.FactoryConfig{ + LocalID: id, + LocalAddress: id, + DataDir: t.TempDir(), + Bootstrap: true, + StateMachine: fsm, + }) require.NoError(t, err) - bootstrapErr := r.BootstrapCluster(raft.Configuration{ - Servers: []raft.Server{ - { - Suffrage: raft.Voter, - ID: raft.ServerID(id), - Address: addr, - }, - }, - }).Error() - require.NoError(t, bootstrapErr) - require.Eventually(t, func() bool { - return r.State() == raft.Leader - }, time.Second, 10*time.Millisecond) + return result.Engine.State() == raftengine.StateLeader + }, 5*time.Second, 10*time.Millisecond) stop := func() { - require.NoError(t, r.Shutdown().Error()) + require.NoError(t, result.Engine.Close()) + if result.Close != nil { + require.NoError(t, result.Close()) + } } - return r, stop + return result.Engine, stop } diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 945d81719..96b3cd8be 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -9,7 +9,6 @@ import ( "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" - "github.com/hashicorp/raft" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -721,7 +720,7 @@ func (s *distributionCoordinatorStub) VerifyLeader() error { return nil } -func (s *distributionCoordinatorStub) RaftLeader() raft.ServerAddress { +func (s *distributionCoordinatorStub) RaftLeader() string { return "" } @@ -736,7 +735,7 @@ func (s *distributionCoordinatorStub) VerifyLeaderForKey(_ []byte) error { return nil } -func (s *distributionCoordinatorStub) RaftLeaderForKey(_ []byte) raft.ServerAddress { +func (s *distributionCoordinatorStub) RaftLeaderForKey(_ []byte) string { return "" } diff --git a/adapter/dynamodb.go b/adapter/dynamodb.go index 59a973591..0de6aa393 100644 --- a/adapter/dynamodb.go +++ b/adapter/dynamodb.go @@ -26,7 +26,6 @@ import ( "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" json "github.com/goccy/go-json" - "github.com/hashicorp/raft" ) const ( @@ -135,7 +134,7 @@ type DynamoDBServer struct { requestObserver monitoring.DynamoDBRequestObserver itemUpdateLocks [itemUpdateLockStripeCount]sync.Mutex tableLocks [tableLockStripeCount]sync.Mutex - leaderDynamo map[raft.ServerAddress]string + leaderDynamo map[string]string } // WithDynamoDBRequestObserver enables Prometheus-compatible request metrics. @@ -154,9 +153,9 @@ func WithDynamoDBActiveTimestampTracker(tracker *kv.ActiveTimestampTracker) Dyna // WithDynamoDBLeaderMap configures the Raft-address-to-DynamoDB-address mapping // used to forward requests from followers to the current leader. // The format mirrors the raftRedisMap / raftS3Map convention. -func WithDynamoDBLeaderMap(m map[raft.ServerAddress]string) DynamoDBServerOption { +func WithDynamoDBLeaderMap(m map[string]string) DynamoDBServerOption { return func(server *DynamoDBServer) { - server.leaderDynamo = make(map[raft.ServerAddress]string, len(m)) + server.leaderDynamo = make(map[string]string, len(m)) for k, v := range m { server.leaderDynamo[k] = v } diff --git a/adapter/dynamodb_failure_test.go b/adapter/dynamodb_failure_test.go deleted file mode 100644 index 7e5d5f9c9..000000000 --- a/adapter/dynamodb_failure_test.go +++ /dev/null @@ -1,139 +0,0 @@ -package adapter - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/dynamodb" - ddbTypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - pb "github.com/bootjp/elastickv/proto" - "github.com/hashicorp/raft" - "github.com/stretchr/testify/require" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -func TestDynamoDB_QueryAndScan_SurviveFollowerIsolationAndLeaderTransfer(t *testing.T) { - t.Parallel() - - const ( - waitTimeout = 12 * time.Second - waitInterval = 100 * time.Millisecond - tableName = "events" - ) - - ctx := context.Background() - nodes, servers := setupAddVoterJoinPathNodes(t, ctx) - t.Cleanup(func() { - shutdown(nodes) - servers.AwaitNoError(t, waitTimeout) - }) - - waitForNodeListeners(t, ctx, nodes, waitTimeout, waitInterval) - require.Eventually(t, func() bool { - return nodes[0].raft.State() == raft.Leader - }, waitTimeout, waitInterval) - - adminConn, err := grpc.NewClient(nodes[0].grpcAddress, grpc.WithTransportCredentials(insecure.NewCredentials())) - require.NoError(t, err) - t.Cleanup(func() { _ = adminConn.Close() }) - admin := pb.NewRaftAdminClient(adminConn) - - addVotersAndAwait(t, ctx, 2*time.Second, admin, nodes, []int{1, 2}) - waitForConfigReplication(t, expectedVoterConfig(nodes), nodes, waitTimeout, waitInterval) - waitForRaftReadiness(t, nodes, waitTimeout, waitInterval) - - client0 := newDynamoTestClient(t, nodes[0].dynamoAddress) - client1 := newDynamoTestClient(t, nodes[1].dynamoAddress) - - createCompositeKeyTable(t, ctx, client0, tableName) - putCompositeItems(t, ctx, client0, tableName, 2) - - require.Eventually(t, func() bool { - return queryPartitionCount(ctx, client1, tableName, "tenant") == 2 && - scanTableCount(ctx, client1, tableName) == 2 - }, waitTimeout, waitInterval) - - require.NoError(t, nodes[2].tm.Close()) - nodes[2].tm = nil - - putCompositeItems(t, ctx, client0, tableName, 3) - require.Eventually(t, func() bool { - return queryPartitionCount(ctx, client1, tableName, "tenant") == 3 && - scanTableCount(ctx, client1, tableName) == 3 - }, waitTimeout, waitInterval) - - require.NoError(t, nodes[0].raft.LeadershipTransferToServer( - raft.ServerID("1"), - raft.ServerAddress(nodes[1].raftAddress), - ).Error()) - require.Eventually(t, func() bool { - return nodes[1].raft.State() == raft.Leader - }, waitTimeout, waitInterval) - - putCompositeItems(t, ctx, client1, tableName, 4) - require.Eventually(t, func() bool { - return queryPartitionCount(ctx, client0, tableName, "tenant") == 4 && - scanTableCount(ctx, client0, tableName) == 4 - }, waitTimeout, waitInterval) -} - -func createCompositeKeyTable(t *testing.T, ctx context.Context, client *dynamodb.Client, tableName string) { - t.Helper() - - _, err := client.CreateTable(ctx, &dynamodb.CreateTableInput{ - TableName: aws.String(tableName), - AttributeDefinitions: []ddbTypes.AttributeDefinition{ - {AttributeName: aws.String("pk"), AttributeType: ddbTypes.ScalarAttributeTypeS}, - {AttributeName: aws.String("sk"), AttributeType: ddbTypes.ScalarAttributeTypeS}, - }, - KeySchema: []ddbTypes.KeySchemaElement{ - {AttributeName: aws.String("pk"), KeyType: ddbTypes.KeyTypeHash}, - {AttributeName: aws.String("sk"), KeyType: ddbTypes.KeyTypeRange}, - }, - }) - require.NoError(t, err) -} - -func putCompositeItems(t *testing.T, ctx context.Context, client *dynamodb.Client, tableName string, total int) { - t.Helper() - - for i := range total { - _, err := client.PutItem(ctx, &dynamodb.PutItemInput{ - TableName: aws.String(tableName), - Item: map[string]ddbTypes.AttributeValue{ - "pk": &ddbTypes.AttributeValueMemberS{Value: "tenant"}, - "sk": &ddbTypes.AttributeValueMemberS{Value: fmt.Sprintf("item-%02d", i)}, - "payload": &ddbTypes.AttributeValueMemberS{Value: fmt.Sprintf("value-%02d", i)}, - }, - }) - require.NoError(t, err) - } -} - -func queryPartitionCount(ctx context.Context, client *dynamodb.Client, tableName string, pk string) int { - out, err := client.Query(ctx, &dynamodb.QueryInput{ - TableName: aws.String(tableName), - KeyConditionExpression: aws.String("pk = :pk"), - ExpressionAttributeValues: map[string]ddbTypes.AttributeValue{ - ":pk": &ddbTypes.AttributeValueMemberS{Value: pk}, - }, - }) - if err != nil { - return -1 - } - return int(out.Count) -} - -func scanTableCount(ctx context.Context, client *dynamodb.Client, tableName string) int { - out, err := client.Scan(ctx, &dynamodb.ScanInput{ - TableName: aws.String(tableName), - }) - if err != nil { - return -1 - } - return int(out.Count) -} diff --git a/adapter/dynamodb_test.go b/adapter/dynamodb_test.go index de9e6b97d..1bfaaf57d 100644 --- a/adapter/dynamodb_test.go +++ b/adapter/dynamodb_test.go @@ -19,7 +19,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" - "github.com/hashicorp/raft" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -1836,7 +1835,7 @@ func (w *testCoordinatorWrapper) VerifyLeader() error { return w.inner.VerifyLeader() } -func (w *testCoordinatorWrapper) RaftLeader() raft.ServerAddress { +func (w *testCoordinatorWrapper) RaftLeader() string { return w.inner.RaftLeader() } @@ -1848,7 +1847,7 @@ func (w *testCoordinatorWrapper) VerifyLeaderForKey(key []byte) error { return w.inner.VerifyLeaderForKey(key) } -func (w *testCoordinatorWrapper) RaftLeaderForKey(key []byte) raft.ServerAddress { +func (w *testCoordinatorWrapper) RaftLeaderForKey(key []byte) string { return w.inner.RaftLeaderForKey(key) } diff --git a/adapter/internal.go b/adapter/internal.go index 7038de7b5..1e36854d7 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -5,17 +5,11 @@ import ( "context" "github.com/bootjp/elastickv/internal/raftengine" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" ) -func NewInternal(txm kv.Transactional, r *raft.Raft, clock *kv.HLC, relay *RedisPubSubRelay) *Internal { - return NewInternalWithEngine(txm, hashicorpraftengine.New(r), clock, relay) -} - func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, clock *kv.HLC, relay *RedisPubSubRelay) *Internal { return &Internal{ leader: leader, diff --git a/adapter/redis.go b/adapter/redis.go index 6058faffb..8b2b0fc7d 100644 --- a/adapter/redis.go +++ b/adapter/redis.go @@ -23,7 +23,6 @@ import ( pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" "github.com/redis/go-redis/v9" "github.com/tidwall/redcon" ) @@ -283,7 +282,7 @@ type RedisServer struct { baseCtx context.Context baseCancel context.CancelFunc // TODO manage membership from raft log - leaderRedis map[raft.ServerAddress]string + leaderRedis map[string]string // leaderClients caches go-redis clients per leader address to avoid // creating a new connection pool for every proxied request. @@ -392,7 +391,7 @@ type redisResult struct { err error } -func NewRedisServer(listen net.Listener, redisAddr string, store store.MVCCStore, coordinate kv.Coordinator, leaderRedis map[raft.ServerAddress]string, relay *RedisPubSubRelay, opts ...RedisServerOption) *RedisServer { +func NewRedisServer(listen net.Listener, redisAddr string, store store.MVCCStore, coordinate kv.Coordinator, leaderRedis map[string]string, relay *RedisPubSubRelay, opts ...RedisServerOption) *RedisServer { if relay == nil { relay = NewRedisPubSubRelay() } @@ -902,28 +901,28 @@ func (r *RedisServer) publishLocal(channel, message []byte) int64 { return int64(r.pubsub.Publish(string(channel), string(message))) } -func (r *RedisServer) relayPeers() []raft.ServerAddress { +func (r *RedisServer) relayPeers() []string { if len(r.leaderRedis) == 0 { return nil } - byRedis := make(map[string]raft.ServerAddress, len(r.leaderRedis)) + byRedis := make(map[string]string, len(r.leaderRedis)) for addr, redisAddr := range r.leaderRedis { if redisAddr == "" || redisAddr == r.redisAddr { continue } prev, ok := byRedis[redisAddr] - if !ok || string(addr) < string(prev) { + if !ok || addr < prev { byRedis[redisAddr] = addr } } - peers := make([]raft.ServerAddress, 0, len(byRedis)) + peers := make([]string, 0, len(byRedis)) for _, addr := range byRedis { peers = append(peers, addr) } sort.Slice(peers, func(i, j int) bool { - return string(peers[i]) < string(peers[j]) + return peers[i] < peers[j] }) return peers } @@ -944,7 +943,7 @@ func (r *RedisServer) publishCluster(ctx context.Context, channel, message []byt defer overallCancel() for _, peer := range peers { - go func(peer raft.ServerAddress) { + go func(peer string) { //nolint:dupl conn, err := r.relayConnCache.ConnFor(peer) if err != nil { log.Printf("redis relay publish dial peer=%s err=%v", peer, err) diff --git a/adapter/redis_info_test.go b/adapter/redis_info_test.go index f815f6ae6..c1381e090 100644 --- a/adapter/redis_info_test.go +++ b/adapter/redis_info_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/bootjp/elastickv/kv" - "github.com/hashicorp/raft" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/tidwall/redcon" @@ -14,7 +13,7 @@ import ( type infoTestCoordinator struct { isLeader bool - raftLeader raft.ServerAddress + raftLeader string clock *kv.HLC } @@ -23,10 +22,10 @@ func (c *infoTestCoordinator) Dispatch(context.Context, *kv.OperationGroup[kv.OP } func (c *infoTestCoordinator) IsLeader() bool { return c.isLeader } func (c *infoTestCoordinator) VerifyLeader() error { return nil } -func (c *infoTestCoordinator) RaftLeader() raft.ServerAddress { return c.raftLeader } +func (c *infoTestCoordinator) RaftLeader() string { return c.raftLeader } func (c *infoTestCoordinator) IsLeaderForKey([]byte) bool { return c.isLeader } func (c *infoTestCoordinator) VerifyLeaderForKey([]byte) error { return nil } -func (c *infoTestCoordinator) RaftLeaderForKey([]byte) raft.ServerAddress { return c.raftLeader } +func (c *infoTestCoordinator) RaftLeaderForKey([]byte) string { return c.raftLeader } func (c *infoTestCoordinator) Clock() *kv.HLC { if c.clock == nil { c.clock = kv.NewHLC() @@ -45,7 +44,7 @@ func (c *infoTestCoordinator) LeaseReadForKey(ctx context.Context, _ []byte) (ui func TestRedisServer_Info_LeaderRole(t *testing.T) { r := &RedisServer{ redisAddr: "10.0.0.1:6379", - leaderRedis: map[raft.ServerAddress]string{"raft-1": "10.0.0.1:6379"}, + leaderRedis: map[string]string{"raft-1": "10.0.0.1:6379"}, coordinator: &infoTestCoordinator{isLeader: true, raftLeader: "raft-1"}, } @@ -63,7 +62,7 @@ func TestRedisServer_Info_LeaderRole(t *testing.T) { func TestRedisServer_Info_FollowerRole(t *testing.T) { r := &RedisServer{ redisAddr: "10.0.0.2:6379", - leaderRedis: map[raft.ServerAddress]string{ + leaderRedis: map[string]string{ "raft-1": "10.0.0.1:6379", "raft-2": "10.0.0.2:6379", }, @@ -88,7 +87,7 @@ func TestRedisServer_Info_FollowerRole(t *testing.T) { func TestRedisServer_Info_UnknownLeader(t *testing.T) { r := &RedisServer{ redisAddr: "10.0.0.3:6379", - leaderRedis: map[raft.ServerAddress]string{}, + leaderRedis: map[string]string{}, coordinator: &infoTestCoordinator{isLeader: false, raftLeader: ""}, } diff --git a/adapter/redis_keys_pattern_test.go b/adapter/redis_keys_pattern_test.go index 3d4b7a8ab..75010edfc 100644 --- a/adapter/redis_keys_pattern_test.go +++ b/adapter/redis_keys_pattern_test.go @@ -7,7 +7,6 @@ import ( "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" - "github.com/hashicorp/raft" "github.com/stretchr/testify/require" ) @@ -35,7 +34,7 @@ func (s *stubAdapterCoordinator) VerifyLeader() error { return s.verifyLeaderErr } -func (s *stubAdapterCoordinator) RaftLeader() raft.ServerAddress { +func (s *stubAdapterCoordinator) RaftLeader() string { return "" } @@ -50,7 +49,7 @@ func (s *stubAdapterCoordinator) VerifyLeaderForKey([]byte) error { return nil } -func (s *stubAdapterCoordinator) RaftLeaderForKey([]byte) raft.ServerAddress { +func (s *stubAdapterCoordinator) RaftLeaderForKey([]byte) string { return "" } diff --git a/adapter/redis_list_raft_benchmark_test.go b/adapter/redis_list_raft_benchmark_test.go deleted file mode 100644 index c7a2a0583..000000000 --- a/adapter/redis_list_raft_benchmark_test.go +++ /dev/null @@ -1,268 +0,0 @@ -package adapter - -// BullMQ-style workload benchmark on a real 3-node in-process Raft cluster. -// -// Why this complements the in-memory benchmark: -// The in-memory benchmark (redis_list_pop_benchmark_test.go) uses an -// occAdapterCoordinator backed by a plain MVCCStore with a simulated sleep -// for OCC retries. In a real Raft cluster each failed commit (write-write -// conflict) wastes one full Raft round-trip (~1 ms loopback). This -// benchmark captures that real RTT cost. -// -// Legacy vs Claim on Raft: -// Legacy: every RPUSH and LPOP writes the same metadata key. -// Raft serializes all writers; OCC detects conflicts in ApplyMutations. -// Each conflict wastes exactly one committed (but rejected) Raft log entry. -// Claim: RPUSH writes a unique delta key (commitTS-keyed); -// LPOP writes a claim key (seq-keyed) + delta key. -// RPUSH-LPOP conflicts are eliminated entirely. -// Concurrent LPOPs still race for the same claim seq; concurrent RPUSHes -// still race for the same item seq — but at lower probability than Legacy -// where ALL operation types conflict with ALL others. -// -// Store: NewPebbleStore(b.TempDir()) — disk-backed Pebble in a temp directory. -// Background LSM compaction runs and removes tombstones, so the delta scan -// stays O(live deltas) ≤ compactor threshold throughout the benchmark. -// Disk I/O overhead is present but Raft round-trip (~0.2 ms loopback) dominates. -// -// Observed on Apple M1 Max (3-node loopback cluster, GOMAXPROCS=10, benchtime=30s): -// -// Benchmark ns/op total ops system ops/s -// ────────────────────────────────────────────────────────────────────── -// Legacy/Parallel1 (10 goroutines) 181 µs 472 713 15 757 -// Legacy/Parallel4 (40 goroutines) 229 µs 392 815 13 094 -17 % -// Claim /Parallel1 (10 goroutines) 3.6 ms 10 000 333 -// Claim /Parallel4 (40 goroutines) 2.5 ms 12 194 406 +22 % -// -// Key insight — read TOTAL THROUGHPUT (ops/s = total_iterations/30s), not ns/op: -// Legacy: 4× goroutines → total throughput DROPS -17 %. -// RPUSH and LPOP both write the same meta key; OCC conflicts grow -// super-linearly and saturate the Raft pipeline with rejected commits. -// Claim: 4× goroutines → total throughput RISES +22 %. -// RPUSH-LPOP conflicts are eliminated; Raft commit batching improves -// with more concurrent proposals, so throughput scales with concurrency. -// -// Absolute latency note: -// Claim's ns/op is ~15× higher than Legacy because resolveListMeta -// calls ScanAt (Pebble range iterator) vs GetAt (point lookup) for Legacy. -// Even with ≤8 live deltas the iterator open/seek/close overhead adds ~2 ms. -// In production this is paid against a 10–100 ms Raft RTT where it is -// negligible; the throughput-scaling difference is what matters under load. -// -// Run with: -// go test ./adapter/ -run='^$' -bench='BenchmarkBullMQ_Raft' -benchtime=30s -benchmem -timeout=600s - -import ( - "context" - "fmt" - "net" - "strconv" - "testing" - "time" - - internalutil "github.com/bootjp/elastickv/internal" - "github.com/bootjp/elastickv/kv" - pb "github.com/bootjp/elastickv/proto" - "github.com/bootjp/elastickv/store" - "github.com/hashicorp/raft" - "google.golang.org/grpc" -) - -func createBenchNode(b *testing.B, ctx context.Context, isLeader bool, port portsAdress, cfg raft.Configuration, leaderRedisMap map[raft.ServerAddress]string, lis listeners) Node { - b.Helper() - st, err := store.NewPebbleStore(b.TempDir()) - if err != nil { - b.Fatal(err) - } - hlc := kv.NewHLC() - fsm := kv.NewKvFSMWithHLC(st, hlc) - - electionTimeout := leaderElectionTimeout - if !isLeader { - electionTimeout = followerElectionTimeout - } - - id := strconv.Itoa(port.raft) - r, tm, err := newRaft(id, port.raftAddress, fsm, isLeader, cfg, electionTimeout) - if err != nil { - b.Fatal(err) - } - - s := grpc.NewServer(internalutil.GRPCServerOptions()...) - trx := kv.NewTransaction(r) - coordinator := kv.NewCoordinator(trx, r, kv.WithHLC(hlc)) - relay := NewRedisPubSubRelay() - routedStore := kv.NewLeaderRoutedStore(st, coordinator) - gs := NewGRPCServer(routedStore, coordinator, WithCloseStore()) - _, opsCancel := context.WithCancel(ctx) - - tm.Register(s) - pb.RegisterRawKVServer(s, gs) - pb.RegisterTransactionalKVServer(s, gs) - - go func(srv *grpc.Server, l net.Listener) { - _ = srv.Serve(l) - }(s, lis.grpc) - - rd := NewRedisServer(lis.redis, port.redisAddress, routedStore, coordinator, leaderRedisMap, relay) - if err := lis.dynamo.Close(); err != nil { - b.Logf("failed to close unused dynamo listener: %v", err) - } - - return newNode(port.grpcAddress, port.raftAddress, port.redisAddress, port.dynamoAddress, r, tm, s, gs, rd, nil, opsCancel) -} - -// createBenchCluster spins up an n-node in-process Raft cluster suitable for -// benchmarks. Unlike createNode (which requires *testing.T), this function -// uses b.Fatal for error reporting. -// -// The returned cleanup function stops all nodes and closes listeners. -func createBenchCluster(b *testing.B, n int) ([]Node, func()) { - b.Helper() - ctx := context.Background() - ports := assignPorts(n) - - lc := net.ListenConfig{} - lis := make([]listeners, n) - for i := range n { - for { - bound, l, retry, err := bindListeners(ctx, &lc, ports[i]) - if err != nil { - b.Fatal(err) - } - if retry { - ports[i] = portAssigner() - continue - } - ports[i] = bound - lis[i] = l - break - } - } - - // Build Raft config after port binding so retried ports are reflected. - cfg := buildRaftConfig(n, ports) - leaderRedisMap := make(map[raft.ServerAddress]string, n) - for _, p := range ports { - leaderRedisMap[raft.ServerAddress(p.raftAddress)] = p.redisAddress - } - - var nodes []Node - for i := range n { - node := createBenchNode(b, ctx, i == 0, ports[i], cfg, leaderRedisMap, lis[i]) - nodes = append(nodes, node) - } - - // Wait for node[0] to win the leader election. - deadline := time.Now().Add(10 * time.Second) - for time.Now().Before(deadline) { - if nodes[0].raft.State() == raft.Leader { - break - } - time.Sleep(50 * time.Millisecond) - } - if nodes[0].raft.State() != raft.Leader { - b.Fatal("node 0 did not become leader within 10s") - } - - return nodes, func() { shutdown(nodes) } -} - -// BenchmarkBullMQ_Raft_Legacy_RPushLPOP measures the main-branch RPUSH+LPOP -// pattern on a 3-node in-process Raft cluster. -// -// Both RPUSH and LPOP write the base metadata key, so Raft serializes them -// and the FSM's ApplyMutations detects write-write conflicts whenever two -// operations overlap. Each conflict costs one wasted Raft round-trip. -func BenchmarkBullMQ_Raft_Legacy_RPushLPOP(b *testing.B) { - for _, par := range []int{1, 4} { - b.Run(fmt.Sprintf("Parallel%d", par), func(b *testing.B) { - nodes, cleanup := createBenchCluster(b, 3) - defer cleanup() - - leader := nodes[0].redisServer - key := []byte("raft-bm-queue-legacy") - ctx := context.Background() - - // Seed queue so consumers always find data. - if err := leader.listRPushLegacy(ctx, key, makeItems(256)); err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.ReportAllocs() - b.SetParallelism(par) - b.RunParallel(func(pb *testing.PB) { - item := [][]byte{[]byte("job")} - for pb.Next() { - if err := retryUntilSuccess(ctx, func() error { - return leader.listRPushLegacy(ctx, key, item) - }); err != nil { - b.Error(err) - return - } - if err := retryUntilSuccess(ctx, func() error { - return leader.listPopLegacyRMW(ctx, key) - }); err != nil { - b.Error(err) - return - } - } - }) - }) - } -} - -// BenchmarkBullMQ_Raft_Claim_RPushLPOP measures the current-branch RPUSH+LPOP -// pattern on a 3-node in-process Raft cluster. -// -// RPUSH emits a unique delta key (commitTS-keyed); LPOP emits a claim key -// (seq-keyed, unique per item) + delta key. RPUSH and LPOP write completely -// different keys so they never conflict. Each operation uses exactly one -// Raft round-trip regardless of concurrency. -func BenchmarkBullMQ_Raft_Claim_RPushLPOP(b *testing.B) { - for _, par := range []int{1, 4} { - b.Run(fmt.Sprintf("Parallel%d", par), func(b *testing.B) { - nodes, cleanup := createBenchCluster(b, 3) - defer cleanup() - - leader := nodes[0].redisServer - key := []byte("raft-bm-queue-claim") - ctx := context.Background() - - // DeltaCompactor folds accumulated delta keys in the background. - // It uses the leader's store and coordinator, so compaction results - // go through Raft and are replicated to all nodes. - cancel := startCompactor(leader) - defer cancel() - - // Seed queue so consumers always find data. - if _, err := leader.listRPush(ctx, key, makeItems(256)); err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.ReportAllocs() - b.SetParallelism(par) - b.RunParallel(func(pb *testing.PB) { - item := [][]byte{[]byte("job")} - for pb.Next() { - if err := retryUntilSuccess(ctx, func() error { - _, err := leader.listRPush(ctx, key, item) - return err - }); err != nil { - b.Error(err) - return - } - if err := retryUntilSuccess(ctx, func() error { - _, err := leader.listPopClaim(ctx, key, 1, true) - return err - }); err != nil { - b.Error(err) - return - } - } - }) - }) - } -} diff --git a/adapter/redis_retry_test.go b/adapter/redis_retry_test.go index 1720418c3..d17742b90 100644 --- a/adapter/redis_retry_test.go +++ b/adapter/redis_retry_test.go @@ -10,7 +10,6 @@ import ( "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" "github.com/stretchr/testify/require" "github.com/tidwall/redcon" ) @@ -61,7 +60,7 @@ func (c *retryOnceCoordinator) VerifyLeader() error { return nil } -func (c *retryOnceCoordinator) RaftLeader() raft.ServerAddress { +func (c *retryOnceCoordinator) RaftLeader() string { return "" } @@ -73,7 +72,7 @@ func (c *retryOnceCoordinator) VerifyLeaderForKey([]byte) error { return nil } -func (c *retryOnceCoordinator) RaftLeaderForKey([]byte) raft.ServerAddress { +func (c *retryOnceCoordinator) RaftLeaderForKey([]byte) string { return "" } diff --git a/adapter/s3.go b/adapter/s3.go index ea629ff66..8c888cce2 100644 --- a/adapter/s3.go +++ b/adapter/s3.go @@ -28,7 +28,6 @@ import ( "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" json "github.com/goccy/go-json" - "github.com/hashicorp/raft" ) const ( @@ -73,7 +72,7 @@ type S3Server struct { region string store store.MVCCStore coordinator kv.Coordinator - leaderS3 map[raft.ServerAddress]string + leaderS3 map[string]string staticCreds map[string]string readTracker *kv.ActiveTimestampTracker httpServer *http.Server @@ -282,7 +281,7 @@ type s3ListPartEntry struct { Size int64 `xml:"Size"` } -func NewS3Server(listen net.Listener, s3Addr string, st store.MVCCStore, coordinate kv.Coordinator, leaderS3 map[raft.ServerAddress]string, opts ...S3ServerOption) *S3Server { +func NewS3Server(listen net.Listener, s3Addr string, st store.MVCCStore, coordinate kv.Coordinator, leaderS3 map[string]string, opts ...S3ServerOption) *S3Server { s := &S3Server{ listen: listen, s3Addr: s3Addr, @@ -299,7 +298,7 @@ func NewS3Server(listen net.Listener, s3Addr string, st store.MVCCStore, coordin } if s.s3Addr != "" { if s.leaderS3 == nil { - s.leaderS3 = map[raft.ServerAddress]string{} + s.leaderS3 = map[string]string{} } if leader := s.coordinatorLeaderAddress(); leader != "" { if _, exists := s.leaderS3[leader]; !exists { @@ -2241,7 +2240,7 @@ func (s *S3Server) maybeProxyToLeader(w http.ResponseWriter, r *http.Request) bo if err != nil { return false } - var leader raft.ServerAddress + var leader string if len(key) > 0 { if s.coordinator.IsLeaderForKey(key) && s.coordinator.VerifyLeaderForKey(key) == nil { return false @@ -2303,7 +2302,7 @@ func (s *S3Server) clock() *kv.HLC { return s.coordinator.Clock() } -func (s *S3Server) coordinatorLeaderAddress() raft.ServerAddress { +func (s *S3Server) coordinatorLeaderAddress() string { if s == nil || s.coordinator == nil { return "" } @@ -2964,11 +2963,11 @@ func validateS3PutPreconditions(r *http.Request, previous *s3ObjectManifest) err return nil } -func cloneLeaderAddrMap(src map[raft.ServerAddress]string) map[raft.ServerAddress]string { +func cloneLeaderAddrMap(src map[string]string) map[string]string { if len(src) == 0 { return nil } - out := make(map[raft.ServerAddress]string, len(src)) + out := make(map[string]string, len(src)) for key, value := range src { out[key] = value } diff --git a/adapter/s3_test.go b/adapter/s3_test.go index cb816fa0b..f3f27d2be 100644 --- a/adapter/s3_test.go +++ b/adapter/s3_test.go @@ -23,12 +23,12 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" "github.com/bootjp/elastickv/distribution" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" + "github.com/bootjp/elastickv/internal/raftengine" + etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" "github.com/bootjp/elastickv/internal/s3keys" "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" json "github.com/goccy/go-json" - "github.com/hashicorp/raft" "github.com/stretchr/testify/require" ) @@ -103,8 +103,8 @@ func TestS3Server_ProxiesFollowerRequests(t *testing.T) { defer upstream.Close() targetHost := strings.TrimPrefix(upstream.URL, "http://") - server := NewS3Server(nil, "", store.NewMVCCStore(), &followerS3Coordinator{}, map[raft.ServerAddress]string{ - raft.ServerAddress("leader"): targetHost, + server := NewS3Server(nil, "", store.NewMVCCStore(), &followerS3Coordinator{}, map[string]string{ + string("leader"): targetHost, }) rec := httptest.NewRecorder() @@ -500,8 +500,8 @@ func TestS3Server_ProxiesFollowerRequestsBeforeAuth(t *testing.T) { "", store.NewMVCCStore(), &followerS3Coordinator{}, - map[raft.ServerAddress]string{ - raft.ServerAddress("leader"): targetHost, + map[string]string{ + string("leader"): targetHost, }, WithS3StaticCredentials(map[string]string{testS3AccessKey: testS3SecretKey}), ) @@ -540,15 +540,15 @@ func TestS3Server_ProxiesObjectRequestsUsingObjectRouteLeader(t *testing.T) { localForKey: func(key []byte) bool { return !bytes.HasPrefix(key, []byte(s3keys.RoutePrefix)) }, - leaderForKey: func(key []byte) raft.ServerAddress { + leaderForKey: func(key []byte) string { if bytes.HasPrefix(key, []byte(s3keys.RoutePrefix)) { - return raft.ServerAddress("object-leader") + return string("object-leader") } return "" }, } - server := NewS3Server(nil, "", st, coord, map[raft.ServerAddress]string{ - raft.ServerAddress("object-leader"): strings.TrimPrefix(upstream.URL, "http://"), + server := NewS3Server(nil, "", st, coord, map[string]string{ + string("object-leader"): strings.TrimPrefix(upstream.URL, "http://"), }) rec := httptest.NewRecorder() @@ -644,8 +644,8 @@ func TestS3Server_ShardedStoreRoutesBucketAndObjectData(t *testing.T) { raft2, stop2 := newSingleRaftForS3Test(t, "g2", kv.NewKvFSMWithHLC(store2, hlc)) defer stop2() - engine1 := hashicorpraftengine.New(raft1) - engine2 := hashicorpraftengine.New(raft2) + engine1 := raft1 + engine2 := raft2 groups := map[uint64]*kv.ShardGroup{ 1: {Engine: engine1, Store: store1, Txn: kv.NewLeaderProxyWithEngine(engine1)}, 2: {Engine: engine2, Store: store2, Txn: kv.NewLeaderProxyWithEngine(engine2)}, @@ -712,14 +712,14 @@ func (c *followerS3Coordinator) LeaseReadForKey(ctx context.Context, _ []byte) ( return c.LinearizableRead(ctx) } -func (c *followerS3Coordinator) RaftLeader() raft.ServerAddress { - return raft.ServerAddress("leader") +func (c *followerS3Coordinator) RaftLeader() string { + return string("leader") } type routeAwareS3Coordinator struct { stubAdapterCoordinator localForKey func([]byte) bool - leaderForKey func([]byte) raft.ServerAddress + leaderForKey func([]byte) string } func (c *routeAwareS3Coordinator) IsLeaderForKey(key []byte) bool { @@ -736,7 +736,7 @@ func (c *routeAwareS3Coordinator) VerifyLeaderForKey(key []byte) error { return kv.ErrLeaderNotFound } -func (c *routeAwareS3Coordinator) RaftLeaderForKey(key []byte) raft.ServerAddress { +func (c *routeAwareS3Coordinator) RaftLeaderForKey(key []byte) string { if c.leaderForKey == nil { return "" } @@ -797,37 +797,34 @@ func decodeListBucketResult(t *testing.T, body []byte) s3ListBucketResult { return out } -func newSingleRaftForS3Test(t *testing.T, id string, fsm raft.FSM) (*raft.Raft, func()) { +func newSingleRaftForS3Test(t *testing.T, id string, fsm raftengine.StateMachine) (raftengine.Engine, func()) { t.Helper() - addr, trans := raft.NewInmemTransport(raft.ServerAddress(id)) - cfg := raft.DefaultConfig() - cfg.LocalID = raft.ServerID(id) - cfg.HeartbeatTimeout = 50 * time.Millisecond - cfg.ElectionTimeout = 100 * time.Millisecond - cfg.LeaderLeaseTimeout = 50 * time.Millisecond - - ldb := raft.NewInmemStore() - sdb := raft.NewInmemStore() - fss := raft.NewInmemSnapshotStore() - r, err := raft.NewRaft(cfg, fsm, ldb, sdb, fss, trans) + factory := etcdraftengine.NewFactory(etcdraftengine.FactoryConfig{ + TickInterval: 10 * time.Millisecond, + HeartbeatTick: 1, + ElectionTick: 10, + MaxSizePerMsg: 1 << 20, + MaxInflightMsg: 256, + }) + result, err := factory.Create(raftengine.FactoryConfig{ + LocalID: id, + LocalAddress: id, + DataDir: t.TempDir(), + Bootstrap: true, + StateMachine: fsm, + }) require.NoError(t, err) - require.NoError(t, r.BootstrapCluster(raft.Configuration{ - Servers: []raft.Server{{ - Suffrage: raft.Voter, - ID: raft.ServerID(id), - Address: addr, - }}, - }).Error()) - - for range 100 { - if r.State() == raft.Leader { - break + + require.Eventually(t, func() bool { + return result.Engine.State() == raftengine.StateLeader + }, 5*time.Second, 10*time.Millisecond) + return result.Engine, func() { + _ = result.Engine.Close() + if result.Close != nil { + _ = result.Close() } - time.Sleep(10 * time.Millisecond) } - require.Equal(t, raft.Leader, r.State()) - return r, func() { _ = r.Shutdown().Error() } } func md5Hex(v string) string { diff --git a/adapter/test_util.go b/adapter/test_util.go index 6573d31b7..b8d60d712 100644 --- a/adapter/test_util.go +++ b/adapter/test_util.go @@ -10,22 +10,38 @@ import ( "testing" "time" - transport "github.com/Jille/raft-grpc-transport" internalutil "github.com/bootjp/elastickv/internal" internalraftadmin "github.com/bootjp/elastickv/internal/raftadmin" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" + "github.com/bootjp/elastickv/internal/raftengine" + etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" - "github.com/hashicorp/go-hclog" - "github.com/hashicorp/raft" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" "google.golang.org/grpc" ) +const ( + testEngineTickInterval = 10 * time.Millisecond + testEngineHeartbeatTick = 1 + testEngineElectionTick = 10 + testEngineMaxSizePerMsg = 1 << 20 + testEngineMaxInflight = 256 +) + +func newTestFactory() raftengine.Factory { + return etcdraftengine.NewFactory(etcdraftengine.FactoryConfig{ + TickInterval: testEngineTickInterval, + HeartbeatTick: testEngineHeartbeatTick, + ElectionTick: testEngineElectionTick, + MaxSizePerMsg: testEngineMaxSizePerMsg, + MaxInflightMsg: testEngineMaxInflight, + }) +} + func shutdown(nodes []Node) { for _, n := range nodes { if n.opsCancel != nil { @@ -41,12 +57,14 @@ func shutdown(nodes []Node) { if n.dynamoServer != nil { n.dynamoServer.Stop() } - if n.raft != nil { - n.raft.Shutdown() + if n.engine != nil { + if err := n.engine.Close(); err != nil { + log.Printf("engine close: %v", err) + } } - if n.tm != nil { - if err := n.tm.Close(); err != nil { - log.Printf("transport close: %v", err) + if n.closeFactory != nil { + if err := n.closeFactory(); err != nil { + log.Printf("factory close: %v", err) } } } @@ -69,9 +87,6 @@ const ( raftPort = 50000 redisPort = 63790 dynamoPort = 28000 - - // followers wait longer before starting elections to give the leader time to bootstrap and share config. - followerElectionTimeout = 10 * time.Second ) var mu sync.Mutex @@ -116,11 +131,11 @@ type Node struct { redisServer *RedisServer dynamoServer *DynamoDBServer opsCancel context.CancelFunc - raft *raft.Raft - tm *transport.Manager + engine raftengine.Engine + closeFactory func() error } -func newNode(grpcAddress, raftAddress, redisAddress, dynamoAddress string, r *raft.Raft, tm *transport.Manager, grpcs *grpc.Server, grpcService *GRPCServer, rd *RedisServer, ds *DynamoDBServer, opsCancel context.CancelFunc) Node { +func newNode(grpcAddress, raftAddress, redisAddress, dynamoAddress string, engine raftengine.Engine, closeFactory func() error, grpcs *grpc.Server, grpcService *GRPCServer, rd *RedisServer, ds *DynamoDBServer, opsCancel context.CancelFunc) Node { return Node{ grpcAddress: grpcAddress, raftAddress: raftAddress, @@ -131,8 +146,8 @@ func newNode(grpcAddress, raftAddress, redisAddress, dynamoAddress string, r *ra redisServer: rd, dynamoServer: ds, opsCancel: opsCancel, - raft: r, - tm: tm, + engine: engine, + closeFactory: closeFactory, } } @@ -148,11 +163,10 @@ func createNode(t *testing.T, n int) ([]Node, []string, []string) { ctx := context.Background() ports := assignPorts(n) - nodes, grpcAdders, redisAdders, cfg := setupNodes(t, ctx, n, ports) + nodes, grpcAdders, redisAdders, peers := setupNodes(t, ctx, n, ports) waitForNodeListeners(t, ctx, nodes, waitTimeout, waitInterval) - waitForConfigReplication(t, cfg, nodes, waitTimeout, waitInterval) - waitForRaftReadiness(t, nodes, waitTimeout, waitInterval) + waitForRaftReadiness(t, nodes, peers, waitTimeout, waitInterval) return nodes, grpcAdders, redisAdders } @@ -218,64 +232,32 @@ func waitForNodeListeners(t *testing.T, ctx context.Context, nodes []Node, waitT } } -func waitForRaftReadiness(t *testing.T, nodes []Node, waitTimeout, waitInterval time.Duration) { - t.Helper() - - expectedLeader := raft.ServerAddress(nodes[0].raftAddress) - assert.Eventually(t, func() bool { - for i, n := range nodes { - state := n.raft.State() - if i == 0 { - if state != raft.Leader { - return false - } - } else if state != raft.Follower { - return false - } - - addr, _ := n.raft.LeaderWithID() - if addr != expectedLeader { - return false - } - } - return true - }, waitTimeout, waitInterval) -} - -func waitForConfigReplication(t *testing.T, cfg raft.Configuration, nodes []Node, waitTimeout, waitInterval time.Duration) { +func waitForRaftReadiness(t *testing.T, nodes []Node, peers []raftengine.Server, waitTimeout, waitInterval time.Duration) { t.Helper() assert.Eventually(t, func() bool { + var leaderAddr string for _, n := range nodes { - future := n.raft.GetConfiguration() - if future.Error() != nil { + leader := n.engine.Leader().Address + if leader == "" { return false } - - current := future.Configuration().Servers - if len(current) != len(cfg.Servers) { + if leaderAddr == "" { + leaderAddr = leader + } else if leader != leaderAddr { return false } - - for _, expected := range cfg.Servers { - if !containsServer(current, expected) { - return false - } + } + // Confirm the leader address belongs to the configured peers. + for _, p := range peers { + if p.Address == leaderAddr { + return true } } - return true + return false }, waitTimeout, waitInterval) } -func containsServer(servers []raft.Server, expected raft.Server) bool { - for _, s := range servers { - if s.ID == expected.ID && s.Address == expected.Address && s.Suffrage == expected.Suffrage { - return true - } - } - return false -} - func assignPorts(n int) []portsAdress { ports := make([]portsAdress, n) for i := range n { @@ -284,27 +266,19 @@ func assignPorts(n int) []portsAdress { return ports } -func buildRaftConfig(n int, ports []portsAdress) raft.Configuration { - cfg := raft.Configuration{} +func buildTestPeers(n int, ports []portsAdress) []raftengine.Server { + peers := make([]raftengine.Server, 0, n) for i := range n { - suffrage := raft.Nonvoter - if i == 0 { - suffrage = raft.Voter - } - - cfg.Servers = append(cfg.Servers, raft.Server{ - Suffrage: suffrage, - ID: raft.ServerID(strconv.Itoa(i)), - Address: raft.ServerAddress(ports[i].raftAddress), + peers = append(peers, raftengine.Server{ + Suffrage: "voter", + ID: strconv.Itoa(i), + Address: ports[i].raftAddress, }) } - - return cfg + return peers } -const leaderElectionTimeout = 0 * time.Second - -func setupNodes(t *testing.T, ctx context.Context, n int, ports []portsAdress) ([]Node, []string, []string, raft.Configuration) { +func setupNodes(t *testing.T, ctx context.Context, n int, ports []portsAdress) ([]Node, []string, []string, []raftengine.Server) { t.Helper() var grpcAdders []string var redisAdders []string @@ -331,12 +305,14 @@ func setupNodes(t *testing.T, ctx context.Context, n int, ports []portsAdress) ( } } - cfg := buildRaftConfig(n, ports) - leaderRedisMap := make(map[raft.ServerAddress]string, len(ports)) + peers := buildTestPeers(n, ports) + leaderRedisMap := make(map[string]string, len(ports)) for _, p := range ports { - leaderRedisMap[raft.ServerAddress(p.raftAddress)] = p.redisAddress + leaderRedisMap[p.raftAddress] = p.redisAddress } + factory := newTestFactory() + for i := range n { st := store.NewMVCCStore() // Share a single HLC between the FSM and the coordinator so the FSM can @@ -350,28 +326,31 @@ func setupNodes(t *testing.T, ctx context.Context, n int, ports []portsAdress) ( redisSock := lis[i].redis dynamoSock := lis[i].dynamo - // リーダーが先に投票を開始させる - electionTimeout := leaderElectionTimeout - if i != 0 { - electionTimeout = followerElectionTimeout - } - - r, tm, err := newRaft(strconv.Itoa(i), port.raftAddress, fsm, i == 0, cfg, electionTimeout) - assert.NoError(t, err) + result, err := factory.Create(raftengine.FactoryConfig{ + LocalID: strconv.Itoa(i), + LocalAddress: port.raftAddress, + DataDir: t.TempDir(), + Peers: peers, + Bootstrap: true, + StateMachine: fsm, + }) + require.NoError(t, err) s := grpc.NewServer(internalutil.GRPCServerOptions()...) - trx := kv.NewTransaction(r) - coordinator := kv.NewCoordinator(trx, r, kv.WithHLC(hlc)) + trx := kv.NewTransactionWithProposer(result.Engine) + coordinator := kv.NewCoordinatorWithEngine(trx, result.Engine, kv.WithHLC(hlc)) relay := NewRedisPubSubRelay() routedStore := kv.NewLeaderRoutedStore(st, coordinator) gs := NewGRPCServer(routedStore, coordinator, WithCloseStore()) opsCtx, opsCancel := context.WithCancel(ctx) go coordinator.RunHLCLeaseRenewal(opsCtx) - tm.Register(s) + if result.RegisterTransport != nil { + result.RegisterTransport(s) + } pb.RegisterRawKVServer(s, gs) pb.RegisterTransactionalKVServer(s, gs) - pb.RegisterInternalServer(s, NewInternal(trx, r, coordinator.Clock(), relay)) - internalraftadmin.RegisterOperationalServices(opsCtx, s, hashicorpraftengine.New(r), []string{"Example"}) + pb.RegisterInternalServer(s, NewInternalWithEngine(trx, result.Engine, coordinator.Clock(), relay)) + internalraftadmin.RegisterOperationalServices(opsCtx, s, result.Engine, []string{"Example"}) grpcAdders = append(grpcAdders, port.grpcAddress) redisAdders = append(redisAdders, port.redisAddress) @@ -398,8 +377,8 @@ func setupNodes(t *testing.T, ctx context.Context, n int, ports []portsAdress) ( port.raftAddress, port.redisAddress, port.dynamoAddress, - r, - tm, + result.Engine, + result.Close, s, gs, rd, @@ -408,41 +387,5 @@ func setupNodes(t *testing.T, ctx context.Context, n int, ports []portsAdress) ( )) } - return nodes, grpcAdders, redisAdders, cfg -} - -func newRaft(myID string, myAddress string, fsm raft.FSM, bootstrap bool, cfg raft.Configuration, electionTimeout time.Duration) (*raft.Raft, *transport.Manager, error) { - c := raft.DefaultConfig() - c.LocalID = raft.ServerID(myID) - c.CommitTimeout = 1 * time.Millisecond - - if electionTimeout > 0 { - c.ElectionTimeout = electionTimeout - } - - // this config is for development - ldb := raft.NewInmemStore() - sdb := raft.NewInmemStore() - fss := raft.NewInmemSnapshotStore() - - c.Logger = hclog.New(&hclog.LoggerOptions{ - Name: "raft-" + myID, - Level: hclog.LevelFromString("WARN"), - }) - - tm := transport.New(raft.ServerAddress(myAddress), internalutil.GRPCDialOptions()) - - r, err := raft.NewRaft(c, fsm, ldb, sdb, fss, tm.Transport()) - if err != nil { - return nil, nil, errors.WithStack(err) - } - - if bootstrap { - f := r.BootstrapCluster(cfg) - if err := f.Error(); err != nil { - return nil, nil, errors.WithStack(err) - } - } - - return r, tm, nil + return nodes, grpcAdders, redisAdders, peers } diff --git a/cmd/etcd-raft-migrate/main.go b/cmd/etcd-raft-migrate/main.go deleted file mode 100644 index 28472dd26..000000000 --- a/cmd/etcd-raft-migrate/main.go +++ /dev/null @@ -1,29 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - - etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" -) - -var ( - sourceFSM = flag.String("fsm-store", "", "Path to the source FSM Pebble store (for example group dir/fsm.db)") - destDir = flag.String("dest", "", "Destination etcd raft data dir") - peerList = flag.String("peers", "", "Comma-separated raft peers (id=host:port,...)") -) - -func main() { - flag.Parse() - - peers, err := etcdraftengine.ParsePeers(*peerList) - if err != nil { - log.Fatal(err) - } - stats, err := etcdraftengine.MigrateFSMStore(*sourceFSM, *destDir, peers) - if err != nil { - log.Fatal(err) - } - fmt.Printf("migrated snapshot_bytes=%d peers=%d\n", stats.SnapshotBytes, stats.Peers) -} diff --git a/cmd/etcd-raft-rollback/main.go b/cmd/etcd-raft-rollback/main.go deleted file mode 100644 index 4099973c3..000000000 --- a/cmd/etcd-raft-rollback/main.go +++ /dev/null @@ -1,29 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" -) - -var ( - sourceFSM = flag.String("fsm-store", "", "Path to the source FSM Pebble store (for example group dir/fsm.db)") - destDir = flag.String("dest", "", "Destination hashicorp raft data dir") - peerList = flag.String("peers", "", "Comma-separated raft peers (id=host:port,...)") -) - -func main() { - flag.Parse() - - peers, err := hashicorpraftengine.ParsePeers(*peerList) - if err != nil { - log.Fatal(err) - } - stats, err := hashicorpraftengine.MigrateFSMStore(*sourceFSM, *destDir, peers) - if err != nil { - log.Fatal(err) - } - fmt.Printf("rolled back snapshot_bytes=%d peers=%d\n", stats.SnapshotBytes, stats.Peers) -} diff --git a/cmd/server/demo.go b/cmd/server/demo.go index d58909c81..8f2f4d69b 100644 --- a/cmd/server/demo.go +++ b/cmd/server/demo.go @@ -14,20 +14,17 @@ import ( "strings" "time" - transport "github.com/Jille/raft-grpc-transport" "github.com/bootjp/elastickv/adapter" "github.com/bootjp/elastickv/distribution" internalutil "github.com/bootjp/elastickv/internal" internalraftadmin "github.com/bootjp/elastickv/internal/raftadmin" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" - "github.com/bootjp/elastickv/internal/raftstore" + "github.com/bootjp/elastickv/internal/raftengine" + etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/monitoring" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" - "github.com/hashicorp/go-hclog" - "github.com/hashicorp/raft" "golang.org/x/sync/errgroup" "google.golang.org/grpc" ) @@ -53,14 +50,18 @@ var ( ) const ( - raftSnapshotsRetain = 2 - kvParts = 2 - defaultFileMode = 0755 - joinRetries = 20 - joinWait = 3 * time.Second - joinRetryInterval = 1 * time.Second - joinRPCTimeout = 3 * time.Second - raftObserveInterval = 5 * time.Second + kvParts = 2 + defaultFileMode = 0755 + joinRetries = 20 + joinWait = 3 * time.Second + joinRetryInterval = 1 * time.Second + joinRPCTimeout = 3 * time.Second + raftObserveInterval = 5 * time.Second + demoTickInterval = 10 * time.Millisecond + demoHeartbeatTick = 1 + demoElectionTick = 10 + demoMaxSizePerMsg = 1 << 20 + demoMaxInflightMsg = 256 ) func init() { @@ -334,43 +335,6 @@ func joinClusterWaitError(err error) error { return err } -func setupStorage(dir string) (raft.LogStore, raft.StableStore, raft.SnapshotStore, error) { - if dir == "" { - return raft.NewInmemStore(), raft.NewInmemStore(), raft.NewInmemSnapshotStore(), nil - } - for _, legacy := range []string{"logs.dat", "stable.dat"} { - if _, err := os.Stat(filepath.Join(dir, legacy)); err == nil { - return nil, nil, nil, errors.WithStack(errors.Newf( - "legacy boltdb Raft storage %q found in %s; manual migration required before using Pebble-backed storage", - legacy, dir, - )) - } - } - raftStore, err := raftstore.NewPebbleStore(filepath.Join(dir, "raft.db")) - if err != nil { - return nil, nil, nil, errors.WithStack(err) - } - fss, err := raft.NewFileSnapshotStore(dir, raftSnapshotsRetain, os.Stdout) - if err != nil { - _ = raftStore.Close() - return nil, nil, nil, errors.WithStack(err) - } - return raftStore, raftStore, fss, nil -} - -// setupStores creates both the Raft log/stable/snapshot stores and the FSM MVCCStore. -func setupStores(raftDataDir string, cleanup *internalutil.CleanupStack) (raft.LogStore, raft.StableStore, raft.SnapshotStore, store.MVCCStore, error) { - ldb, sdb, fss, err := setupStorage(raftDataDir) - if err != nil { - return nil, nil, nil, nil, err - } - st, err := setupFSMStore(raftDataDir, cleanup) - if err != nil { - return nil, nil, nil, nil, err - } - return ldb, sdb, fss, st, nil -} - // setupFSMStore creates and returns the MVCCStore for the Raft FSM. // When raftDataDir is non-empty the store is persisted under that directory; // otherwise a temporary directory is used and registered for cleanup on exit. @@ -397,33 +361,35 @@ func setupFSMStore(raftDataDir string, cleanup *internalutil.CleanupStack) (stor return st, nil } -func setupGRPC(ctx context.Context, r *raft.Raft, st store.MVCCStore, tm *transport.Manager, coordinator *kv.Coordinate, distServer *adapter.DistributionServer, relay *adapter.RedisPubSubRelay, proposalObserver kv.ProposalObserver) (*grpc.Server, *adapter.GRPCServer) { +func setupGRPC(ctx context.Context, engine raftengine.Engine, registerTransport func(grpc.ServiceRegistrar), st store.MVCCStore, coordinator *kv.Coordinate, distServer *adapter.DistributionServer, relay *adapter.RedisPubSubRelay, proposalObserver kv.ProposalObserver) (*grpc.Server, *adapter.GRPCServer) { s := grpc.NewServer(internalutil.GRPCServerOptions()...) - trx := kv.NewTransaction(r, kv.WithProposalObserver(proposalObserver)) + trx := kv.NewTransactionWithProposer(engine, kv.WithProposalObserver(proposalObserver)) routedStore := kv.NewLeaderRoutedStore(st, coordinator) gs := adapter.NewGRPCServer(routedStore, coordinator, adapter.WithCloseStore()) - tm.Register(s) + if registerTransport != nil { + registerTransport(s) + } pb.RegisterRawKVServer(s, gs) pb.RegisterTransactionalKVServer(s, gs) - pb.RegisterInternalServer(s, adapter.NewInternal(trx, r, coordinator.Clock(), relay)) + pb.RegisterInternalServer(s, adapter.NewInternalWithEngine(trx, engine, coordinator.Clock(), relay)) pb.RegisterDistributionServer(s, distServer) - internalraftadmin.RegisterOperationalServices(ctx, s, hashicorpraftengine.New(r), []string{"RawKV"}) + internalraftadmin.RegisterOperationalServices(ctx, s, engine, []string{"RawKV"}) return s, gs } func setupRedis(ctx context.Context, lc net.ListenConfig, st store.MVCCStore, coordinator *kv.Coordinate, addr, redisAddr, raftRedisMapStr string, relay *adapter.RedisPubSubRelay, readTracker *kv.ActiveTimestampTracker, deltaCompactor *adapter.DeltaCompactor) (*adapter.RedisServer, error) { - leaderRedis := make(map[raft.ServerAddress]string) + leaderRedis := make(map[string]string) if raftRedisMapStr != "" { parts := strings.SplitSeq(raftRedisMapStr, ",") for part := range parts { kv := strings.Split(part, "=") if len(kv) == kvParts { - leaderRedis[raft.ServerAddress(kv[0])] = kv[1] + leaderRedis[kv[0]] = kv[1] } } } // Ensure self is in map (override if present) - leaderRedis[raft.ServerAddress(addr)] = redisAddr + leaderRedis[addr] = redisAddr l, err := lc.Listen(ctx, "tcp", redisAddr) if err != nil { @@ -455,7 +421,7 @@ func setupS3( if coordinator == nil { return nil, errors.New("coordinator must not be nil") } - leaderS3 := make(map[raft.ServerAddress]string) + leaderS3 := make(map[string]string) if raftS3MapStr != "" { parts := strings.SplitSeq(raftS3MapStr, ",") for part := range parts { @@ -468,10 +434,10 @@ func setupS3( slog.Warn("ignoring invalid raft-s3 map entry; expected format addr=s3addr", "entry", part) continue } - leaderS3[raft.ServerAddress(strings.TrimSpace(kv[0]))] = strings.TrimSpace(kv[1]) + leaderS3[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1]) } } - leaderS3[raft.ServerAddress(addr)] = s3Addr + leaderS3[addr] = s3Addr l, err := lc.Listen(ctx, "tcp", s3Addr) if err != nil { @@ -496,16 +462,16 @@ func setupS3( } func setupDynamo(ctx context.Context, lc net.ListenConfig, st store.MVCCStore, coordinator *kv.Coordinate, addr, dynamoAddr, raftDynamoMapStr string, observer monitoring.DynamoDBRequestObserver) (*adapter.DynamoDBServer, error) { - leaderDynamo := make(map[raft.ServerAddress]string) + leaderDynamo := make(map[string]string) if raftDynamoMapStr != "" { for part := range strings.SplitSeq(raftDynamoMapStr, ",") { pair := strings.SplitN(part, "=", kvParts) if len(pair) == kvParts { - leaderDynamo[raft.ServerAddress(pair[0])] = pair[1] + leaderDynamo[pair[0]] = pair[1] } } } - leaderDynamo[raft.ServerAddress(addr)] = dynamoAddr + leaderDynamo[addr] = dynamoAddr l, err := lc.Listen(ctx, "tcp", dynamoAddr) if err != nil { return nil, errors.WithStack(err) @@ -522,7 +488,7 @@ func run(ctx context.Context, eg *errgroup.Group, cfg config) error { cleanup := internalutil.CleanupStack{} defer cleanup.Run() - ldb, sdb, fss, st, err := setupStores(cfg.raftDataDir, &cleanup) + st, err := setupFSMStore(cfg.raftDataDir, &cleanup) if err != nil { return err } @@ -531,30 +497,46 @@ func run(ctx context.Context, eg *errgroup.Group, cfg config) error { fsm := kv.NewKvFSMWithHLC(st, hlc) readTracker := kv.NewActiveTimestampTracker() - // Config - c := raft.DefaultConfig() - c.LocalID = raft.ServerID(cfg.raftID) - c.Logger = hclog.New(&hclog.LoggerOptions{ - Name: "raft-" + cfg.raftID, - JSONFormat: true, - Level: hclog.Info, + factory := etcdraftengine.NewFactory(etcdraftengine.FactoryConfig{ + TickInterval: demoTickInterval, + HeartbeatTick: demoHeartbeatTick, + ElectionTick: demoElectionTick, + MaxSizePerMsg: demoMaxSizePerMsg, + MaxInflightMsg: demoMaxInflightMsg, }) - // Transport - tm := transport.New(raft.ServerAddress(cfg.address), internalutil.GRPCDialOptions()) - - r, err := raft.NewRaft(c, fsm, ldb, sdb, fss, tm.Transport()) - if err != nil { + raftDir := cfg.raftDataDir + if raftDir == "" { + tmp, err := os.MkdirTemp("", "elastickv-raft-*") + if err != nil { + return errors.WithStack(err) + } + cleanup.Add(func() { os.RemoveAll(tmp) }) + raftDir = tmp + } else if err := os.MkdirAll(raftDir, defaultFileMode); err != nil { return errors.WithStack(err) } - if err := bootstrapClusterIfNeeded(r, cfg); err != nil { - return err + result, err := factory.Create(raftengine.FactoryConfig{ + LocalID: cfg.raftID, + LocalAddress: cfg.address, + DataDir: raftDir, + Bootstrap: cfg.raftBootstrap, + StateMachine: fsm, + }) + if err != nil { + return errors.WithStack(err) } + cleanup.Add(func() { + _ = result.Engine.Close() + if result.Close != nil { + _ = result.Close() + } + }) metricsRegistry := monitoring.NewRegistry(cfg.raftID, cfg.address) proposalObserver := metricsRegistry.RaftProposalObserver(1) - engine := hashicorpraftengine.New(r) + engine := result.Engine trx := kv.NewTransactionWithProposer(engine, kv.WithProposalObserver(proposalObserver)) coordinator := kv.NewCoordinatorWithEngine(trx, engine, kv.WithHLC(hlc)) defer func() { @@ -590,7 +572,7 @@ func run(ctx context.Context, eg *errgroup.Group, cfg config) error { ) relay := adapter.NewRedisPubSubRelay() - s, grpcSvc := setupGRPC(ctx, r, st, tm, coordinator, distServer, relay, proposalObserver) + s, grpcSvc := setupGRPC(ctx, engine, result.RegisterTransport, st, coordinator, distServer, relay, proposalObserver) grpcSock, err := lc.Listen(ctx, "tcp", cfg.address) if err != nil { @@ -699,24 +681,6 @@ func setupPprofHTTPServer(ctx context.Context, lc net.ListenConfig, pprofAddress return pprofL, ps, nil } -func bootstrapClusterIfNeeded(r *raft.Raft, cfg config) error { - if !cfg.raftBootstrap { - return nil - } - bootstrapCfg := raft.Configuration{ - Servers: []raft.Server{ - { - Suffrage: raft.Voter, - ID: raft.ServerID(cfg.raftID), - Address: raft.ServerAddress(cfg.address), - }, - }, - } - if err := r.BootstrapCluster(bootstrapCfg).Error(); err != nil && !errors.Is(err, raft.ErrCantBootstrap) { - return errors.WithStack(err) - } - return nil -} func catalogWatcherTask(ctx context.Context, distCatalog *distribution.CatalogStore, distEngine *distribution.Engine) func() error { return func() error { diff --git a/go.mod b/go.mod index b13778ffb..e34dc8911 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ toolchain go1.26.2 require ( github.com/Jille/grpc-multi-resolver v1.3.0 - github.com/Jille/raft-grpc-transport v1.6.1 github.com/aws/aws-sdk-go-v2 v1.41.5 github.com/aws/aws-sdk-go-v2/config v1.32.14 github.com/aws/aws-sdk-go-v2/credentials v1.19.14 @@ -17,8 +16,6 @@ require ( github.com/emirpasic/gods v1.18.1 github.com/getsentry/sentry-go v0.45.0 github.com/goccy/go-json v0.10.6 - github.com/hashicorp/go-hclog v1.6.3 - github.com/hashicorp/raft v1.7.3 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.23.2 github.com/redis/go-redis/v9 v9.18.0 @@ -41,7 +38,6 @@ require ( github.com/DataDog/zstd v1.5.7 // indirect github.com/RaduBerinde/axisds v0.1.0 // indirect github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect - github.com/armon/go-metrics v0.4.1 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect @@ -65,7 +61,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/fatih/color v1.15.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -74,19 +69,11 @@ require ( github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/go-immutable-radix v1.3.1 // indirect - github.com/hashicorp/go-metrics v0.5.4 // indirect - github.com/hashicorp/go-msgpack/v2 v2.1.5 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 1ec8049e4..41662c8ce 100644 --- a/go.sum +++ b/go.sum @@ -1,30 +1,13 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Jille/grpc-multi-resolver v1.3.0 h1:cbVm1TtWP7YxdiCCZ8gU4/78pYO2OXpzZSFAAUMdFLs= github.com/Jille/grpc-multi-resolver v1.3.0/go.mod h1:vEHO+TZo6TUee3VbNdXq4iiUQGvItfmeGcdNOX2usnM= -github.com/Jille/raft-grpc-transport v1.6.1 h1:gN3sjapb+fVbiebS7AfQQgbV2ecTOI7ur7NPPC7Mhoc= -github.com/Jille/raft-grpc-transport v1.6.1/go.mod h1:HbOjEdu/yzCJ/mjTF6wEOJNbAUpHfU2UOA2hVD4CNFg= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/RaduBerinde/axisds v0.1.0 h1:YItk/RmU5nvlsv/awo2Fjx97Mfpt4JfgtEVAGPrLdz8= github.com/RaduBerinde/axisds v0.1.0/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y= github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk= github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc= -github.com/Sereal/Sereal/Go/sereal v0.0.0-20231009093132-b9187f1a92c6/go.mod h1:JwrycNnC8+sZPDyzM3MQ86LvaGzSpfxg885KOOwFRW4= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= -github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI= @@ -57,25 +40,14 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBU github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= github.com/aws/smithy-go v1.24.3 h1:XgOAaUgx+HhVBoP4v8n6HCQoTRDhoMghKqw4LNHsDNg= github.com/aws/smithy-go v1.24.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU= github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI= @@ -99,239 +71,84 @@ github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03V github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE= -github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc= -github.com/dgryski/go-ddmin v0.0.0-20210904190556-96a6d69f1034/go.mod h1:zz4KxBkcXUWKjIcrc+uphJ1gPh/t18ymGm3PmQ+VGTk= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/getsentry/sentry-go v0.45.0 h1:/ZlbfGcaOzG4QkCACCfxrbuABemjem7UnY5o+V5HmeM= github.com/getsentry/sentry-go v0.45.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0= github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9 h1:r5GgOLGbza2wVHRzK7aAj6lWZjfbAwiu/RDCVOKjRyM= github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e h1:4bw4WeyTYPp0smaXiJZCNnLrvVBqirQVreixayXezGc= github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-hclog v1.6.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= -github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= -github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-metrics v0.5.4 h1:8mmPiIJkTPPEbAiV97IxdAGNdRdaWwVap1BU6elejKY= -github.com/hashicorp/go-metrics v0.5.4/go.mod h1:CG5yz4NZ/AI/aQt9Ucm/vdBnbh7fvmv4lxZ350i+QQI= -github.com/hashicorp/go-msgpack/v2 v2.1.1/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4= -github.com/hashicorp/go-msgpack/v2 v2.1.5 h1:Ue879bPnutj/hXfmUk6s/jtIK90XxgiUIcXRl656T44= -github.com/hashicorp/go-msgpack/v2 v2.1.5/go.mod h1:bjCsRXpZ7NsJdk45PoCQnzRGDaK8TKm5ZnDI/9y3J4M= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= -github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/raft v1.7.0/go.mod h1:N1sKh6Vn47mrWvEArQgILTyng8GoDRNYlgKyK7PMjs0= -github.com/hashicorp/raft v1.7.3 h1:DxpEqZJysHN0wK+fviai5mFcSYsCkNpFUl1xpAW8Rbo= -github.com/hashicorp/raft v1.7.3/go.mod h1:DfvCGFxpAUPE0L4Uc8JLlTPtc3GzSbdH0MTJCLgnmJQ= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw= github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/btree v1.1.0 h1:5P+9WU8ui5uhmcg3SoPyTwoI0mVyZ1nps7YQzTZFkYM= @@ -340,16 +157,12 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/redcon v1.6.2 h1:5qfvrrybgtO85jnhSravmkZyC0D+7WstbfCs3MmPhow= github.com/tidwall/redcon v1.6.2/go.mod h1:p5Wbsgeyi2VSTBWOcA5vRXrOb9arFTcU2+ZzFjqV75Y= -github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA= github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= @@ -382,10 +195,8 @@ go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4A go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -394,201 +205,57 @@ go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto/googleapis/api v0.0.0-20260406154035-8fb7ec149431 h1:mOx7UuWq7wITAe4Qn4Yqq8h/LZwDMQT2wW2w1BcrxRE= google.golang.org/genproto/googleapis/api v0.0.0-20260406154035-8fb7ec149431/go.mod h1:etfGUgejTiadZAUaEP14NP97xi1RGeawqkjDARA/UOs= google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/vmihailenco/msgpack.v2 v2.9.2/go.mod h1:/3Dn1Npt9+MYyLpYYXjInO/5jvMLamn+AEGwNEOatn8= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/internal/raftengine/etcd/hashicorp_fsm_adapter.go b/internal/raftengine/etcd/hashicorp_fsm_adapter.go deleted file mode 100644 index 79312a4ed..000000000 --- a/internal/raftengine/etcd/hashicorp_fsm_adapter.go +++ /dev/null @@ -1,76 +0,0 @@ -package etcd - -import ( - "io" - - "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" -) - -// AdaptHashicorpFSM bridges a hashicorp/raft FSM into the etcd-backed engine. -func AdaptHashicorpFSM(fsm raft.FSM) StateMachine { - if fsm == nil { - return nil - } - return hashicorpFSMAdapter{fsm: fsm} -} - -type hashicorpFSMAdapter struct { - fsm raft.FSM -} - -func (a hashicorpFSMAdapter) Apply(data []byte) any { - return a.fsm.Apply(&raft.Log{Type: raft.LogCommand, Data: data}) -} - -func (a hashicorpFSMAdapter) Snapshot() (Snapshot, error) { - snapshot, err := a.fsm.Snapshot() - if err != nil { - return nil, errors.WithStack(err) - } - return raftFSMSnapshotAdapter{snapshot: snapshot}, nil -} - -func (a hashicorpFSMAdapter) Restore(r io.Reader) error { - return errors.WithStack(a.fsm.Restore(io.NopCloser(r))) -} - -type raftFSMSnapshotAdapter struct { - snapshot raft.FSMSnapshot -} - -func (a raftFSMSnapshotAdapter) WriteTo(w io.Writer) (int64, error) { - sink := &raftSnapshotSinkAdapter{writer: w} - if err := a.snapshot.Persist(sink); err != nil { - return sink.written, errors.WithStack(err) - } - return sink.written, nil -} - -func (a raftFSMSnapshotAdapter) Close() error { - a.snapshot.Release() - return nil -} - -type raftSnapshotSinkAdapter struct { - writer io.Writer - written int64 -} - -func (s *raftSnapshotSinkAdapter) ID() string { - return "etcd-hashicorp-fsm-adapter" -} - -func (s *raftSnapshotSinkAdapter) Cancel() error { - return nil -} - -func (s *raftSnapshotSinkAdapter) Close() error { - return nil -} - -func (s *raftSnapshotSinkAdapter) Write(p []byte) (int, error) { - n, err := s.writer.Write(p) - s.written += int64(n) - return n, errors.WithStack(err) -} diff --git a/internal/raftengine/etcd/migrate_test.go b/internal/raftengine/etcd/migrate_test.go index baa099e4c..63fa3fa5b 100644 --- a/internal/raftengine/etcd/migrate_test.go +++ b/internal/raftengine/etcd/migrate_test.go @@ -2,67 +2,14 @@ package etcd import ( "context" - "io" "path/filepath" "testing" "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" - "github.com/hashicorp/raft" "github.com/stretchr/testify/require" ) -type kvFSMAdapter struct { - fsm raft.FSM -} - -func (a kvFSMAdapter) Apply(data []byte) any { - return a.fsm.Apply(&raft.Log{Type: raft.LogCommand, Data: data}) -} - -func (a kvFSMAdapter) Snapshot() (Snapshot, error) { - snapshot, err := a.fsm.Snapshot() - if err != nil { - return nil, err - } - return hashicorpSnapshotAdapter{snapshot: snapshot}, nil -} - -func (a kvFSMAdapter) Restore(r io.Reader) error { - return a.fsm.Restore(io.NopCloser(r)) -} - -type hashicorpSnapshotAdapter struct { - snapshot raft.FSMSnapshot -} - -func (a hashicorpSnapshotAdapter) WriteTo(w io.Writer) (int64, error) { - sink := &snapshotSinkAdapter{writer: w} - if err := a.snapshot.Persist(sink); err != nil { - return sink.written, err - } - return sink.written, nil -} - -func (a hashicorpSnapshotAdapter) Close() error { - a.snapshot.Release() - return nil -} - -type snapshotSinkAdapter struct { - writer io.Writer - written int64 -} - -func (s *snapshotSinkAdapter) ID() string { return "migration" } -func (s *snapshotSinkAdapter) Cancel() error { return nil } -func (s *snapshotSinkAdapter) Close() error { return nil } -func (s *snapshotSinkAdapter) Write(p []byte) (int, error) { - n, err := s.writer.Write(p) - s.written += int64(n) - return n, err -} - func TestMigrateFSMStoreSeedsEtcdDataDir(t *testing.T) { sourcePath := filepath.Join(t.TempDir(), "fsm.db") source, err := store.NewPebbleStore(sourcePath) @@ -87,7 +34,7 @@ func TestMigrateFSMStoreSeedsEtcdDataDir(t *testing.T) { LocalID: "n1", LocalAddress: "127.0.0.1:7001", DataDir: destDataDir, - StateMachine: kvFSMAdapter{fsm: kv.NewKvFSMWithHLC(destStore, kv.NewHLC())}, + StateMachine: kv.NewKvFSMWithHLC(destStore, kv.NewHLC()), }) require.NoError(t, err) t.Cleanup(func() { diff --git a/internal/raftengine/hashicorp/engine.go b/internal/raftengine/hashicorp/engine.go deleted file mode 100644 index b025ba328..000000000 --- a/internal/raftengine/hashicorp/engine.go +++ /dev/null @@ -1,468 +0,0 @@ -package hashicorp - -import ( - "context" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/bootjp/elastickv/internal/raftengine" - "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" -) - -const defaultProposalTimeout = time.Second -const unknownLastContact = time.Duration(-1) - -var errNilEngine = errors.New("raft engine is not configured") - -// translateLeadershipErr wraps hashicorp/raft leadership-related sentinels -// with the shared raftengine sentinels so callers can use a single -// errors.Is check across engine backends. -func translateLeadershipErr(err error) error { - if err == nil { - return nil - } - switch { - case errors.Is(err, raft.ErrNotLeader): - return errors.WithStack(errors.Mark(err, raftengine.ErrNotLeader)) - case errors.Is(err, raft.ErrLeadershipLost): - return errors.WithStack(errors.Mark(err, raftengine.ErrLeadershipLost)) - case errors.Is(err, raft.ErrLeadershipTransferInProgress): - return errors.WithStack(errors.Mark(err, raftengine.ErrLeadershipTransferInProgress)) - } - return errors.WithStack(err) -} - -type Engine struct { - raft *raft.Raft - - barrierMu sync.Mutex // serialises Barrier execution (slow path only) - barrierTerm atomic.Uint64 // term of last successful Barrier (lock-free read) -} - -func New(r *raft.Raft) *Engine { - if r == nil { - return nil - } - return &Engine{raft: r} -} - -// RaftInstance returns the underlying hashicorp raft instance. This is -// provided for backward compatibility during the engine migration; callers -// should use the Engine interface instead. -func (e *Engine) RaftInstance() *raft.Raft { - if e == nil { - return nil - } - return e.raft -} - -func (e *Engine) Close() error { - if e == nil || e.raft == nil { - return nil - } - if err := e.raft.Shutdown().Error(); err != nil { - return errors.WithStack(err) - } - return nil -} - -func (e *Engine) Propose(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) { - timeout, err := timeoutFromContext(ctx) - if err != nil { - return nil, err - } - if e == nil || e.raft == nil { - return nil, errors.WithStack(errNilEngine) - } - - af := e.raft.Apply(data, timeout) - if err := af.Error(); err != nil { - if ctxErr := contextErr(ctx); ctxErr != nil { - return nil, ctxErr - } - return nil, translateLeadershipErr(err) - } - - return &raftengine.ProposalResult{ - CommitIndex: af.Index(), - Response: af.Response(), - }, nil -} - -func (e *Engine) State() raftengine.State { - if e == nil || e.raft == nil { - return raftengine.StateUnknown - } - - switch e.raft.State() { - case raft.Follower: - return raftengine.StateFollower - case raft.Candidate: - return raftengine.StateCandidate - case raft.Leader: - return raftengine.StateLeader - case raft.Shutdown: - return raftengine.StateShutdown - default: - return raftengine.StateUnknown - } -} - -func (e *Engine) Leader() raftengine.LeaderInfo { - if e == nil || e.raft == nil { - return raftengine.LeaderInfo{} - } - - addr, id := e.raft.LeaderWithID() - return raftengine.LeaderInfo{ - ID: string(id), - Address: string(addr), - } -} - -func (e *Engine) VerifyLeader(ctx context.Context) error { - if err := contextErr(ctx); err != nil { - return err - } - if e == nil || e.raft == nil { - return errors.WithStack(errNilEngine) - } - - if err := e.raft.VerifyLeader().Error(); err != nil { - if ctxErr := contextErr(ctx); ctxErr != nil { - return ctxErr - } - return translateLeadershipErr(err) - } - return nil -} - -// readIndexPollInterval is the interval between AppliedIndex polls while -// waiting for the FSM to catch up to the commit index. 1ms keeps tail -// latency low for a KV store while AppliedIndex is an atomic load. -const readIndexPollInterval = 1 * time.Millisecond - -func (e *Engine) CheckServing(ctx context.Context) error { - if err := contextErr(ctx); err != nil { - return err - } - if e == nil || e.raft == nil { - return errors.WithStack(errNilEngine) - } - if e.State() != raftengine.StateLeader { - return errors.WithStack(errors.Mark(raft.ErrNotLeader, raftengine.ErrNotLeader)) - } - return nil -} - -// LinearizableRead blocks until the local FSM has applied all entries up to -// the current commit index, guaranteeing that a subsequent local read -// observes the latest committed state (linearizable read). It first -// ensures that at least one Barrier has been issued since the last -// leadership transition (Raft §5.4.2), then records CommitIndex, and -// finally confirms leadership via a quorum check (VerifyLeader) so that -// a partitioned leader cannot return stale data. When Barrier was -// executed in this call, VerifyLeader is skipped because Barrier already -// implies a quorum commit. -func (e *Engine) LinearizableRead(ctx context.Context) (uint64, error) { - if e == nil || e.raft == nil { - return 0, errors.WithStack(errNilEngine) - } - if e.raft.State() != raft.Leader { - return 0, errors.WithStack(errors.Mark(raft.ErrNotLeader, raftengine.ErrNotLeader)) - } - - // Raft §5.4.2: ensure at least one Barrier has been issued in the - // current term so that CommitIndex is authoritative. - performedBarrier, err := e.ensureBarrierForTerm(ctx) - if err != nil { - return 0, err - } - - // Record CommitIndex before the quorum check. The ReadIndex protocol - // requires the leadership verification to happen after the read index - // has been determined. When Barrier was just performed it already - // confirmed leadership via quorum, so VerifyLeader is redundant. - commitIndex := e.raft.CommitIndex() - if !performedBarrier { - if err := e.VerifyLeader(ctx); err != nil { - return 0, err - } - } - - return e.waitForApplied(ctx, commitIndex) -} - -// waitForApplied blocks until the FSM has applied all entries up to the -// given commit index, or the context is cancelled. -func (e *Engine) waitForApplied(ctx context.Context, commitIndex uint64) (uint64, error) { - if e.raft.AppliedIndex() >= commitIndex { - return commitIndex, nil - } - - ticker := time.NewTicker(readIndexPollInterval) - defer ticker.Stop() - - for { - if e.raft.AppliedIndex() >= commitIndex { - return commitIndex, nil - } - select { - case <-ctx.Done(): - return 0, errors.WithStack(ctx.Err()) - case <-ticker.C: - } - } -} - -// ensureBarrierForTerm issues a single Barrier per Raft term so that -// CommitIndex reflects all entries from previous terms (Raft §5.4.2). -// It returns true when a Barrier was actually performed (which implies a -// quorum commit), false when the current term already has a valid Barrier. -// -// The fast path uses an atomic load of barrierTerm to avoid mutex -// contention on concurrent reads. Only the slow path (new term detected) -// acquires barrierMu to serialise a single Barrier per term. -func (e *Engine) ensureBarrierForTerm(ctx context.Context) (bool, error) { - // Fast path: lock-free check using the atomically published barrierTerm. - term, err := parseTermFromStats(e.raft.Stats()) - if err != nil { - return false, err - } - if e.barrierTerm.Load() >= term { - return false, nil - } - - return e.executeBarrier(ctx) -} - -// executeBarrier is the slow path of ensureBarrierForTerm: it acquires -// barrierMu so that only one goroutine runs Barrier per term. -func (e *Engine) executeBarrier(ctx context.Context) (bool, error) { - e.barrierMu.Lock() - defer e.barrierMu.Unlock() - - // Re-check under lock; another goroutine may have completed Barrier. - term, err := parseTermFromStats(e.raft.Stats()) - if err != nil { - return false, err - } - if e.barrierTerm.Load() >= term { - return false, nil - } - - if e.raft.State() != raft.Leader { - return false, errors.WithStack(errors.Mark(raft.ErrNotLeader, raftengine.ErrNotLeader)) - } - - timeout, err := timeoutFromContext(ctx) - if err != nil { - return false, err - } - if err := e.raft.Barrier(timeout).Error(); err != nil { - if ctxErr := contextErr(ctx); ctxErr != nil { - return false, ctxErr - } - return false, translateLeadershipErr(err) - } - - e.barrierTerm.Store(term) - return true, nil -} - -func (e *Engine) Status() raftengine.Status { - if e == nil || e.raft == nil { - return raftengine.Status{State: raftengine.StateUnknown} - } - - stats := e.raft.Stats() - state := e.State() - return raftengine.Status{ - State: state, - Leader: e.Leader(), - Term: parseUint(stats["term"]), - CommitIndex: e.raft.CommitIndex(), - AppliedIndex: e.raft.AppliedIndex(), - LastLogIndex: e.raft.LastIndex(), - LastSnapshotIndex: parseUint(stats["last_snapshot_index"]), - FSMPending: parseUint(stats["fsm_pending"]), - NumPeers: parseUint(stats["num_peers"]), - LastContact: lastContact(state, e.raft.LastContact()), - } -} - -func (e *Engine) Configuration(ctx context.Context) (raftengine.Configuration, error) { - if err := contextErr(ctx); err != nil { - return raftengine.Configuration{}, err - } - if e == nil || e.raft == nil { - return raftengine.Configuration{}, errors.WithStack(errNilEngine) - } - - future := e.raft.GetConfiguration() - if err := future.Error(); err != nil { - if ctxErr := contextErr(ctx); ctxErr != nil { - return raftengine.Configuration{}, ctxErr - } - return raftengine.Configuration{}, errors.WithStack(err) - } - - cfg := raftengine.Configuration{Servers: make([]raftengine.Server, 0, len(future.Configuration().Servers))} - for _, server := range future.Configuration().Servers { - cfg.Servers = append(cfg.Servers, raftengine.Server{ - ID: string(server.ID), - Address: string(server.Address), - Suffrage: normalizeSuffrage(server.Suffrage), - }) - } - return cfg, nil -} - -func (e *Engine) AddVoter(ctx context.Context, id string, address string, prevIndex uint64) (uint64, error) { - timeout, err := timeoutFromContext(ctx) - if err != nil { - return 0, err - } - if e == nil || e.raft == nil { - return 0, errors.WithStack(errNilEngine) - } - - future := e.raft.AddVoter(raft.ServerID(id), raft.ServerAddress(address), prevIndex, timeout) - if err := future.Error(); err != nil { - if ctxErr := contextErr(ctx); ctxErr != nil { - return 0, ctxErr - } - return 0, errors.WithStack(err) - } - return future.Index(), nil -} - -func (e *Engine) RemoveServer(ctx context.Context, id string, prevIndex uint64) (uint64, error) { - timeout, err := timeoutFromContext(ctx) - if err != nil { - return 0, err - } - if e == nil || e.raft == nil { - return 0, errors.WithStack(errNilEngine) - } - - future := e.raft.RemoveServer(raft.ServerID(id), prevIndex, timeout) - if err := future.Error(); err != nil { - if ctxErr := contextErr(ctx); ctxErr != nil { - return 0, ctxErr - } - return 0, errors.WithStack(err) - } - return future.Index(), nil -} - -func (e *Engine) TransferLeadership(ctx context.Context) error { - if err := contextErr(ctx); err != nil { - return err - } - if e == nil || e.raft == nil { - return errors.WithStack(errNilEngine) - } - - if err := e.raft.LeadershipTransfer().Error(); err != nil { - if ctxErr := contextErr(ctx); ctxErr != nil { - return ctxErr - } - return errors.WithStack(err) - } - return nil -} - -func (e *Engine) TransferLeadershipToServer(ctx context.Context, id string, address string) error { - if err := contextErr(ctx); err != nil { - return err - } - if e == nil || e.raft == nil { - return errors.WithStack(errNilEngine) - } - - if err := e.raft.LeadershipTransferToServer(raft.ServerID(id), raft.ServerAddress(address)).Error(); err != nil { - if ctxErr := contextErr(ctx); ctxErr != nil { - return ctxErr - } - return errors.WithStack(err) - } - return nil -} - -func timeoutFromContext(ctx context.Context) (time.Duration, error) { - if ctx == nil { - return defaultProposalTimeout, nil - } - if err := contextErr(ctx); err != nil { - return 0, err - } - deadline, ok := ctx.Deadline() - if !ok { - return defaultProposalTimeout, nil - } - timeout := time.Until(deadline) - if timeout <= 0 { - return 0, errors.WithStack(context.DeadlineExceeded) - } - return timeout, nil -} - -func contextErr(ctx context.Context) error { - if ctx == nil { - return nil - } - if err := ctx.Err(); err != nil { - return errors.WithStack(err) - } - return nil -} - -func normalizeSuffrage(s raft.ServerSuffrage) string { - switch s { - case raft.Voter: - return "voter" - case raft.Nonvoter: - return "nonvoter" - case raft.Staging: - return "staging" - default: - return "unknown" - } -} - -func parseUint(raw string) uint64 { - v, err := strconv.ParseUint(strings.TrimSpace(raw), 10, 64) - if err != nil { - return 0 - } - return v -} - -var errTermParse = errors.New("failed to determine raft term from stats") - -func parseTermFromStats(stats map[string]string) (uint64, error) { - raw, ok := stats["term"] - if !ok || strings.TrimSpace(raw) == "" { - return 0, errors.WithStack(errTermParse) - } - term := parseUint(raw) - if term == 0 { - return 0, errors.WithStack(errTermParse) - } - return term, nil -} - -func lastContact(state raftengine.State, last time.Time) time.Duration { - if state == raftengine.StateLeader { - return 0 - } - if last.IsZero() { - return unknownLastContact - } - return time.Since(last) -} diff --git a/internal/raftengine/hashicorp/engine_test.go b/internal/raftengine/hashicorp/engine_test.go deleted file mode 100644 index d4d33b798..000000000 --- a/internal/raftengine/hashicorp/engine_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package hashicorp_test - -import ( - "os" - "path/filepath" - "testing" - "time" - - "github.com/bootjp/elastickv/internal/raftengine" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" - "github.com/bootjp/elastickv/internal/raftengine/raftenginetest" - "github.com/stretchr/testify/require" -) - -func TestConformance(t *testing.T) { - factory := hashicorpraftengine.NewFactory(hashicorpraftengine.FactoryConfig{ - CommitTimeout: 50 * time.Millisecond, - HeartbeatTimeout: 200 * time.Millisecond, - ElectionTimeout: 2000 * time.Millisecond, - LeaderLeaseTimeout: 100 * time.Millisecond, - SnapshotRetainCount: 3, - }) - raftenginetest.RunConformanceSuite(t, factory) -} - -func TestEngineCloseNilSafe(t *testing.T) { - t.Parallel() - - t.Run("nil_engine", func(t *testing.T) { - var e *hashicorpraftengine.Engine - require.NoError(t, e.Close()) - }) - - t.Run("nil_raft_via_New", func(t *testing.T) { - e := hashicorpraftengine.New(nil) - require.Nil(t, e) - }) -} - -func TestRejectLegacyBoltDB(t *testing.T) { - t.Parallel() - - factory := hashicorpraftengine.NewFactory(hashicorpraftengine.FactoryConfig{ - CommitTimeout: 50 * time.Millisecond, - HeartbeatTimeout: 200 * time.Millisecond, - ElectionTimeout: 2000 * time.Millisecond, - LeaderLeaseTimeout: 100 * time.Millisecond, - SnapshotRetainCount: 3, - }) - - for _, legacy := range []string{"logs.dat", "stable.dat"} { - t.Run(legacy, func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, legacy), []byte("legacy"), 0o600)) - - _, err := factory.Create(raftengine.FactoryConfig{ - LocalID: "n1", - LocalAddress: "127.0.0.1:0", - DataDir: dir, - Bootstrap: true, - StateMachine: &raftenginetest.TestStateMachine{}, - }) - require.Error(t, err) - require.Contains(t, err.Error(), "legacy boltdb") - }) - } -} diff --git a/internal/raftengine/hashicorp/factory.go b/internal/raftengine/hashicorp/factory.go deleted file mode 100644 index ee9d70e3a..000000000 --- a/internal/raftengine/hashicorp/factory.go +++ /dev/null @@ -1,206 +0,0 @@ -package hashicorp - -import ( - "fmt" - "io" - "os" - "path/filepath" - "time" - - transport "github.com/Jille/raft-grpc-transport" - internalutil "github.com/bootjp/elastickv/internal" - "github.com/bootjp/elastickv/internal/raftengine" - "github.com/bootjp/elastickv/internal/raftstore" - "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" -) - -const factoryDirPerm = 0o755 - -// FactoryConfig holds hashicorp-specific engine parameters. -type FactoryConfig struct { - CommitTimeout time.Duration - HeartbeatTimeout time.Duration - ElectionTimeout time.Duration - LeaderLeaseTimeout time.Duration - SnapshotRetainCount int -} - -// Factory creates hashicorp raft engine instances. -type Factory struct { - cfg FactoryConfig -} - -// NewFactory returns a Factory with the given hashicorp-specific settings. -func NewFactory(cfg FactoryConfig) *Factory { - return &Factory{cfg: cfg} -} - -func (f *Factory) EngineType() string { return "hashicorp" } - -func (f *Factory) Create(cfg raftengine.FactoryConfig) (*raftengine.FactoryResult, error) { - dir := cfg.DataDir - if err := os.MkdirAll(dir, factoryDirPerm); err != nil { - return nil, errors.WithStack(err) - } - - if err := rejectLegacyBoltDB(dir); err != nil { - return nil, err - } - - r, rs, tm, err := f.createRaft(dir, cfg) - if err != nil { - return nil, err - } - - cleanup := func() error { - return errors.CombineErrors( - closeIfNotNil(tm), - closeIfNotNil(rs), - ) - } - - if cfg.Bootstrap { - if err := bootstrapCluster(r, cfg); err != nil { - if cleanupErr := cleanup(); cleanupErr != nil { - fmt.Fprintf(os.Stderr, "warning: cleanup after bootstrap failure: %v\n", cleanupErr) - } - return nil, err - } - } - - return &raftengine.FactoryResult{ - Engine: New(r), - RegisterTransport: tm.Register, - Close: cleanup, - }, nil -} - -func rejectLegacyBoltDB(dir string) error { - for _, legacy := range []string{"logs.dat", "stable.dat"} { - if _, err := os.Stat(filepath.Join(dir, legacy)); err == nil { - return errors.WithStack(errors.Newf( - "legacy boltdb Raft storage %q found in %s; manual migration required before using Pebble-backed storage", - legacy, dir, - )) - } - } - return nil -} - -func (f *Factory) createRaft(dir string, cfg raftengine.FactoryConfig) (*raft.Raft, *raftstore.PebbleStore, *transport.Manager, error) { - rs, err := raftstore.NewPebbleStore(filepath.Join(dir, "raft.db")) - if err != nil { - return nil, nil, nil, errors.WithStack(err) - } - - fss, err := raft.NewFileSnapshotStore(dir, f.cfg.SnapshotRetainCount, os.Stderr) - if err != nil { - return nil, nil, nil, errors.WithStack(errors.CombineErrors(err, rs.Close())) - } - - tm := transport.New(raft.ServerAddress(cfg.LocalAddress), internalutil.GRPCDialOptions()) - - c := raft.DefaultConfig() - c.LocalID = raft.ServerID(cfg.LocalID) - c.CommitTimeout = f.cfg.CommitTimeout - c.HeartbeatTimeout = f.cfg.HeartbeatTimeout - c.ElectionTimeout = f.cfg.ElectionTimeout - c.LeaderLeaseTimeout = f.cfg.LeaderLeaseTimeout - - r, err := raft.NewRaft(c, adaptStateMachineToFSM(cfg.StateMachine), rs, rs, fss, tm.Transport()) - if err != nil { - return nil, nil, nil, errors.WithStack(errors.CombineErrors(err, errors.CombineErrors(tm.Close(), rs.Close()))) - } - return r, rs, tm, nil -} - -func bootstrapCluster(r *raft.Raft, cfg raftengine.FactoryConfig) error { - servers := peersToRaftServers(cfg) - raftCfg := raft.Configuration{Servers: servers} - if err := r.BootstrapCluster(raftCfg).Error(); err != nil { - if shutdownErr := r.Shutdown().Error(); shutdownErr != nil { - fmt.Fprintf(os.Stderr, "warning: raft shutdown after bootstrap failure: %v\n", shutdownErr) - } - return errors.WithStack(err) - } - return nil -} - -func peersToRaftServers(cfg raftengine.FactoryConfig) []raft.Server { - if len(cfg.Peers) == 0 { - return []raft.Server{ - { - Suffrage: raft.Voter, - ID: raft.ServerID(cfg.LocalID), - Address: raft.ServerAddress(cfg.LocalAddress), - }, - } - } - servers := make([]raft.Server, 0, len(cfg.Peers)) - for _, p := range cfg.Peers { - servers = append(servers, raft.Server{ - Suffrage: raft.Voter, - ID: raft.ServerID(p.ID), - Address: raft.ServerAddress(p.Address), - }) - } - return servers -} - -// adaptStateMachineToFSM converts an engine-agnostic StateMachine to -// hashicorp/raft's FSM interface. -func adaptStateMachineToFSM(sm raftengine.StateMachine) raft.FSM { - return &stateMachineFSMAdapter{sm: sm} -} - -type stateMachineFSMAdapter struct { - sm raftengine.StateMachine -} - -func (a *stateMachineFSMAdapter) Apply(log *raft.Log) interface{} { - return a.sm.Apply(log.Data) -} - -func (a *stateMachineFSMAdapter) Snapshot() (raft.FSMSnapshot, error) { - snap, err := a.sm.Snapshot() - if err != nil { - return nil, errors.WithStack(err) - } - return &snapshotFSMAdapter{snap: snap}, nil -} - -func (a *stateMachineFSMAdapter) Restore(r io.ReadCloser) error { - defer func() { - if err := r.Close(); err != nil { - fmt.Fprintf(os.Stderr, "warning: failed to close raft restore stream: %v\n", err) - } - }() - return errors.WithStack(a.sm.Restore(r)) -} - -type snapshotFSMAdapter struct { - snap raftengine.Snapshot -} - -func (a *snapshotFSMAdapter) Persist(sink raft.SnapshotSink) error { - if _, err := a.snap.WriteTo(sink); err != nil { - _ = sink.Cancel() - return errors.WithStack(err) - } - return errors.WithStack(sink.Close()) -} - -func (a *snapshotFSMAdapter) Release() { - if err := a.snap.Close(); err != nil { - fmt.Fprintf(os.Stderr, "warning: failed to close snapshot: %v\n", err) - } -} - -// closeIfNotNil closes c if it is not nil and returns the error. -func closeIfNotNil(c io.Closer) error { - if c == nil { - return nil - } - return errors.WithStack(c.Close()) -} diff --git a/internal/raftengine/hashicorp/leadership_err_test.go b/internal/raftengine/hashicorp/leadership_err_test.go deleted file mode 100644 index ada254fc7..000000000 --- a/internal/raftengine/hashicorp/leadership_err_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package hashicorp - -import ( - "testing" - - "github.com/bootjp/elastickv/internal/raftengine" - "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" - "github.com/stretchr/testify/require" -) - -// TestTranslateLeadershipErrMatchesRaftEngineSentinel pins the invariant -// that hashicorp/raft leadership-loss errors are marked against the -// shared raftengine sentinels. The lease-read fast path in package kv -// relies on a single cross-backend errors.Is(err, raftengine.ErrNotLeader) -// check; a future refactor that forgets to mark these errors would -// silently force every read onto the slow LinearizableRead path. -func TestTranslateLeadershipErrMatchesRaftEngineSentinel(t *testing.T) { - t.Parallel() - - cases := []struct { - name string - in error - want error - }{ - {"not leader", raft.ErrNotLeader, raftengine.ErrNotLeader}, - {"leadership lost", raft.ErrLeadershipLost, raftengine.ErrLeadershipLost}, - {"leadership transfer in progress", raft.ErrLeadershipTransferInProgress, raftengine.ErrLeadershipTransferInProgress}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - out := translateLeadershipErr(tc.in) - require.True(t, errors.Is(out, tc.want), - "translated error must errors.Is-match the raftengine sentinel") - require.True(t, errors.Is(out, tc.in), - "translated error must retain the original raft sentinel for debugging") - }) - } - - t.Run("unrelated error is passed through", func(t *testing.T) { - t.Parallel() - orig := errors.New("write conflict") - out := translateLeadershipErr(orig) - require.False(t, errors.Is(out, raftengine.ErrNotLeader)) - require.False(t, errors.Is(out, raftengine.ErrLeadershipLost)) - require.False(t, errors.Is(out, raftengine.ErrLeadershipTransferInProgress)) - }) - - t.Run("nil stays nil", func(t *testing.T) { - t.Parallel() - require.NoError(t, translateLeadershipErr(nil)) - }) -} diff --git a/internal/raftengine/hashicorp/migrate.go b/internal/raftengine/hashicorp/migrate.go deleted file mode 100644 index 07a03bf1c..000000000 --- a/internal/raftengine/hashicorp/migrate.go +++ /dev/null @@ -1,242 +0,0 @@ -package hashicorp - -import ( - "fmt" - "io" - "os" - "path/filepath" - "strings" - - "github.com/bootjp/elastickv/internal/raftstore" - "github.com/bootjp/elastickv/store" - "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" -) - -const ( - migrationTempSuffix = ".migrating" - peerSpecParts = 2 - migrationDirPerm = 0o755 -) - -// MigrationPeer represents a single node in the hashicorp raft cluster. -type MigrationPeer struct { - ID string - Address string -} - -// MigrationStats holds summary info about a completed migration. -type MigrationStats struct { - SnapshotBytes int64 - Peers int -} - -// ParsePeers parses a comma-separated "id=host:port" list into MigrationPeer -// values. The format matches the etcd migration tool for consistency. -func ParsePeers(raw string) ([]MigrationPeer, error) { - raw = strings.TrimSpace(raw) - if raw == "" { - return nil, nil - } - - parts := strings.Split(raw, ",") - peers := make([]MigrationPeer, 0, len(parts)) - for _, part := range parts { - part = strings.TrimSpace(part) - if part == "" { - continue - } - idAddr := strings.SplitN(part, "=", peerSpecParts) - if len(idAddr) != peerSpecParts { - return nil, errors.WithStack(errors.Newf("invalid peer format %q, expected id=host:port", part)) - } - id := strings.TrimSpace(idAddr[0]) - addr := strings.TrimSpace(idAddr[1]) - if id == "" || addr == "" { - return nil, errors.WithStack(errors.Newf("invalid peer format %q, id and address must be non-empty", part)) - } - peers = append(peers, MigrationPeer{ID: id, Address: addr}) - } - return peers, nil -} - -// MigrateFSMStore performs a reverse migration from etcd/raft to hashicorp -// raft. It reads an FSM PebbleStore snapshot and creates the directory -// structure that hashicorp/raft expects: a raft.db PebbleStore for log/stable -// state and a snapshots/ directory containing the FSM snapshot with peer -// configuration. -// -// The source FSM store (fsm.db) is read-only and shared between both engines; -// this function only creates the hashicorp-specific artifacts. -// -// IMPORTANT: The source engine must be fully stopped before running this -// tool. Running it against a live engine may produce an inconsistent -// snapshot that is missing recently applied entries. -func MigrateFSMStore(storePath string, destDataDir string, peers []MigrationPeer) (*MigrationStats, error) { - destDataDir, tempDir, err := prepareMigrationDest(storePath, destDataDir, peers) - if err != nil { - return nil, err - } - snapshotBytes, err := seedHashicorpDir(storePath, tempDir, peers) - if err != nil { - _ = os.RemoveAll(tempDir) - return nil, err - } - if err := finalizeMigrationDir(tempDir, destDataDir); err != nil { - return nil, err - } - return &MigrationStats{ - SnapshotBytes: snapshotBytes, - Peers: len(peers), - }, nil -} - -func prepareMigrationDest(storePath string, destDataDir string, peers []MigrationPeer) (string, string, error) { - switch { - case storePath == "": - return "", "", errors.WithStack(errors.New("source FSM store path is required")) - case destDataDir == "": - return "", "", errors.WithStack(errors.New("destination data dir is required")) - case len(peers) == 0: - return "", "", errors.WithStack(errors.New("at least one peer is required")) - } - - destDataDir = filepath.Clean(destDataDir) - if err := ensureMigrationPathAbsent(destDataDir, "destination"); err != nil { - return "", "", err - } - tempDir := destDataDir + migrationTempSuffix - if err := ensureMigrationPathAbsent(tempDir, "temporary destination"); err != nil { - return "", "", err - } - return destDataDir, tempDir, nil -} - -func ensureMigrationPathAbsent(path string, kind string) error { - if _, err := os.Stat(path); err == nil { - return errors.WithStack(errors.Newf("%s already exists: %s", kind, path)) - } else if !os.IsNotExist(err) { - return errors.WithStack(err) - } - return nil -} - -// seedHashicorpDir creates the hashicorp raft directory structure inside -// tempDir with a raft.db stable store and a snapshot containing the FSM data. -// The snapshot is streamed directly from the source PebbleStore to the -// raft.SnapshotSink to avoid buffering the entire FSM in memory. -func seedHashicorpDir(storePath string, tempDir string, peers []MigrationPeer) (int64, error) { - if err := os.MkdirAll(tempDir, migrationDirPerm); err != nil { - return 0, errors.WithStack(err) - } - - // Create the raft.db PebbleStore with initial stable state. - rs, err := raftstore.NewPebbleStore(filepath.Join(tempDir, "raft.db")) - if err != nil { - return 0, errors.WithStack(err) - } - defer func() { - if err := rs.Close(); err != nil { - fmt.Fprintf(os.Stderr, "warning: failed to close raft store: %v\n", err) - } - }() - - // Set initial term to 1. Hashicorp raft reads "CurrentTerm" on startup. - if err := rs.SetUint64([]byte("CurrentTerm"), 1); err != nil { - return 0, errors.WithStack(err) - } - - // Create the FileSnapshotStore to hold the FSM snapshot. - fss, err := raft.NewFileSnapshotStore(tempDir, 1, os.Stderr) - if err != nil { - return 0, errors.WithStack(err) - } - - // Build the raft configuration from peers. - configuration := raft.Configuration{Servers: peersToRaftMigrationServers(peers)} - - // Create an in-memory transport (required by FileSnapshotStore.Create but - // not used for actual communication during migration). - _, transport := raft.NewInmemTransport("") - defer transport.Close() - - // Create a snapshot sink at index=1, term=1. - sink, err := fss.Create(raft.SnapshotVersionMax, 1, 1, configuration, 1, transport) - if err != nil { - return 0, errors.WithStack(err) - } - - // Stream FSM snapshot directly to the sink (no in-memory buffering). - snapshotBytes, err := streamFSMSnapshotToSink(storePath, sink) - if err != nil { - _ = sink.Cancel() - return 0, err - } - - if err := sink.Close(); err != nil { - return 0, errors.WithStack(err) - } - - fmt.Fprintf(os.Stderr, " created snapshot with %d bytes of FSM data\n", snapshotBytes) - return snapshotBytes, nil -} - -// streamFSMSnapshotToSink opens the source PebbleStore, takes a snapshot, -// and streams it directly to the given io.Writer (typically a raft.SnapshotSink). -func streamFSMSnapshotToSink(storePath string, w io.Writer) (int64, error) { - source, err := store.NewPebbleStore(storePath) - if err != nil { - return 0, errors.WithStack(err) - } - defer func() { - if err := source.Close(); err != nil { - fmt.Fprintf(os.Stderr, "warning: failed to close source store: %v\n", err) - } - }() - - snapshot, err := source.Snapshot() - if err != nil { - return 0, errors.WithStack(err) - } - defer snapshot.Close() - - n, err := snapshot.WriteTo(w) - if err != nil { - return n, errors.WithStack(err) - } - return n, nil -} - -func peersToRaftMigrationServers(peers []MigrationPeer) []raft.Server { - servers := make([]raft.Server, 0, len(peers)) - for _, p := range peers { - servers = append(servers, raft.Server{ - Suffrage: raft.Voter, - ID: raft.ServerID(p.ID), - Address: raft.ServerAddress(p.Address), - }) - } - return servers -} - -func finalizeMigrationDir(tempDir string, destDataDir string) error { - if err := os.Rename(tempDir, destDataDir); err != nil { - _ = os.RemoveAll(tempDir) - return errors.WithStack(err) - } - if err := syncDir(filepath.Dir(destDataDir)); err != nil { - // Don't remove destDataDir here — the rename succeeded and the data - // is already in place. Deleting it would cause total data loss. - return err - } - return nil -} - -func syncDir(path string) error { - f, err := os.Open(path) - if err != nil { - return errors.WithStack(err) - } - defer f.Close() - return errors.WithStack(f.Sync()) -} diff --git a/internal/raftengine/hashicorp/migrate_test.go b/internal/raftengine/hashicorp/migrate_test.go deleted file mode 100644 index c402bf412..000000000 --- a/internal/raftengine/hashicorp/migrate_test.go +++ /dev/null @@ -1,198 +0,0 @@ -package hashicorp_test - -import ( - "bytes" - "context" - "io" - "path/filepath" - "testing" - "time" - - "github.com/bootjp/elastickv/internal/raftengine" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" - "github.com/bootjp/elastickv/store" - "github.com/stretchr/testify/require" -) - -func TestParsePeers(t *testing.T) { - t.Parallel() - - t.Run("empty", func(t *testing.T) { - peers, err := hashicorpraftengine.ParsePeers("") - require.NoError(t, err) - require.Nil(t, peers) - }) - - t.Run("single", func(t *testing.T) { - peers, err := hashicorpraftengine.ParsePeers("n1=127.0.0.1:7001") - require.NoError(t, err) - require.Len(t, peers, 1) - require.Equal(t, "n1", peers[0].ID) - require.Equal(t, "127.0.0.1:7001", peers[0].Address) - }) - - t.Run("multiple", func(t *testing.T) { - peers, err := hashicorpraftengine.ParsePeers("n1=127.0.0.1:7001,n2=127.0.0.1:7002,n3=127.0.0.1:7003") - require.NoError(t, err) - require.Len(t, peers, 3) - }) - - t.Run("invalid_format", func(t *testing.T) { - _, err := hashicorpraftengine.ParsePeers("bad-entry") - require.Error(t, err) - }) - - t.Run("empty_id", func(t *testing.T) { - _, err := hashicorpraftengine.ParsePeers("=127.0.0.1:7001") - require.Error(t, err) - }) - - t.Run("empty_address", func(t *testing.T) { - _, err := hashicorpraftengine.ParsePeers("n1=") - require.Error(t, err) - }) - - t.Run("trailing_commas_skipped", func(t *testing.T) { - peers, err := hashicorpraftengine.ParsePeers("n1=127.0.0.1:7001,,") - require.NoError(t, err) - require.Len(t, peers, 1) - }) -} - -func TestMigrateFSMStoreValidation(t *testing.T) { - t.Parallel() - - t.Run("missing_source", func(t *testing.T) { - _, err := hashicorpraftengine.MigrateFSMStore("", t.TempDir(), []hashicorpraftengine.MigrationPeer{{ID: "n1", Address: "127.0.0.1:7001"}}) - require.Error(t, err) - require.Contains(t, err.Error(), "source FSM store path is required") - }) - - t.Run("missing_dest", func(t *testing.T) { - _, err := hashicorpraftengine.MigrateFSMStore("/some/path", "", []hashicorpraftengine.MigrationPeer{{ID: "n1", Address: "127.0.0.1:7001"}}) - require.Error(t, err) - require.Contains(t, err.Error(), "destination data dir is required") - }) - - t.Run("missing_peers", func(t *testing.T) { - _, err := hashicorpraftengine.MigrateFSMStore("/some/path", t.TempDir()+"/dest", nil) - require.Error(t, err) - require.Contains(t, err.Error(), "at least one peer is required") - }) - - t.Run("dest_already_exists", func(t *testing.T) { - dest := t.TempDir() // already exists - peers := []hashicorpraftengine.MigrationPeer{{ID: "n1", Address: "127.0.0.1:7001"}} - _, err := hashicorpraftengine.MigrateFSMStore("/some/path", dest, peers) - require.Error(t, err) - require.Contains(t, err.Error(), "already exists") - }) -} - -func TestMigrateFSMStoreSeedsHashicorpDataDir(t *testing.T) { - // Write test data to a PebbleStore (the shared FSM store). - sourcePath := filepath.Join(t.TempDir(), "fsm.db") - source, err := store.NewPebbleStore(sourcePath) - require.NoError(t, err) - require.NoError(t, source.PutAt(context.Background(), []byte("alpha"), []byte("one"), 10, 0)) - require.NoError(t, source.PutAt(context.Background(), []byte("beta"), []byte("two"), 11, 0)) - require.NoError(t, source.Close()) - - // Migrate to hashicorp raft format. - destDataDir := filepath.Join(t.TempDir(), "raft") - peers := []hashicorpraftengine.MigrationPeer{ - {ID: "n1", Address: "127.0.0.1:7001"}, - } - stats, err := hashicorpraftengine.MigrateFSMStore(sourcePath, destDataDir, peers) - require.NoError(t, err) - require.Positive(t, stats.SnapshotBytes) - require.Equal(t, 1, stats.Peers) - - // Boot a hashicorp raft engine from the migrated directory and verify - // the FSM snapshot is restored. - destStore, err := store.NewPebbleStore(filepath.Join(t.TempDir(), "dest-fsm.db")) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, destStore.Close()) }) - - sm := &storeStateMachine{store: destStore} - factory := hashicorpraftengine.NewFactory(hashicorpraftengine.FactoryConfig{ - CommitTimeout: 50 * time.Millisecond, - HeartbeatTimeout: 200 * time.Millisecond, - ElectionTimeout: 2000 * time.Millisecond, - LeaderLeaseTimeout: 100 * time.Millisecond, - SnapshotRetainCount: 3, - }) - - result, err := factory.Create(raftengine.FactoryConfig{ - LocalID: "n1", - LocalAddress: "127.0.0.1:0", - DataDir: destDataDir, - StateMachine: sm, - }) - require.NoError(t, err) - t.Cleanup(func() { - _ = result.Engine.Close() - if result.Close != nil { - _ = result.Close() - } - }) - - // Wait for leader election. - require.Eventually(t, func() bool { - return result.Engine.State() == raftengine.StateLeader - }, 5*time.Second, 10*time.Millisecond, "engine did not become leader") - - // Verify FSM data survived the migration. - v, err := destStore.GetAt(context.Background(), []byte("alpha"), ^uint64(0)) - require.NoError(t, err) - require.Equal(t, []byte("one"), v) - - v, err = destStore.GetAt(context.Background(), []byte("beta"), ^uint64(0)) - require.NoError(t, err) - require.Equal(t, []byte("two"), v) -} - -// storeStateMachine adapts store.MVCCStore to raftengine.StateMachine for -// migration tests. Only Snapshot/Restore are needed; Apply is unused. -type storeStateMachine struct { - store store.MVCCStore -} - -func (s *storeStateMachine) Apply(_ []byte) any { return nil } - -func (s *storeStateMachine) Snapshot() (raftengine.Snapshot, error) { - return s.store.Snapshot() -} - -func (s *storeStateMachine) Restore(r io.Reader) error { - return s.store.Restore(r) -} - -func TestFSMSnapshotRoundTrip(t *testing.T) { - t.Parallel() - - // Create source store with data. - sourcePath := filepath.Join(t.TempDir(), "src.db") - src, err := store.NewPebbleStore(sourcePath) - require.NoError(t, err) - require.NoError(t, src.PutAt(context.Background(), []byte("k"), []byte("v"), 1, 0)) - - snap, err := src.Snapshot() - require.NoError(t, err) - var buf bytes.Buffer - _, err = snap.WriteTo(&buf) - require.NoError(t, err) - require.NoError(t, snap.Close()) - require.NoError(t, src.Close()) - - // Restore into destination store. - dstPath := filepath.Join(t.TempDir(), "dst.db") - dst, err := store.NewPebbleStore(dstPath) - require.NoError(t, err) - require.NoError(t, dst.Restore(&buf)) - - v, err := dst.GetAt(context.Background(), []byte("k"), ^uint64(0)) - require.NoError(t, err) - require.Equal(t, []byte("v"), v) - require.NoError(t, dst.Close()) -} diff --git a/internal/raftstore/pebble.go b/internal/raftstore/pebble.go deleted file mode 100644 index 58bd90b20..000000000 --- a/internal/raftstore/pebble.go +++ /dev/null @@ -1,210 +0,0 @@ -package raftstore - -import ( - "encoding/binary" - "io" - "os" - - "github.com/cockroachdb/errors" - "github.com/cockroachdb/pebble/v2" - "github.com/hashicorp/raft" - "github.com/vmihailenco/msgpack/v5" -) - -const ( - logPrefix byte = 'l' - stablePrefix byte = 's' - pebbleDirPerm = 0o755 - pebbleUint64Bytes = 8 - pebbleLogKeyBytes = 1 + pebbleUint64Bytes -) - -var _ raft.LogStore = (*PebbleStore)(nil) -var _ raft.StableStore = (*PebbleStore)(nil) -var _ io.Closer = (*PebbleStore)(nil) - -type PebbleStore struct { - db *pebble.DB -} - -func NewPebbleStore(dir string) (*PebbleStore, error) { - if err := os.MkdirAll(dir, pebbleDirPerm); err != nil { - return nil, errors.WithStack(err) - } - - db, err := pebble.Open(dir, pebbleOptions()) - if err != nil { - return nil, errors.WithStack(err) - } - return &PebbleStore{db: db}, nil -} - -// Pinned so existing v1-era DBs are ratcheted above pebble v2's -// FormatMinSupported (FormatFlushableIngest) before the v2 upgrade lands. -func pebbleOptions() *pebble.Options { - opts := &pebble.Options{ - FormatMajorVersion: pebble.FormatVirtualSSTables, - } - opts.EnsureDefaults() - return opts -} - -func (s *PebbleStore) FirstIndex() (uint64, error) { - iter, err := s.db.NewIter(logIterOptions()) - if err != nil { - return 0, errors.WithStack(err) - } - defer func() { _ = iter.Close() }() - - if !iter.First() { - if err := iter.Error(); err != nil { - return 0, errors.WithStack(err) - } - return 0, nil - } - return decodeLogIndex(iter.Key()), nil -} - -func (s *PebbleStore) LastIndex() (uint64, error) { - iter, err := s.db.NewIter(logIterOptions()) - if err != nil { - return 0, errors.WithStack(err) - } - defer func() { _ = iter.Close() }() - - if !iter.Last() { - if err := iter.Error(); err != nil { - return 0, errors.WithStack(err) - } - return 0, nil - } - return decodeLogIndex(iter.Key()), nil -} - -func (s *PebbleStore) GetLog(index uint64, out *raft.Log) error { - value, closer, err := s.db.Get(logKey(index)) - if errors.Is(err, pebble.ErrNotFound) { - return raft.ErrLogNotFound - } - if err != nil { - return errors.WithStack(err) - } - defer func() { _ = closer.Close() }() - - return errors.WithStack(msgpack.Unmarshal(value, out)) -} - -func (s *PebbleStore) StoreLog(log *raft.Log) error { - return errors.WithStack(s.StoreLogs([]*raft.Log{log})) -} - -func (s *PebbleStore) StoreLogs(logs []*raft.Log) error { - if len(logs) == 0 { - return nil - } - - batch := s.db.NewBatch() - defer func() { _ = batch.Close() }() - - for _, logEntry := range logs { - if logEntry == nil { - continue - } - payload, err := msgpack.Marshal(logEntry) - if err != nil { - return errors.WithStack(err) - } - if err := batch.Set(logKey(logEntry.Index), payload, pebble.NoSync); err != nil { - return errors.WithStack(err) - } - } - - return errors.WithStack(batch.Commit(pebble.Sync)) -} - -func (s *PebbleStore) DeleteRange(min, max uint64) error { - if max < min { - return nil - } - - batch := s.db.NewBatch() - defer func() { _ = batch.Close() }() - - end := []byte{logPrefix + 1} - if max < ^uint64(0) { - end = logKey(max + 1) - } - if err := batch.DeleteRange(logKey(min), end, pebble.NoSync); err != nil { - return errors.WithStack(err) - } - return errors.WithStack(batch.Commit(pebble.Sync)) -} - -func (s *PebbleStore) Set(key []byte, value []byte) error { - return errors.WithStack(s.db.Set(stableKey(key), value, pebble.Sync)) -} - -func (s *PebbleStore) Get(key []byte) ([]byte, error) { - value, closer, err := s.db.Get(stableKey(key)) - if errors.Is(err, pebble.ErrNotFound) { - return nil, nil - } - if err != nil { - return nil, errors.WithStack(err) - } - defer func() { _ = closer.Close() }() - - return append([]byte(nil), value...), nil -} - -func (s *PebbleStore) SetUint64(key []byte, value uint64) error { - buf := make([]byte, pebbleUint64Bytes) - binary.BigEndian.PutUint64(buf, value) - return errors.WithStack(s.Set(key, buf)) -} - -func (s *PebbleStore) GetUint64(key []byte) (uint64, error) { - value, err := s.Get(key) - if err != nil || len(value) == 0 { - return 0, err - } - if len(value) != pebbleUint64Bytes { - return 0, errors.WithStack(errors.Newf("invalid uint64 value length: %d", len(value))) - } - return binary.BigEndian.Uint64(value), nil -} - -func (s *PebbleStore) Close() error { - if s == nil || s.db == nil { - return nil - } - return errors.WithStack(s.db.Close()) -} - -func logKey(index uint64) []byte { - key := make([]byte, pebbleLogKeyBytes) - key[0] = logPrefix - binary.BigEndian.PutUint64(key[1:], index) - return key -} - -func stableKey(key []byte) []byte { - out := make([]byte, 1+len(key)) - out[0] = stablePrefix - copy(out[1:], key) - return out -} - -func decodeLogIndex(key []byte) uint64 { - if len(key) != pebbleLogKeyBytes || key[0] != logPrefix { - return 0 - } - return binary.BigEndian.Uint64(key[1:]) -} - -func logIterOptions() *pebble.IterOptions { - return &pebble.IterOptions{ - LowerBound: []byte{logPrefix}, - UpperBound: []byte{logPrefix + 1}, - } -} diff --git a/internal/raftstore/pebble_test.go b/internal/raftstore/pebble_test.go deleted file mode 100644 index f6a1db89a..000000000 --- a/internal/raftstore/pebble_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package raftstore - -import ( - "testing" - - "github.com/cockroachdb/pebble/v2" - "github.com/hashicorp/raft" - "github.com/stretchr/testify/require" -) - -// Pins the on-open format so unintended downgrades below pebble v2 -// FormatMinSupported are caught early. -func TestPebbleStore_OpensAtPinnedFormat(t *testing.T) { - dir := t.TempDir() - - s, err := NewPebbleStore(dir) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, s.Close()) }) - - require.Equal(t, pebble.FormatVirtualSSTables, s.db.FormatMajorVersion()) -} - -// End-to-end check that StoreLog/GetLog round-trip across a close/reopen -// against pebble v2. -func TestPebbleStore_PersistsLogsAcrossReopen(t *testing.T) { - dir := t.TempDir() - - s1, err := NewPebbleStore(dir) - require.NoError(t, err) - require.NoError(t, s1.StoreLog(&raft.Log{Index: 1, Term: 1, Data: []byte("hello")})) - require.NoError(t, s1.StoreLog(&raft.Log{Index: 2, Term: 1, Data: []byte("world")})) - require.NoError(t, s1.Close()) - - s2, err := NewPebbleStore(dir) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, s2.Close()) }) - - var got raft.Log - require.NoError(t, s2.GetLog(1, &got)) - require.Equal(t, []byte("hello"), got.Data) - require.NoError(t, s2.GetLog(2, &got)) - require.Equal(t, []byte("world"), got.Data) - - last, err := s2.LastIndex() - require.NoError(t, err) - require.Equal(t, uint64(2), last) -} diff --git a/kv/coordinator.go b/kv/coordinator.go index 372900ca6..de4264c4e 100644 --- a/kv/coordinator.go +++ b/kv/coordinator.go @@ -9,10 +9,8 @@ import ( "time" "github.com/bootjp/elastickv/internal/raftengine" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" ) const redirectForwardTimeout = 5 * time.Second @@ -93,10 +91,6 @@ func normalizeLeaseObserver(observer LeaseReadObserver) LeaseReadObserver { return observer } -func NewCoordinator(txm Transactional, r *raft.Raft, opts ...CoordinatorOption) *Coordinate { - return NewCoordinatorWithEngine(txm, hashicorpraftengine.New(r), opts...) -} - func NewCoordinatorWithEngine(txm Transactional, engine raftengine.Engine, opts ...CoordinatorOption) *Coordinate { c := &Coordinate{ transactionManager: txm, @@ -180,10 +174,10 @@ type Coordinator interface { IsLeader() bool VerifyLeader() error LinearizableRead(ctx context.Context) (uint64, error) - RaftLeader() raft.ServerAddress + RaftLeader() string IsLeaderForKey(key []byte) bool VerifyLeaderForKey(key []byte) error - RaftLeaderForKey(key []byte) raft.ServerAddress + RaftLeaderForKey(key []byte) string Clock() *HLC } @@ -325,7 +319,7 @@ func (c *Coordinate) VerifyLeader() error { } // RaftLeader returns the current leader's address as known by this node. -func (c *Coordinate) RaftLeader() raft.ServerAddress { +func (c *Coordinate) RaftLeader() string { return leaderAddrFromEngine(c.engine) } @@ -384,7 +378,7 @@ func (c *Coordinate) VerifyLeaderForKey(_ []byte) error { return c.VerifyLeader() } -func (c *Coordinate) RaftLeaderForKey(_ []byte) raft.ServerAddress { +func (c *Coordinate) RaftLeaderForKey(_ []byte) string { return c.RaftLeader() } diff --git a/kv/coordinator_dispatch_test.go b/kv/coordinator_dispatch_test.go index 14d73ee95..8be3950fd 100644 --- a/kv/coordinator_dispatch_test.go +++ b/kv/coordinator_dispatch_test.go @@ -4,7 +4,6 @@ import ( "context" "testing" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" ) @@ -17,8 +16,8 @@ func TestCoordinateDispatch_RawPut(t *testing.T) { r, stop := newSingleRaft(t, "dispatch-raw-put", fsm) t.Cleanup(stop) - tm := NewTransaction(r) - c := NewCoordinator(tm, r) + tm := NewTransactionWithProposer(r) + c := NewCoordinatorWithEngine(tm, r) ctx := context.Background() resp, err := c.Dispatch(ctx, &OperationGroup[OP]{ @@ -43,8 +42,8 @@ func TestCoordinateDispatch_RawDel(t *testing.T) { r, stop := newSingleRaft(t, "dispatch-raw-del", fsm) t.Cleanup(stop) - tm := NewTransaction(r) - c := NewCoordinator(tm, r) + tm := NewTransactionWithProposer(r) + c := NewCoordinatorWithEngine(tm, r) ctx := context.Background() // Write a value first. @@ -76,8 +75,8 @@ func TestCoordinateDispatch_TxnOnePhase(t *testing.T) { r, stop := newSingleRaft(t, "dispatch-txn", fsm) t.Cleanup(stop) - tm := NewTransaction(r) - c := NewCoordinator(tm, r) + tm := NewTransactionWithProposer(r) + c := NewCoordinatorWithEngine(tm, r) ctx := context.Background() startTS := c.clock.Next() @@ -135,7 +134,7 @@ func TestCoordinateDispatch_TxnAssignsStartTS(t *testing.T) { c := &Coordinate{ transactionManager: tx, - engine: hashicorpraftengine.New(r), + engine: r, clock: NewHLC(), } @@ -167,7 +166,7 @@ func TestCoordinateDispatchRaw_CallsTransactionManager(t *testing.T) { c := &Coordinate{ transactionManager: tx, - engine: hashicorpraftengine.New(r), + engine: r, clock: NewHLC(), } diff --git a/kv/coordinator_leader_test.go b/kv/coordinator_leader_test.go index 07932753f..1df197a1d 100644 --- a/kv/coordinator_leader_test.go +++ b/kv/coordinator_leader_test.go @@ -14,7 +14,7 @@ func TestCoordinateVerifyLeader_LeaderReturnsNil(t *testing.T) { r, stop := newSingleRaft(t, "coord-leader", NewKvFSMWithHLC(st, NewHLC())) t.Cleanup(stop) - c := NewCoordinator(&stubTransactional{}, r) + c := NewCoordinatorWithEngine(&stubTransactional{}, r) require.NoError(t, c.VerifyLeader()) } diff --git a/kv/fsm.go b/kv/fsm.go index 532396224..772e0607f 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -8,10 +8,10 @@ import ( "log/slog" "os" + "github.com/bootjp/elastickv/internal/raftengine" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" "google.golang.org/protobuf/proto" ) @@ -32,7 +32,7 @@ type kvFSM struct { } type FSM interface { - raft.FSM + raftengine.StateMachine } // NewKvFSMWithHLC creates a KV FSM that updates hlc.physicalCeiling whenever @@ -49,7 +49,7 @@ func NewKvFSMWithHLC(store store.MVCCStore, hlc *HLC) FSM { } var _ FSM = (*kvFSM)(nil) -var _ raft.FSM = (*kvFSM)(nil) +var _ raftengine.StateMachine = (*kvFSM)(nil) var ErrUnknownRequestType = errors.New("unknown request type") @@ -57,16 +57,16 @@ type fsmApplyResponse struct { results []error } -func (f *kvFSM) Apply(l *raft.Log) any { +func (f *kvFSM) Apply(data []byte) any { // HLC lease entries advance only the physical ceiling; they do not touch // the MVCC store. The logical counter continues to be managed in memory. - if len(l.Data) > 0 && l.Data[0] == raftEncodeHLCLease { - return f.applyHLCLease(l.Data[1:]) + if len(data) > 0 && data[0] == raftEncodeHLCLease { + return f.applyHLCLease(data[1:]) } ctx := context.TODO() - reqs, err := decodeRaftRequests(l.Data) + reqs, err := decodeRaftRequests(data) if err != nil { return errors.WithStack(err) } @@ -249,7 +249,7 @@ func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uin var ErrNotImplemented = errors.New("not implemented") -func (f *kvFSM) Snapshot() (raft.FSMSnapshot, error) { +func (f *kvFSM) Snapshot() (raftengine.Snapshot, error) { snapshot, err := f.store.Snapshot() if err != nil { return nil, errors.WithStack(err) @@ -261,9 +261,7 @@ func (f *kvFSM) Snapshot() (raft.FSMSnapshot, error) { }, nil } -func (f *kvFSM) Restore(r io.ReadCloser) error { - defer r.Close() - +func (f *kvFSM) Restore(r io.Reader) error { // Read the potential 16-byte header (magic + ceiling ms). var hdr [hlcSnapshotHeaderLen]byte n, err := io.ReadFull(r, hdr[:]) diff --git a/kv/fsm_occ_test.go b/kv/fsm_occ_test.go index b0bb97a5b..f6d1fdd6c 100644 --- a/kv/fsm_occ_test.go +++ b/kv/fsm_occ_test.go @@ -6,7 +6,6 @@ import ( pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" - "github.com/hashicorp/raft" "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" ) @@ -44,7 +43,7 @@ func TestApplyReturnsErrorOnConflict(t *testing.T) { data, err := proto.Marshal(put) require.NoError(t, err) - resp := fsm.Apply(&raft.Log{Type: raft.LogCommand, Data: data}) + resp := fsm.Apply(data) require.Nil(t, resp) // Stale transaction attempts to prewrite with startTS=90. @@ -60,7 +59,7 @@ func TestApplyReturnsErrorOnConflict(t *testing.T) { data, err = proto.Marshal(conflict) require.NoError(t, err) - resp = fsm.Apply(&raft.Log{Type: raft.LogCommand, Data: data}) + resp = fsm.Apply(data) err, ok = resp.(error) require.True(t, ok) @@ -93,7 +92,7 @@ func TestOnePhaseTxnDetectsWriteConflict(t *testing.T) { data, err := proto.Marshal(req) require.NoError(t, err) - resp := fsm.Apply(&raft.Log{Type: raft.LogCommand, Data: data}) + resp := fsm.Apply(data) err, ok = resp.(error) require.True(t, ok) require.ErrorIs(t, err, store.ErrWriteConflict) diff --git a/kv/fsm_txn_test.go b/kv/fsm_txn_test.go index a027ffe87..1a6a4a808 100644 --- a/kv/fsm_txn_test.go +++ b/kv/fsm_txn_test.go @@ -7,7 +7,6 @@ import ( pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" - "github.com/hashicorp/raft" "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" ) @@ -18,7 +17,7 @@ func applyFSMRequest(t *testing.T, fsm *kvFSM, req *pb.Request) error { data, err := proto.Marshal(req) require.NoError(t, err) - resp := fsm.Apply(&raft.Log{Type: raft.LogCommand, Data: data}) + resp := fsm.Apply(data) if resp == nil { return nil } diff --git a/kv/grpc_conn_cache.go b/kv/grpc_conn_cache.go index 819806f4e..e42bf0bcd 100644 --- a/kv/grpc_conn_cache.go +++ b/kv/grpc_conn_cache.go @@ -5,7 +5,6 @@ import ( internalutil "github.com/bootjp/elastickv/internal" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" ) @@ -15,15 +14,15 @@ import ( // already been closed (Shutdown). type GRPCConnCache struct { mu sync.Mutex - conns map[raft.ServerAddress]*grpc.ClientConn + conns map[string]*grpc.ClientConn } -func (c *GRPCConnCache) cachedConn(addr raft.ServerAddress) *grpc.ClientConn { +func (c *GRPCConnCache) cachedConn(addr string) *grpc.ClientConn { c.mu.Lock() defer c.mu.Unlock() if c.conns == nil { - c.conns = make(map[raft.ServerAddress]*grpc.ClientConn) + c.conns = make(map[string]*grpc.ClientConn) } conn, ok := c.conns[addr] @@ -42,12 +41,12 @@ func (c *GRPCConnCache) cachedConn(addr raft.ServerAddress) *grpc.ClientConn { return conn } -func (c *GRPCConnCache) storeConn(addr raft.ServerAddress, conn *grpc.ClientConn) *grpc.ClientConn { +func (c *GRPCConnCache) storeConn(addr string, conn *grpc.ClientConn) *grpc.ClientConn { c.mu.Lock() defer c.mu.Unlock() if c.conns == nil { - c.conns = make(map[raft.ServerAddress]*grpc.ClientConn) + c.conns = make(map[string]*grpc.ClientConn) } existing, ok := c.conns[addr] @@ -66,7 +65,7 @@ func (c *GRPCConnCache) storeConn(addr raft.ServerAddress, conn *grpc.ClientConn return conn } -func (c *GRPCConnCache) ConnFor(addr raft.ServerAddress) (*grpc.ClientConn, error) { +func (c *GRPCConnCache) ConnFor(addr string) (*grpc.ClientConn, error) { if addr == "" { return nil, errors.WithStack(ErrLeaderNotFound) } @@ -76,7 +75,7 @@ func (c *GRPCConnCache) ConnFor(addr raft.ServerAddress) (*grpc.ClientConn, erro } conn, err := grpc.NewClient( - string(addr), + addr, append(internalutil.GRPCDialOptions(), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)))..., ) if err != nil { diff --git a/kv/grpc_conn_cache_test.go b/kv/grpc_conn_cache_test.go index 04fee8a33..89fb27ee5 100644 --- a/kv/grpc_conn_cache_test.go +++ b/kv/grpc_conn_cache_test.go @@ -7,7 +7,6 @@ import ( "testing" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" "github.com/stretchr/testify/require" "google.golang.org/grpc" ) @@ -38,7 +37,7 @@ func TestGRPCConnCache_ReusesConnection(t *testing.T) { _ = lis.Close() }) - addr := raft.ServerAddress(lis.Addr().String()) + addr := lis.Addr().String() var c GRPCConnCache conn1, err := c.ConnFor(addr) @@ -72,7 +71,7 @@ func TestGRPCConnCache_ConcurrentConnFor(t *testing.T) { _ = lis.Close() }) - addr := raft.ServerAddress(lis.Addr().String()) + addr := lis.Addr().String() var c GRPCConnCache diff --git a/kv/leader_proxy.go b/kv/leader_proxy.go index 93cfc58d7..3e675cd5c 100644 --- a/kv/leader_proxy.go +++ b/kv/leader_proxy.go @@ -6,10 +6,8 @@ import ( "time" "github.com/bootjp/elastickv/internal/raftengine" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" ) const leaderForwardTimeout = 5 * time.Second @@ -24,11 +22,6 @@ type LeaderProxy struct { connCache GRPCConnCache } -// NewLeaderProxy creates a leader-aware transactional proxy for a raft group. -func NewLeaderProxy(r *raft.Raft, opts ...TransactionOption) *LeaderProxy { - return NewLeaderProxyWithEngine(hashicorpraftengine.New(r), opts...) -} - func NewLeaderProxyWithEngine(engine raftengine.Engine, opts ...TransactionOption) *LeaderProxy { return &LeaderProxy{ engine: engine, diff --git a/kv/leader_proxy_test.go b/kv/leader_proxy_test.go index 3bd5eeea7..4a16fb09c 100644 --- a/kv/leader_proxy_test.go +++ b/kv/leader_proxy_test.go @@ -7,9 +7,9 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/raftengine" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" - "github.com/hashicorp/raft" "github.com/stretchr/testify/require" "google.golang.org/grpc" ) @@ -34,6 +34,33 @@ func (f *fakeInternal) Forward(_ context.Context, req *pb.ForwardRequest) (*pb.F return &pb.ForwardResponse{Success: true, CommitIndex: 0}, nil } +// stubFollowerEngine is a minimal raftengine.Engine stub that reports the +// local node as a follower and returns a configured leader address. It is +// used by TestLeaderProxy_ForwardsWhenFollower to exercise the forwarding +// code path without running a real two-node raft cluster. +type stubFollowerEngine struct { + leaderAddr string +} + +func (s *stubFollowerEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) { + return nil, raftengine.ErrNotLeader +} +func (s *stubFollowerEngine) State() raftengine.State { return raftengine.StateFollower } +func (s *stubFollowerEngine) Leader() raftengine.LeaderInfo { + return raftengine.LeaderInfo{ID: "leader", Address: s.leaderAddr} +} +func (s *stubFollowerEngine) VerifyLeader(context.Context) error { return raftengine.ErrNotLeader } +func (s *stubFollowerEngine) LinearizableRead(context.Context) (uint64, error) { + return 0, raftengine.ErrNotLeader +} +func (s *stubFollowerEngine) Status() raftengine.Status { + return raftengine.Status{State: raftengine.StateFollower, Leader: s.Leader()} +} +func (s *stubFollowerEngine) Configuration(context.Context) (raftengine.Configuration, error) { + return raftengine.Configuration{}, nil +} +func (s *stubFollowerEngine) Close() error { return nil } + func TestLeaderProxy_CommitLocalWhenLeader(t *testing.T) { t.Parallel() @@ -41,7 +68,7 @@ func TestLeaderProxy_CommitLocalWhenLeader(t *testing.T) { r, stop := newSingleRaft(t, "lp-local", NewKvFSMWithHLC(st, NewHLC())) defer stop() - p := NewLeaderProxy(r) + p := NewLeaderProxyWithEngine(r) reqs := []*pb.Request{ { @@ -79,60 +106,18 @@ func TestLeaderProxy_ForwardsWhenFollower(t *testing.T) { _ = lis.Close() }) - leaderAddr, leaderTrans := raft.NewInmemTransport(raft.ServerAddress(lis.Addr().String())) - followerAddr, followerTrans := raft.NewInmemTransport("follower") - leaderTrans.Connect(followerAddr, followerTrans) - followerTrans.Connect(leaderAddr, leaderTrans) - - raftCfg := raft.Configuration{ - Servers: []raft.Server{ - {Suffrage: raft.Voter, ID: "leader", Address: leaderAddr}, - {Suffrage: raft.Voter, ID: "follower", Address: followerAddr}, - }, - } - - leader := func() *raft.Raft { - c := raft.DefaultConfig() - c.LocalID = "leader" - c.HeartbeatTimeout = 50 * time.Millisecond - c.ElectionTimeout = 100 * time.Millisecond - c.LeaderLeaseTimeout = 50 * time.Millisecond - - ldb := raft.NewInmemStore() - sdb := raft.NewInmemStore() - fss := raft.NewInmemSnapshotStore() - r, err := raft.NewRaft(c, NewKvFSMWithHLC(store.NewMVCCStore(), NewHLC()), ldb, sdb, fss, leaderTrans) - require.NoError(t, err) - require.NoError(t, r.BootstrapCluster(raftCfg).Error()) - t.Cleanup(func() { _ = r.Shutdown().Error() }) - return r - }() - - follower := func() *raft.Raft { - c := raft.DefaultConfig() - c.LocalID = "follower" - c.HeartbeatTimeout = 250 * time.Millisecond - c.ElectionTimeout = 500 * time.Millisecond - c.LeaderLeaseTimeout = 250 * time.Millisecond - - ldb := raft.NewInmemStore() - sdb := raft.NewInmemStore() - fss := raft.NewInmemSnapshotStore() - r, err := raft.NewRaft(c, NewKvFSMWithHLC(store.NewMVCCStore(), NewHLC()), ldb, sdb, fss, followerTrans) - require.NoError(t, err) - require.NoError(t, r.BootstrapCluster(raftCfg).Error()) - t.Cleanup(func() { _ = r.Shutdown().Error() }) - return r - }() - - require.Eventually(t, func() bool { return leader.State() == raft.Leader }, 5*time.Second, 10*time.Millisecond) - require.Eventually(t, func() bool { return follower.State() == raft.Follower }, 5*time.Second, 10*time.Millisecond) + // Wait briefly so the gRPC server is ready to serve. require.Eventually(t, func() bool { - addr, _ := follower.LeaderWithID() - return addr == leaderAddr - }, 5*time.Second, 10*time.Millisecond) - - p := NewLeaderProxy(follower) + c, err := net.DialTimeout("tcp", lis.Addr().String(), 100*time.Millisecond) + if err != nil { + return false + } + _ = c.Close() + return true + }, 2*time.Second, 10*time.Millisecond) + + follower := &stubFollowerEngine{leaderAddr: lis.Addr().String()} + p := NewLeaderProxyWithEngine(follower) t.Cleanup(func() { _ = p.connCache.Close() }) reqs := []*pb.Request{ diff --git a/kv/leader_routed_store.go b/kv/leader_routed_store.go index 708c693cf..996275fae 100644 --- a/kv/leader_routed_store.go +++ b/kv/leader_routed_store.go @@ -9,7 +9,6 @@ import ( pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" ) // LeaderRoutedStore is an MVCCStore wrapper that serves reads from the local @@ -64,7 +63,7 @@ func (s *LeaderRoutedStore) leaderOKForKey(ctx context.Context, key []byte) bool return ok } -func (s *LeaderRoutedStore) leaderAddrForKey(key []byte) raft.ServerAddress { +func (s *LeaderRoutedStore) leaderAddrForKey(key []byte) string { if s.coordinator == nil { return "" } diff --git a/kv/leader_routed_store_test.go b/kv/leader_routed_store_test.go index 63b555830..c1eb62169 100644 --- a/kv/leader_routed_store_test.go +++ b/kv/leader_routed_store_test.go @@ -8,7 +8,7 @@ import ( pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" - "github.com/hashicorp/raft" + "github.com/stretchr/testify/require" "google.golang.org/grpc" ) @@ -18,7 +18,7 @@ type stubLeaderCoordinator struct { verify error linearizableErr error linearizableCalls int - leader raft.ServerAddress + leader string clock *HLC } @@ -34,7 +34,7 @@ func (s *stubLeaderCoordinator) VerifyLeader() error { return s.verify } -func (s *stubLeaderCoordinator) RaftLeader() raft.ServerAddress { +func (s *stubLeaderCoordinator) RaftLeader() string { return s.leader } @@ -46,7 +46,7 @@ func (s *stubLeaderCoordinator) VerifyLeaderForKey([]byte) error { return s.verify } -func (s *stubLeaderCoordinator) RaftLeaderForKey([]byte) raft.ServerAddress { +func (s *stubLeaderCoordinator) RaftLeaderForKey([]byte) string { return s.leader } @@ -121,7 +121,7 @@ func (f *fakeRawKVServer) RawLatestCommitTS(context.Context, *pb.RawLatestCommit return &pb.RawLatestCommitTSResponse{}, nil } -func startRawKVServer(t *testing.T, svc pb.RawKVServer) (raft.ServerAddress, func()) { +func startRawKVServer(t *testing.T, svc pb.RawKVServer) (string, func()) { t.Helper() var lc net.ListenConfig @@ -138,7 +138,7 @@ func startRawKVServer(t *testing.T, svc pb.RawKVServer) (raft.ServerAddress, fun grpcServer.Stop() _ = lis.Close() } - return raft.ServerAddress(lis.Addr().String()), stop + return string(lis.Addr().String()), stop } func TestLeaderRoutedStore_UsesLocalStoreWhenLeaderVerified(t *testing.T) { diff --git a/kv/lease_state_test.go b/kv/lease_state_test.go index 3ea37dd0f..2602ce830 100644 --- a/kv/lease_state_test.go +++ b/kv/lease_state_test.go @@ -8,8 +8,6 @@ import ( "time" "github.com/bootjp/elastickv/internal/raftengine" - cockroachdberrors "github.com/cockroachdb/errors" - hashicorpraft "github.com/hashicorp/raft" "github.com/stretchr/testify/require" ) @@ -27,26 +25,6 @@ func TestIsLeadershipLossError(t *testing.T) { {"raftengine ErrNotLeader direct", raftengine.ErrNotLeader, true}, {"raftengine ErrLeadershipLost direct", raftengine.ErrLeadershipLost, true}, {"raftengine ErrLeadershipTransferInProgress direct", raftengine.ErrLeadershipTransferInProgress, true}, - { - "hashicorp ErrNotLeader marked with raftengine sentinel", - cockroachdberrors.WithStack(cockroachdberrors.Mark(hashicorpraft.ErrNotLeader, raftengine.ErrNotLeader)), - true, - }, - { - "hashicorp ErrLeadershipLost marked with raftengine sentinel", - cockroachdberrors.WithStack(cockroachdberrors.Mark(hashicorpraft.ErrLeadershipLost, raftengine.ErrLeadershipLost)), - true, - }, - { - "hashicorp ErrLeadershipTransferInProgress marked with raftengine sentinel", - cockroachdberrors.WithStack(cockroachdberrors.Mark(hashicorpraft.ErrLeadershipTransferInProgress, raftengine.ErrLeadershipTransferInProgress)), - true, - }, - { - "bare hashicorp ErrNotLeader (no raftengine mark) is NOT detected", - hashicorpraft.ErrNotLeader, - false, - }, } for _, tc := range cases { diff --git a/kv/lock_resolver_test.go b/kv/lock_resolver_test.go index 98dada885..9d37771c0 100644 --- a/kv/lock_resolver_test.go +++ b/kv/lock_resolver_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/bootjp/elastickv/distribution" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" @@ -26,8 +25,8 @@ func setupLockResolverEnv(t *testing.T) (*LockResolver, *ShardStore, map[uint64] st2 := store.NewMVCCStore() r2, stop2 := newSingleRaft(t, "lr-g2", NewKvFSMWithHLC(st2, NewHLC())) - e1 := hashicorpraftengine.New(r1) - e2 := hashicorpraftengine.New(r2) + e1 := r1 + e2 := r2 groups := map[uint64]*ShardGroup{ 1: {Engine: e1, Store: st1, Txn: NewLeaderProxyWithEngine(e1)}, 2: {Engine: e2, Store: st2, Txn: NewLeaderProxyWithEngine(e2)}, @@ -209,7 +208,7 @@ func TestLockResolver_LeaderOnlyExecution(t *testing.T) { r, stop := newSingleRaft(t, "lr-leader", NewKvFSMWithHLC(st, NewHLC())) defer stop() - e := hashicorpraftengine.New(r) + e := r groups := map[uint64]*ShardGroup{ 1: {Engine: e, Store: st, Txn: NewLeaderProxyWithEngine(e)}, } @@ -240,7 +239,7 @@ func TestLockResolver_CloseStopsBackground(t *testing.T) { r, stop := newSingleRaft(t, "lr-close", NewKvFSMWithHLC(st, NewHLC())) defer stop() - e := hashicorpraftengine.New(r) + e := r groups := map[uint64]*ShardGroup{ 1: {Engine: e, Store: st, Txn: NewLeaderProxyWithEngine(e)}, } diff --git a/kv/raft_engine.go b/kv/raft_engine.go index 334216411..75cf57857 100644 --- a/kv/raft_engine.go +++ b/kv/raft_engine.go @@ -6,7 +6,6 @@ import ( "github.com/bootjp/elastickv/internal/raftengine" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" ) func engineForGroup(g *ShardGroup) raftengine.Engine { @@ -92,9 +91,9 @@ func leaseReadEngineCtx(ctx context.Context, engine raftengine.LeaderView) (uint return index, nil } -func leaderAddrFromEngine(engine raftengine.LeaderView) raft.ServerAddress { +func leaderAddrFromEngine(engine raftengine.LeaderView) string { if engine == nil { return "" } - return raft.ServerAddress(engine.Leader().Address) + return engine.Leader().Address } diff --git a/kv/shard_router_test.go b/kv/shard_router_test.go index 710e71107..0de6c0ce7 100644 --- a/kv/shard_router_test.go +++ b/kv/shard_router_test.go @@ -4,117 +4,19 @@ import ( "context" "fmt" "testing" - "time" "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" - "github.com/hashicorp/raft" ) -// helper to create a multi-node raft cluster and return the leader -func newTestRaft(t *testing.T, id string, fsm raft.FSM) (*raft.Raft, func()) { +// newTestRaft creates a single-node raft engine for tests. Historically this +// built a 3-node in-memory hashicorp cluster; with the hashicorp backend +// removed we simply delegate to the newSingleRaft helper used elsewhere. +func newTestRaft(t *testing.T, id string, fsm FSM) (raftengine.Engine, func()) { t.Helper() - - const n = 3 - addrs, trans := setupInmemTransports(id, n) - connectInmemTransports(addrs, trans) - cfg := buildRaftConfig(id, addrs) - rafts := initTestRafts(t, cfg, trans, fsm) - waitForLeader(t, id, rafts[0]) - - shutdown := func() { - for _, r := range rafts { - r.Shutdown() - } - } - return rafts[0], shutdown -} - -func setupInmemTransports(id string, n int) ([]raft.ServerAddress, []*raft.InmemTransport) { - addrs := make([]raft.ServerAddress, n) - trans := make([]*raft.InmemTransport, n) - for i := range n { - addr, tr := raft.NewInmemTransport(raft.ServerAddress(fmt.Sprintf("%s-%d", id, i))) - addrs[i] = addr - trans[i] = tr - } - return addrs, trans -} - -func connectInmemTransports(addrs []raft.ServerAddress, trans []*raft.InmemTransport) { - // fully connect transports - for i := range trans { - for j := i + 1; j < len(trans); j++ { - trans[i].Connect(addrs[j], trans[j]) - trans[j].Connect(addrs[i], trans[i]) - } - } -} - -func buildRaftConfig(id string, addrs []raft.ServerAddress) raft.Configuration { - // cluster configuration - cfg := raft.Configuration{} - for i := range addrs { - cfg.Servers = append(cfg.Servers, raft.Server{ - ID: raft.ServerID(fmt.Sprintf("%s-%d", id, i)), - Address: addrs[i], - }) - } - return cfg -} - -func initTestRafts(t *testing.T, cfg raft.Configuration, trans []*raft.InmemTransport, fsm raft.FSM) []*raft.Raft { - t.Helper() - - rafts := make([]*raft.Raft, len(trans)) - for i := range trans { - c := raft.DefaultConfig() - c.LocalID = cfg.Servers[i].ID - if i == 0 { - c.HeartbeatTimeout = 200 * time.Millisecond - c.ElectionTimeout = 400 * time.Millisecond - c.LeaderLeaseTimeout = 100 * time.Millisecond - } else { - c.HeartbeatTimeout = 1 * time.Second - c.ElectionTimeout = 2 * time.Second - c.LeaderLeaseTimeout = 500 * time.Millisecond - } - ldb := raft.NewInmemStore() - sdb := raft.NewInmemStore() - fss := raft.NewInmemSnapshotStore() - var rfsm raft.FSM - if i == 0 { - rfsm = fsm - } else { - rfsm = NewKvFSMWithHLC(store.NewMVCCStore(), NewHLC()) - } - r, err := raft.NewRaft(c, rfsm, ldb, sdb, fss, trans[i]) - if err != nil { - t.Fatalf("new raft %d: %v", i, err) - } - if err := r.BootstrapCluster(cfg).Error(); err != nil { - t.Fatalf("bootstrap %d: %v", i, err) - } - rafts[i] = r - } - - return rafts -} - -func waitForLeader(t *testing.T, id string, leader *raft.Raft) { - t.Helper() - - // node 0 should become leader quickly during tests - for range 100 { - if leader.State() == raft.Leader { - break - } - time.Sleep(50 * time.Millisecond) - } - if leader.State() != raft.Leader { - t.Fatalf("node %s-0 is not leader", id) - } + return newSingleRaft(t, id, fsm) } func TestShardRouterCommit(t *testing.T) { @@ -130,13 +32,13 @@ func TestShardRouterCommit(t *testing.T) { s1 := store.NewMVCCStore() r1, stop1 := newTestRaft(t, "1", NewKvFSMWithHLC(s1, NewHLC())) defer stop1() - router.Register(1, NewTransaction(r1), s1) + router.Register(1, NewTransactionWithProposer(r1), s1) // group 2 s2 := store.NewMVCCStore() r2, stop2 := newTestRaft(t, "2", NewKvFSMWithHLC(s2, NewHLC())) defer stop2() - router.Register(2, NewTransaction(r2), s2) + router.Register(2, NewTransactionWithProposer(r2), s2) reqs := []*pb.Request{ {IsTxn: false, Phase: pb.Phase_NONE, Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v1")}}}, @@ -170,13 +72,13 @@ func TestShardRouterSplitAndMerge(t *testing.T) { s1 := store.NewMVCCStore() r1, stop1 := newTestRaft(t, "1", NewKvFSMWithHLC(s1, NewHLC())) defer stop1() - router.Register(1, NewTransaction(r1), s1) + router.Register(1, NewTransactionWithProposer(r1), s1) // group 2 (will be used after split) s2 := store.NewMVCCStore() r2, stop2 := newTestRaft(t, "2", NewKvFSMWithHLC(s2, NewHLC())) defer stop2() - router.Register(2, NewTransaction(r2), s2) + router.Register(2, NewTransactionWithProposer(r2), s2) // initial write routed to group 1 req := []*pb.Request{ diff --git a/kv/shard_store_txn_lock_test.go b/kv/shard_store_txn_lock_test.go index ff97266c0..e5654645f 100644 --- a/kv/shard_store_txn_lock_test.go +++ b/kv/shard_store_txn_lock_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/bootjp/elastickv/distribution" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" @@ -26,8 +25,8 @@ func setupTwoShardStore(t *testing.T) (*ShardStore, map[uint64]*ShardGroup, func st2 := store.NewMVCCStore() r2, stop2 := newSingleRaft(t, "g2", NewKvFSMWithHLC(st2, NewHLC())) - e1 := hashicorpraftengine.New(r1) - e2 := hashicorpraftengine.New(r2) + e1 := r1 + e2 := r2 groups := map[uint64]*ShardGroup{ 1: {Engine: e1, Store: st1, Txn: NewLeaderProxyWithEngine(e1)}, 2: {Engine: e2, Store: st2, Txn: NewLeaderProxyWithEngine(e2)}, @@ -97,7 +96,7 @@ func TestShardStoreGetAt_ReturnsTxnLockedForPendingLock(t *testing.T) { r1, stop1 := newSingleRaft(t, "g1", NewKvFSMWithHLC(st1, NewHLC())) defer stop1() - e1 := hashicorpraftengine.New(r1) + e1 := r1 groups := map[uint64]*ShardGroup{ 1: {Engine: e1, Store: st1, Txn: NewLeaderProxyWithEngine(e1)}, } @@ -234,7 +233,7 @@ func TestShardStoreScanAt_ReturnsTxnLockedForPendingLock(t *testing.T) { r1, stop1 := newSingleRaft(t, "g1", NewKvFSMWithHLC(st1, NewHLC())) defer stop1() - e1 := hashicorpraftengine.New(r1) + e1 := r1 groups := map[uint64]*ShardGroup{ 1: {Engine: e1, Store: st1, Txn: NewLeaderProxyWithEngine(e1)}, } @@ -264,7 +263,7 @@ func TestShardStoreScanAt_ReturnsTxnLockedForPendingLockWithoutCommittedValue(t r1, stop1 := newSingleRaft(t, "g1", NewKvFSMWithHLC(st1, NewHLC())) defer stop1() - e1 := hashicorpraftengine.New(r1) + e1 := r1 groups := map[uint64]*ShardGroup{ 1: {Engine: e1, Store: st1, Txn: NewLeaderProxyWithEngine(e1)}, } @@ -294,7 +293,7 @@ func TestShardStoreScanAt_ReturnsTxnLockedWhenPendingLockExceedsUserLimit(t *tes r1, stop1 := newSingleRaft(t, "g1", NewKvFSMWithHLC(st1, NewHLC())) defer stop1() - e1 := hashicorpraftengine.New(r1) + e1 := r1 groups := map[uint64]*ShardGroup{ 1: {Engine: e1, Store: st1, Txn: NewLeaderProxyWithEngine(e1)}, } diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 6c018d837..b6fc036c7 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -15,7 +15,6 @@ import ( pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" ) type ShardGroup struct { @@ -686,7 +685,7 @@ func (c *ShardedCoordinator) VerifyLeader() error { return verifyLeaderEngine(engineForGroup(g)) } -func (c *ShardedCoordinator) RaftLeader() raft.ServerAddress { +func (c *ShardedCoordinator) RaftLeader() string { g, ok := c.groups[c.defaultGroup] if !ok { return "" @@ -718,7 +717,7 @@ func (c *ShardedCoordinator) VerifyLeaderForKey(key []byte) error { return verifyLeaderEngine(engineForGroup(g)) } -func (c *ShardedCoordinator) RaftLeaderForKey(key []byte) raft.ServerAddress { +func (c *ShardedCoordinator) RaftLeaderForKey(key []byte) string { g, ok := c.groupForKey(key) if !ok { return "" diff --git a/kv/sharded_coordinator_abort_test.go b/kv/sharded_coordinator_abort_test.go index 375ff5db8..c6d553a8d 100644 --- a/kv/sharded_coordinator_abort_test.go +++ b/kv/sharded_coordinator_abort_test.go @@ -7,7 +7,6 @@ import ( "testing" "github.com/bootjp/elastickv/distribution" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" @@ -48,7 +47,7 @@ func TestShardedAbortRollback_PrepareFailOnShard2_CleansShard1Locks(t *testing.T s2 := store.NewMVCCStore() failTxn := &failingTransactional{err: errors.New("simulated shard2 prepare failure")} - e1 := hashicorpraftengine.New(r1) + e1 := r1 groups := map[uint64]*ShardGroup{ 1: {Engine: e1, Store: s1, Txn: NewLeaderProxyWithEngine(e1)}, 2: {Store: s2, Txn: failTxn}, diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index e812cd878..11de9de78 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/bootjp/elastickv/distribution" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" @@ -277,8 +276,8 @@ func TestShardedCoordinator_DelPrefixIntegration(t *testing.T) { r2, stop2 := newSingleRaft(t, "dp-g2", NewKvFSMWithHLC(s2, NewHLC())) t.Cleanup(stop2) - e1 := hashicorpraftengine.New(r1) - e2 := hashicorpraftengine.New(r2) + e1 := r1 + e2 := r2 groups := map[uint64]*ShardGroup{ 1: {Engine: e1, Store: s1, Txn: NewLeaderProxyWithEngine(e1)}, 2: {Engine: e2, Store: s2, Txn: NewLeaderProxyWithEngine(e2)}, @@ -341,7 +340,7 @@ func TestShardedCoordinator_DelPrefixPreservesTxnKeys(t *testing.T) { r1, stop1 := newSingleRaft(t, "dp-txn-g1", NewKvFSMWithHLC(s1, NewHLC())) t.Cleanup(stop1) - e1 := hashicorpraftengine.New(r1) + e1 := r1 groups := map[uint64]*ShardGroup{ 1: {Engine: e1, Store: s1, Txn: NewLeaderProxyWithEngine(e1)}, } diff --git a/kv/sharded_coordinator_leader_test.go b/kv/sharded_coordinator_leader_test.go index 186651dcc..201a899bb 100644 --- a/kv/sharded_coordinator_leader_test.go +++ b/kv/sharded_coordinator_leader_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/bootjp/elastickv/distribution" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" ) @@ -19,7 +18,7 @@ func TestShardedCoordinatorVerifyLeader_LeaderReturnsNil(t *testing.T) { r, stop := newSingleRaft(t, "shard-leader", NewKvFSMWithHLC(st, NewHLC())) t.Cleanup(stop) - re := hashicorpraftengine.New(r) + re := r groups := map[uint64]*ShardGroup{ 1: {Engine: re, Store: st, Txn: NewLeaderProxyWithEngine(re)}, } diff --git a/kv/sharded_integration_test.go b/kv/sharded_integration_test.go index 22604a6ee..fecc25af5 100644 --- a/kv/sharded_integration_test.go +++ b/kv/sharded_integration_test.go @@ -6,53 +6,65 @@ import ( "time" "github.com/bootjp/elastickv/distribution" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" + "github.com/bootjp/elastickv/internal/raftengine" + etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" ) -func newSingleRaft(t *testing.T, id string, fsm raft.FSM) (*raft.Raft, func()) { +const ( + testSingleRaftTickInterval = 10 * time.Millisecond + testSingleRaftHeartbeatTick = 1 + testSingleRaftElectionTick = 10 + testSingleRaftMaxSizePerMsg = 1 << 20 + testSingleRaftMaxInflight = 256 +) + +func newSingleRaftFactory() raftengine.Factory { + return etcdraftengine.NewFactory(etcdraftengine.FactoryConfig{ + TickInterval: testSingleRaftTickInterval, + HeartbeatTick: testSingleRaftHeartbeatTick, + ElectionTick: testSingleRaftElectionTick, + MaxSizePerMsg: testSingleRaftMaxSizePerMsg, + MaxInflightMsg: testSingleRaftMaxInflight, + }) +} + +func newSingleRaft(t *testing.T, id string, fsm FSM) (raftengine.Engine, func()) { t.Helper() - addr, trans := raft.NewInmemTransport(raft.ServerAddress(id)) - c := raft.DefaultConfig() - c.LocalID = raft.ServerID(id) - c.HeartbeatTimeout = 50 * time.Millisecond - c.ElectionTimeout = 100 * time.Millisecond - c.LeaderLeaseTimeout = 50 * time.Millisecond - - ldb := raft.NewInmemStore() - sdb := raft.NewInmemStore() - fss := raft.NewInmemSnapshotStore() - r, err := raft.NewRaft(c, fsm, ldb, sdb, fss, trans) + factory := newSingleRaftFactory() + result, err := factory.Create(raftengine.FactoryConfig{ + LocalID: id, + LocalAddress: id, + DataDir: t.TempDir(), + Bootstrap: true, + StateMachine: fsm, + }) if err != nil { t.Fatalf("new raft: %v", err) } - cfg := raft.Configuration{ - Servers: []raft.Server{ - { - Suffrage: raft.Voter, - ID: raft.ServerID(id), - Address: addr, - }, - }, - } - if err := r.BootstrapCluster(cfg).Error(); err != nil { - t.Fatalf("bootstrap: %v", err) - } - for range 100 { - if r.State() == raft.Leader { + for range 200 { + if result.Engine.State() == raftengine.StateLeader { break } time.Sleep(10 * time.Millisecond) } - if r.State() != raft.Leader { + if result.Engine.State() != raftengine.StateLeader { + _ = result.Engine.Close() + if result.Close != nil { + _ = result.Close() + } t.Fatalf("node %s is not leader", id) } - return r, func() { r.Shutdown() } + return result.Engine, func() { + _ = result.Engine.Close() + if result.Close != nil { + _ = result.Close() + } + } } func TestShardedCoordinatorDispatch(t *testing.T) { @@ -70,8 +82,8 @@ func TestShardedCoordinatorDispatch(t *testing.T) { r2, stop2 := newSingleRaft(t, "g2", NewKvFSMWithHLC(s2, NewHLC())) defer stop2() - e1 := hashicorpraftengine.New(r1) - e2 := hashicorpraftengine.New(r2) + e1 := r1 + e2 := r2 groups := map[uint64]*ShardGroup{ 1: {Engine: e1, Store: s1, Txn: NewLeaderProxyWithEngine(e1)}, 2: {Engine: e2, Store: s2, Txn: NewLeaderProxyWithEngine(e2)}, @@ -124,8 +136,8 @@ func TestShardedCoordinatorDispatch_CrossShardTxnSucceeds(t *testing.T) { r2, stop2 := newSingleRaft(t, "g2", NewKvFSMWithHLC(s2, NewHLC())) defer stop2() - e1 := hashicorpraftengine.New(r1) - e2 := hashicorpraftengine.New(r2) + e1 := r1 + e2 := r2 groups := map[uint64]*ShardGroup{ 1: {Engine: e1, Store: s1, Txn: NewLeaderProxyWithEngine(e1)}, 2: {Engine: e2, Store: s2, Txn: NewLeaderProxyWithEngine(e2)}, diff --git a/kv/snapshot.go b/kv/snapshot.go index f8bf7d689..221b9b9d4 100644 --- a/kv/snapshot.go +++ b/kv/snapshot.go @@ -2,11 +2,12 @@ package kv import ( "encoding/binary" + "io" "sync" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" ) // hlcSnapshotMagic is an 8-byte sentinel written at the start of every FSM @@ -17,7 +18,7 @@ var hlcSnapshotMagic = [8]byte{'E', 'K', 'V', 'T', 'H', 'L', 'C', '1'} // hlcSnapshotHeaderLen is the total header size: 8 magic + 8 ceiling ms. const hlcSnapshotHeaderLen = 16 //nolint:mnd -var _ raft.FSMSnapshot = (*kvFSMSnapshot)(nil) +var _ raftengine.Snapshot = (*kvFSMSnapshot)(nil) type kvFSMSnapshot struct { snapshot store.Snapshot @@ -26,29 +27,22 @@ type kvFSMSnapshot struct { err error } -func (f *kvFSMSnapshot) Persist(sink raft.SnapshotSink) (err error) { - defer func() { - err = errors.CombineErrors(err, f.closeSnapshot()) - }() - +func (f *kvFSMSnapshot) WriteTo(w io.Writer) (int64, error) { // Write the 16-byte header: magic (8 bytes) + ceiling ms (8 bytes). var hdr [hlcSnapshotHeaderLen]byte copy(hdr[:8], hlcSnapshotMagic[:]) binary.BigEndian.PutUint64(hdr[8:], uint64(f.ceilingMs)) //nolint:gosec // ceiling is a Unix ms timestamp, always positive - if _, err = sink.Write(hdr[:]); err != nil { - cancelErr := sink.Cancel() - return errors.WithStack(errors.CombineErrors(errors.WithStack(err), errors.WithStack(cancelErr))) + n, err := w.Write(hdr[:]) + if err != nil { + return int64(n), errors.WithStack(err) } - if _, err = f.snapshot.WriteTo(sink); err != nil { - cancelErr := sink.Cancel() - return errors.WithStack(errors.CombineErrors(errors.WithStack(err), errors.WithStack(cancelErr))) + m, err := f.snapshot.WriteTo(w) + total := int64(n) + m + if err != nil { + return total, errors.WithStack(err) } - return errors.WithStack(sink.Close()) -} - -func (f *kvFSMSnapshot) Release() { - _ = f.closeSnapshot() + return total, nil } func (f *kvFSMSnapshot) Close() error { diff --git a/kv/snapshot_test.go b/kv/snapshot_test.go index 6baec024f..3f78a3d83 100644 --- a/kv/snapshot_test.go +++ b/kv/snapshot_test.go @@ -8,7 +8,6 @@ import ( pb "github.com/bootjp/elastickv/proto" store3 "github.com/bootjp/elastickv/store" - "github.com/hashicorp/raft" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" @@ -33,13 +32,7 @@ func TestSnapshot(t *testing.T) { b, err := proto.Marshal(&mutation) assert.NoError(t, err) - fsm.Apply(&raft.Log{ - Type: raft.LogCommand, - Data: b, - }) - fsm.Apply(&raft.Log{ - Type: raft.LogBarrier, - }) + fsm.Apply(b) ctx := context.Background() v, err := store.GetAt(ctx, []byte("hoge"), ^uint64(0)) @@ -57,9 +50,9 @@ func TestSnapshot(t *testing.T) { assert.NoError(t, err) var buf bytes.Buffer - _, err = kvFSMSnap.snapshot.WriteTo(&buf) + _, err = kvFSMSnap.WriteTo(&buf) assert.NoError(t, err) - kvFSMSnap.Release() + assert.NoError(t, kvFSMSnap.Close()) err = fsm2.Restore(io.NopCloser(bytes.NewReader(buf.Bytes()))) assert.NoError(t, err) @@ -93,15 +86,16 @@ func TestFSMSnapshotPreservesCeiling(t *testing.T) { } b, err := proto.Marshal(req) require.NoError(t, err) - fsm1.Apply(&raft.Log{Type: raft.LogCommand, Data: b}) + fsm1.Apply(b) - // Take snapshot and persist it via Persist (the real code path). + // Take snapshot and write it out. snap, err := fsm1.Snapshot() require.NoError(t, err) var buf bytes.Buffer - sink := &snapshotSinkAdapter{writer: &buf} - require.NoError(t, snap.Persist(sink)) + _, err = snap.WriteTo(&buf) + require.NoError(t, err) + require.NoError(t, snap.Close()) // Restore into a fresh FSM with its own HLC starting at zero. hlc2 := NewHLC() @@ -172,7 +166,7 @@ func TestFSMSnapshotRestoreOldFormat(t *testing.T) { } b, err := proto.Marshal(req) require.NoError(t, err) - fsm1.Apply(&raft.Log{Type: raft.LogCommand, Data: b}) + fsm1.Apply(b) // Grab the raw store snapshot bytes (bypasses the new header). storeSnap, err := st1.Snapshot() diff --git a/kv/transaction.go b/kv/transaction.go index ab9020193..531eea9ef 100644 --- a/kv/transaction.go +++ b/kv/transaction.go @@ -7,10 +7,8 @@ import ( "time" "github.com/bootjp/elastickv/internal/raftengine" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" "google.golang.org/protobuf/proto" ) @@ -62,10 +60,6 @@ func WithProposalObserver(observer ProposalObserver) TransactionOption { } } -func NewTransaction(raft *raft.Raft, opts ...TransactionOption) *TransactionManager { - return NewTransactionWithProposer(hashicorpraftengine.New(raft), opts...) -} - func NewTransactionWithProposer(proposer raftengine.Proposer, opts ...TransactionOption) *TransactionManager { t := &TransactionManager{ proposer: proposer, diff --git a/kv/transaction_batch_test.go b/kv/transaction_batch_test.go index 2540e0c7b..31f37e056 100644 --- a/kv/transaction_batch_test.go +++ b/kv/transaction_batch_test.go @@ -2,16 +2,13 @@ package kv import ( "context" - "io" "sync" "testing" "time" etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" - "github.com/hashicorp/raft" "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" ) @@ -67,7 +64,7 @@ func TestFSMApplyBatchKeepsPerRequestResults(t *testing.T) { data, err := proto.Marshal(cmd) require.NoError(t, err) - resp := fsm.Apply(&raft.Log{Type: raft.LogCommand, Data: data}) + resp := fsm.Apply(data) applyResp, ok := resp.(*fsmApplyResponse) require.True(t, ok) require.Len(t, applyResp.results, 3) @@ -95,7 +92,7 @@ func TestTransactionManagerBatchesConcurrentRawCommits(t *testing.T) { r, stop := newSingleRaft(t, "raw-batch", NewKvFSMWithHLC(st, NewHLC())) defer stop() - tm := NewTransaction(r) + tm := NewTransactionWithProposer(r) req1 := []*pb.Request{{ IsTxn: false, @@ -170,7 +167,7 @@ func TestApplyRequestsCountsProposalFailureOnRaftApplyError(t *testing.T) { }, }} - _, _, err := applyRequests(hashicorpraftengine.New(r), reqs, observer) + _, _, err := applyRequests(r, reqs, observer) require.Error(t, err) require.Equal(t, 1, observer.FailureCount()) } @@ -189,73 +186,13 @@ func TestApplyRequestsDoesNotCountBusinessErrorAsProposalFailure(t *testing.T) { }, }} - _, results, err := applyRequests(hashicorpraftengine.New(r), reqs, observer) + _, results, err := applyRequests(r, reqs, observer) require.NoError(t, err) require.Len(t, results, 1) require.ErrorIs(t, results[0], ErrInvalidRequest) require.Zero(t, observer.FailureCount()) } -type etcdFSMAdapter struct { - fsm raft.FSM -} - -func (a etcdFSMAdapter) Apply(data []byte) any { - return a.fsm.Apply(&raft.Log{Type: raft.LogCommand, Data: data}) -} - -func (a etcdFSMAdapter) Snapshot() (etcdraftengine.Snapshot, error) { - snapshot, err := a.fsm.Snapshot() - if err != nil { - return nil, err - } - return hashicorpSnapshotAdapter{snapshot: snapshot}, nil -} - -func (a etcdFSMAdapter) Restore(r io.Reader) error { - return a.fsm.Restore(io.NopCloser(r)) -} - -type hashicorpSnapshotAdapter struct { - snapshot raft.FSMSnapshot -} - -func (a hashicorpSnapshotAdapter) WriteTo(w io.Writer) (int64, error) { - sink := &snapshotSinkAdapter{writer: w} - if err := a.snapshot.Persist(sink); err != nil { - return sink.written, err - } - return sink.written, nil -} - -func (a hashicorpSnapshotAdapter) Close() error { - a.snapshot.Release() - return nil -} - -type snapshotSinkAdapter struct { - writer io.Writer - written int64 -} - -func (s *snapshotSinkAdapter) ID() string { - return "etcd-fsm-adapter" -} - -func (s *snapshotSinkAdapter) Cancel() error { - return nil -} - -func (s *snapshotSinkAdapter) Close() error { - return nil -} - -func (s *snapshotSinkAdapter) Write(p []byte) (int, error) { - n, err := s.writer.Write(p) - s.written += int64(n) - return n, err -} - func TestApplyRequestsWithEtcdEngineKeepsKVCommandSemantics(t *testing.T) { st := store.NewMVCCStore() engine, err := etcdraftengine.Open(context.Background(), etcdraftengine.OpenConfig{ @@ -264,7 +201,7 @@ func TestApplyRequestsWithEtcdEngineKeepsKVCommandSemantics(t *testing.T) { LocalAddress: "127.0.0.1:7001", DataDir: t.TempDir(), Bootstrap: true, - StateMachine: etcdFSMAdapter{fsm: NewKvFSMWithHLC(st, NewHLC())}, + StateMachine: NewKvFSMWithHLC(st, NewHLC()), }) require.NoError(t, err) t.Cleanup(func() { diff --git a/main.go b/main.go index f2f66e59b..f26823a03 100644 --- a/main.go +++ b/main.go @@ -19,13 +19,11 @@ import ( internalraftadmin "github.com/bootjp/elastickv/internal/raftadmin" "github.com/bootjp/elastickv/internal/raftengine" etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" - hashicorpraftengine "github.com/bootjp/elastickv/internal/raftengine/hashicorp" "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/monitoring" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/reflection" @@ -34,8 +32,6 @@ import ( const ( heartbeatTimeout = 200 * time.Millisecond electionTimeout = 2000 * time.Millisecond - leaderLease = 100 * time.Millisecond - raftCommitTimeout = 50 * time.Millisecond raftMetricsObserveInterval = 5 * time.Second dirPerm = raftDirPerm @@ -46,18 +42,8 @@ const ( etcdMaxInflightMsg = 256 ) -const snapshotRetainCount = 3 - func newRaftFactory(engineType raftEngineType) (raftengine.Factory, error) { switch engineType { - case raftEngineHashicorp: - return hashicorpraftengine.NewFactory(hashicorpraftengine.FactoryConfig{ - CommitTimeout: raftCommitTimeout, - HeartbeatTimeout: heartbeatTimeout, - ElectionTimeout: electionTimeout, - LeaderLeaseTimeout: leaderLease, - SnapshotRetainCount: snapshotRetainCount, - }), nil case raftEngineEtcd: return etcdraftengine.NewFactory(etcdraftengine.FactoryConfig{ TickInterval: etcdTickInterval, @@ -239,7 +225,7 @@ func run() error { return nil } -func resolveRuntimeInputs() (runtimeConfig, raftEngineType, []raft.Server, bool, error) { +func resolveRuntimeInputs() (runtimeConfig, raftEngineType, []raftengine.Server, bool, error) { if *raftId == "" { return runtimeConfig{}, "", nil, false, errors.New("flag --raftId is required") } @@ -266,9 +252,9 @@ type runtimeConfig struct { groups []groupSpec defaultGroup uint64 engine *distribution.Engine - leaderRedis map[raft.ServerAddress]string - leaderS3 map[raft.ServerAddress]string - leaderDynamo map[raft.ServerAddress]string + leaderRedis map[string]string + leaderS3 map[string]string + leaderDynamo map[string]string multi bool } @@ -319,15 +305,15 @@ func buildEngine(ranges []rangeSpec) *distribution.Engine { return engine } -func buildLeaderRedis(groups []groupSpec, redisAddr string, raftRedisMap string) (map[raft.ServerAddress]string, error) { +func buildLeaderRedis(groups []groupSpec, redisAddr string, raftRedisMap string) (map[string]string, error) { return buildLeaderAddrMap(groups, redisAddr, raftRedisMap, parseRaftRedisMap) } -func buildLeaderS3(groups []groupSpec, s3Addr string, raftS3Map string) (map[raft.ServerAddress]string, error) { +func buildLeaderS3(groups []groupSpec, s3Addr string, raftS3Map string) (map[string]string, error) { return buildLeaderAddrMap(groups, s3Addr, raftS3Map, parseRaftS3Map) } -func buildLeaderDynamo(groups []groupSpec, dynamoAddr string, raftDynamoMap string) (map[raft.ServerAddress]string, error) { +func buildLeaderDynamo(groups []groupSpec, dynamoAddr string, raftDynamoMap string) (map[string]string, error) { return buildLeaderAddrMap(groups, dynamoAddr, raftDynamoMap, parseRaftDynamoMap) } @@ -335,16 +321,15 @@ func buildLeaderAddrMap( groups []groupSpec, defaultAddr string, rawMap string, - parse func(string) (map[raft.ServerAddress]string, error), -) (map[raft.ServerAddress]string, error) { + parse func(string) (map[string]string, error), +) (map[string]string, error) { leaderAddrMap, err := parse(rawMap) if err != nil { return nil, err } for _, g := range groups { - addr := raft.ServerAddress(g.address) - if _, ok := leaderAddrMap[addr]; !ok { - leaderAddrMap[addr] = defaultAddr + if _, ok := leaderAddrMap[g.address]; !ok { + leaderAddrMap[g.address] = defaultAddr } } return leaderAddrMap, nil @@ -357,7 +342,7 @@ var ( ErrNoBootstrapMembersConfigured = errors.New("no bootstrap members configured") ) -func resolveBootstrapServers(raftID string, groups []groupSpec, bootstrapMembers string) ([]raft.Server, error) { +func resolveBootstrapServers(raftID string, groups []groupSpec, bootstrapMembers string) ([]raftengine.Server, error) { if strings.TrimSpace(bootstrapMembers) == "" { return nil, nil } @@ -375,10 +360,10 @@ func resolveBootstrapServers(raftID string, groups []groupSpec, bootstrapMembers localAddr := groups[0].address for _, s := range servers { - if string(s.ID) != raftID { + if s.ID != raftID { continue } - if string(s.Address) != localAddr { + if s.Address != localAddr { return nil, errors.Wrapf(ErrBootstrapMembersLocalAddrMismatch, "expected %q got %q", localAddr, s.Address) } return servers, nil @@ -392,7 +377,7 @@ func buildShardGroups( groups []groupSpec, multi bool, bootstrap bool, - bootstrapServers []raft.Server, + bootstrapServers []raftengine.Server, factory raftengine.Factory, proposalObserverForGroup func(uint64) kv.ProposalObserver, clock *kv.HLC, @@ -410,7 +395,7 @@ func buildShardGroups( } // Each shard FSM shares the same HLC so any shard's lease renewal advances // the global physicalCeiling. The logical counter remains in-memory only. - sm := etcdraftengine.AdaptHashicorpFSM(kv.NewKvFSMWithHLC(st, clock)) + sm := kv.NewKvFSMWithHLC(st, clock) runtime, err := buildRuntimeForGroup(raftID, g, raftDir, multi, bootstrap, bootstrapServers, st, sm, factory) if err != nil { for _, rt := range runtimes { @@ -571,7 +556,7 @@ func startRaftServers( return nil } -func startRedisServer(ctx context.Context, lc *net.ListenConfig, eg *errgroup.Group, redisAddr string, shardStore *kv.ShardStore, coordinate kv.Coordinator, leaderRedis map[raft.ServerAddress]string, relay *adapter.RedisPubSubRelay, metricsRegistry *monitoring.Registry, readTracker *kv.ActiveTimestampTracker) error { +func startRedisServer(ctx context.Context, lc *net.ListenConfig, eg *errgroup.Group, redisAddr string, shardStore *kv.ShardStore, coordinate kv.Coordinator, leaderRedis map[string]string, relay *adapter.RedisPubSubRelay, metricsRegistry *monitoring.Registry, readTracker *kv.ActiveTimestampTracker) error { redisL, err := lc.Listen(ctx, "tcp", redisAddr) if err != nil { return errors.Wrapf(err, "failed to listen on %s", redisAddr) @@ -605,7 +590,7 @@ func startRedisServer(ctx context.Context, lc *net.ListenConfig, eg *errgroup.Gr return nil } -func startDynamoDBServer(ctx context.Context, lc *net.ListenConfig, eg *errgroup.Group, dynamoAddr string, shardStore *kv.ShardStore, coordinate kv.Coordinator, leaderDynamo map[raft.ServerAddress]string, metricsRegistry *monitoring.Registry, readTracker *kv.ActiveTimestampTracker) error { +func startDynamoDBServer(ctx context.Context, lc *net.ListenConfig, eg *errgroup.Group, dynamoAddr string, shardStore *kv.ShardStore, coordinate kv.Coordinator, leaderDynamo map[string]string, metricsRegistry *monitoring.Registry, readTracker *kv.ActiveTimestampTracker) error { dynamoL, err := lc.Listen(ctx, "tcp", dynamoAddr) if err != nil { return errors.Wrapf(err, "failed to listen on %s", dynamoAddr) @@ -754,13 +739,13 @@ type runtimeServerRunner struct { coordinate kv.Coordinator distServer *adapter.DistributionServer redisAddress string - leaderRedis map[raft.ServerAddress]string + leaderRedis map[string]string pubsubRelay *adapter.RedisPubSubRelay readTracker *kv.ActiveTimestampTracker dynamoAddress string - leaderDynamo map[raft.ServerAddress]string + leaderDynamo map[string]string s3Address string - leaderS3 map[raft.ServerAddress]string + leaderS3 map[string]string s3Region string s3CredsFile string s3PathStyleOnly bool diff --git a/main_bootstrap_e2e_test.go b/main_bootstrap_e2e_test.go index 52df08e66..b6040bb45 100644 --- a/main_bootstrap_e2e_test.go +++ b/main_bootstrap_e2e_test.go @@ -19,7 +19,6 @@ import ( "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" - "github.com/hashicorp/raft" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" "google.golang.org/grpc" @@ -83,7 +82,7 @@ func (n *bootstrapE2ENode) close() error { } func TestRaftBootstrapMembers_E2E_FixedClusterWithoutAddVoter(t *testing.T) { - for _, engineType := range []raftEngineType{raftEngineHashicorp, raftEngineEtcd} { + for _, engineType := range []raftEngineType{raftEngineEtcd} { t.Run(string(engineType), func(t *testing.T) { const ( startupAttempts = 5 @@ -422,7 +421,7 @@ func startRuntimeServersWithBoundListeners( shardStore *kv.ShardStore, coordinate kv.Coordinator, distServer *adapter.DistributionServer, - leaderRedis map[raft.ServerAddress]string, + leaderRedis map[string]string, listeners bootstrapE2EListeners, ) error { if len(runtimes) != 1 { @@ -430,7 +429,7 @@ func startRuntimeServersWithBoundListeners( } rt := runtimes[0] relay := adapter.NewRedisPubSubRelay() - redisAddr := leaderRedis[raft.ServerAddress(rt.spec.address)] + redisAddr := leaderRedis[rt.spec.address] if err := startBoundRedisServer(ctx, eg, listeners.redis, shardStore, coordinate, leaderRedis, redisAddr, relay); err != nil { return waitErrgroupAfterStartupFailure(cancel, eg, err) @@ -509,7 +508,7 @@ func startBoundRedisServer( listener net.Listener, shardStore *kv.ShardStore, coordinate kv.Coordinator, - leaderRedis map[raft.ServerAddress]string, + leaderRedis map[string]string, redisAddr string, relay *adapter.RedisPubSubRelay, ) error { diff --git a/main_bootstrap_test.go b/main_bootstrap_test.go index ecdbf6dba..3a364957c 100644 --- a/main_bootstrap_test.go +++ b/main_bootstrap_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/hashicorp/raft" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/stretchr/testify/require" ) @@ -12,8 +12,8 @@ func TestResolveBootstrapServers(t *testing.T) { t.Run("members imply fixed bootstrap servers", func(t *testing.T) { servers, err := resolveBootstrapServers("n1", []groupSpec{{id: 1, address: "10.0.0.11:50051"}}, "n1=10.0.0.11:50051") require.NoError(t, err) - require.Equal(t, []raft.Server{ - {Suffrage: raft.Voter, ID: "n1", Address: "10.0.0.11:50051"}, + require.Equal(t, []raftengine.Server{ + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, }, servers) }) @@ -30,9 +30,9 @@ func TestResolveBootstrapServers(t *testing.T) { "n1=10.0.0.11:50051,n2=10.0.0.12:50051", ) require.NoError(t, err) - require.Equal(t, []raft.Server{ - {Suffrage: raft.Voter, ID: "n1", Address: "10.0.0.11:50051"}, - {Suffrage: raft.Voter, ID: "n2", Address: "10.0.0.12:50051"}, + require.Equal(t, []raftengine.Server{ + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + {Suffrage: "voter", ID: "n2", Address: "10.0.0.12:50051"}, }, servers) }) diff --git a/main_s3.go b/main_s3.go index 6b28162c8..7fae6f16d 100644 --- a/main_s3.go +++ b/main_s3.go @@ -11,7 +11,6 @@ import ( "github.com/bootjp/elastickv/adapter" "github.com/bootjp/elastickv/kv" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" "golang.org/x/sync/errgroup" ) @@ -31,7 +30,7 @@ func startS3Server( s3Addr string, shardStore *kv.ShardStore, coordinate kv.Coordinator, - leaderS3 map[raft.ServerAddress]string, + leaderS3 map[string]string, region string, credentialsFile string, pathStyleOnly bool, diff --git a/multiraft_runtime.go b/multiraft_runtime.go index f4c50fffb..5d0dbc66c 100644 --- a/multiraft_runtime.go +++ b/multiraft_runtime.go @@ -10,7 +10,6 @@ import ( "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" "google.golang.org/grpc" ) @@ -28,22 +27,18 @@ const raftEngineMarkerPerm = 0o600 type raftEngineType string const ( - raftEngineHashicorp raftEngineType = "hashicorp" raftEngineEtcd raftEngineType = "etcd" raftEngineMarkerFile = "raft-engine" ) var ( - ErrUnsupportedRaftEngine = errors.New("unsupported raft engine") - ErrRaftEngineDataDir = errors.New("raft data dir belongs to a different raft engine") - ErrMixedRaftEngineArtifacts = errors.New("raft data dir contains artifacts for multiple raft engines") + ErrUnsupportedRaftEngine = errors.New("unsupported raft engine") + ErrRaftEngineDataDir = errors.New("raft data dir belongs to a different raft engine") ) func parseRaftEngineType(raw string) (raftEngineType, error) { switch engineType := raftEngineType(strings.ToLower(strings.TrimSpace(raw))); engineType { - case "", raftEngineHashicorp: - return raftEngineHashicorp, nil - case raftEngineEtcd: + case "", raftEngineEtcd: return raftEngineEtcd, nil default: return "", errors.Wrapf(ErrUnsupportedRaftEngine, "%q", raw) @@ -131,14 +126,6 @@ func readRaftEngineMarker(path string) (raftEngineType, bool, error) { } func detectRaftEngineFromDataDir(dir string) (raftEngineType, bool, error) { - hashicorpArtifacts, err := hasRaftArtifacts(dir, - "raft.db", - "logs.dat", - "stable.dat", - ) - if err != nil { - return "", false, err - } etcdArtifacts, err := hasRaftArtifacts(dir, "wal", "snap", @@ -154,16 +141,10 @@ func detectRaftEngineFromDataDir(dir string) (raftEngineType, bool, error) { return "", false, err } - switch { - case hashicorpArtifacts && etcdArtifacts: - return "", false, errors.Wrapf(ErrMixedRaftEngineArtifacts, "%s", dir) - case hashicorpArtifacts: - return raftEngineHashicorp, true, nil - case etcdArtifacts: + if etcdArtifacts { return raftEngineEtcd, true, nil - default: - return "", false, nil } + return "", false, nil } func hasRaftArtifacts(dir string, paths ...string) (bool, error) { @@ -196,25 +177,13 @@ func syncDataDir(path string) error { return nil } -func bootstrapPeersToServers(bootstrapServers []raft.Server) []raftengine.Server { - servers := make([]raftengine.Server, 0, len(bootstrapServers)) - for _, s := range bootstrapServers { - servers = append(servers, raftengine.Server{ - ID: string(s.ID), - Address: string(s.Address), - Suffrage: "voter", - }) - } - return servers -} - func buildRuntimeForGroup( raftID string, group groupSpec, baseDir string, multi bool, bootstrap bool, - bootstrapServers []raft.Server, + bootstrapServers []raftengine.Server, st store.MVCCStore, sm raftengine.StateMachine, factory raftengine.Factory, @@ -229,7 +198,7 @@ func buildRuntimeForGroup( LocalID: raftID, LocalAddress: group.address, DataDir: dir, - Peers: bootstrapPeersToServers(bootstrapServers), + Peers: bootstrapServers, Bootstrap: bootstrap, StateMachine: sm, }) diff --git a/multiraft_runtime_test.go b/multiraft_runtime_test.go index 7a8e9c57a..b5270d96b 100644 --- a/multiraft_runtime_test.go +++ b/multiraft_runtime_test.go @@ -7,7 +7,6 @@ import ( "testing" "github.com/bootjp/elastickv/distribution" - etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" @@ -26,44 +25,11 @@ func TestGroupDataDir(t *testing.T) { }) } -func TestBuildRuntimeForGroupBootstrap(t *testing.T) { - baseDir := t.TempDir() - - factory, err := newRaftFactory(raftEngineHashicorp) - require.NoError(t, err) - - st := store.NewMVCCStore() - sm := etcdraftengine.AdaptHashicorpFSM(kv.NewKvFSMWithHLC(st, kv.NewHLC())) - - runtime, err := buildRuntimeForGroup( - "n1", - groupSpec{id: 1, address: "127.0.0.1:0"}, - baseDir, - true, // multi - true, // bootstrap - nil, - st, - sm, - factory, - ) - require.NoError(t, err) - require.NotNil(t, runtime) - require.NotNil(t, runtime.engine) - t.Cleanup(func() { runtime.Close() }) - - dir := groupDataDir(baseDir, "n1", 1, true) - _, err = os.Stat(dir) - require.NoError(t, err) - - _, err = os.Stat(filepath.Join(dir, "raft.db")) - require.NoError(t, err) -} - func TestParseRaftEngineType(t *testing.T) { t.Run("default", func(t *testing.T) { engineType, err := parseRaftEngineType("") require.NoError(t, err) - require.Equal(t, raftEngineHashicorp, engineType) + require.Equal(t, raftEngineEtcd, engineType) }) t.Run("etcd", func(t *testing.T) { @@ -200,23 +166,6 @@ func TestEnsureRaftEngineDataDir(t *testing.T) { require.Equal(t, "etcd\n", string(data)) }) - t.Run("rejects mismatched existing artifacts", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "raft.db"), []byte("not-a-real-db"), 0o600)) - err := ensureRaftEngineDataDir(dir, raftEngineEtcd) - require.Error(t, err) - require.ErrorIs(t, err, ErrRaftEngineDataDir) - }) - - t.Run("detects mixed engine artifacts", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "raft.db"), []byte("not-a-real-db"), 0o600)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "member", "wal"), 0o755)) - _, _, err := detectRaftEngineFromDataDir(dir) - require.Error(t, err) - require.ErrorIs(t, err, ErrMixedRaftEngineArtifacts) - }) - t.Run("detects etcd peers metadata artifact", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "etcd-raft-peers.bin"), []byte("placeholder"), 0o600)) @@ -245,12 +194,4 @@ func TestEnsureRaftEngineDataDir(t *testing.T) { require.Equal(t, raftEngineEtcd, engineType) }) - t.Run("detects mixed engine artifacts with bare wal", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "raft.db"), []byte("dummy"), 0o600)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "wal"), 0o755)) - _, _, err := detectRaftEngineFromDataDir(dir) - require.Error(t, err) - require.ErrorIs(t, err, ErrMixedRaftEngineArtifacts) - }) } diff --git a/shard_config.go b/shard_config.go index a26c64a9f..a7353038b 100644 --- a/shard_config.go +++ b/shard_config.go @@ -5,8 +5,8 @@ import ( "strconv" "strings" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/cockroachdb/errors" - "github.com/hashicorp/raft" ) type groupSpec struct { @@ -116,20 +116,20 @@ func parseShardRanges(raw string, defaultGroup uint64) ([]rangeSpec, error) { return ranges, nil } -func parseRaftRedisMap(raw string) (map[raft.ServerAddress]string, error) { +func parseRaftRedisMap(raw string) (map[string]string, error) { return parseRaftAddressMap(raw, ErrInvalidRaftRedisMapEntry) } -func parseRaftS3Map(raw string) (map[raft.ServerAddress]string, error) { +func parseRaftS3Map(raw string) (map[string]string, error) { return parseRaftAddressMap(raw, ErrInvalidRaftS3MapEntry) } -func parseRaftDynamoMap(raw string) (map[raft.ServerAddress]string, error) { +func parseRaftDynamoMap(raw string) (map[string]string, error) { return parseRaftAddressMap(raw, ErrInvalidRaftDynamoMapEntry) } -func parseRaftAddressMap(raw string, invalidEntry error) (map[raft.ServerAddress]string, error) { - out := make(map[raft.ServerAddress]string) +func parseRaftAddressMap(raw string, invalidEntry error) (map[string]string, error) { + out := make(map[string]string) if raw == "" { return out, nil } @@ -148,17 +148,17 @@ func parseRaftAddressMap(raw string, invalidEntry error) (map[raft.ServerAddress if k == "" || v == "" { return nil, errors.Wrapf(invalidEntry, "%q", part) } - out[raft.ServerAddress(k)] = v + out[k] = v } return out, nil } -func parseRaftBootstrapMembers(raw string) ([]raft.Server, error) { - servers := make([]raft.Server, 0) +func parseRaftBootstrapMembers(raw string) ([]raftengine.Server, error) { + servers := make([]raftengine.Server, 0) if raw == "" { return servers, nil } - seen := make(map[raft.ServerID]struct{}) + seen := make(map[string]struct{}) parts := strings.SplitSeq(raw, ",") for part := range parts { part = strings.TrimSpace(part) @@ -174,15 +174,14 @@ func parseRaftBootstrapMembers(raw string) ([]raft.Server, error) { if id == "" || addr == "" { return nil, errors.Wrapf(ErrInvalidRaftBootstrapMembersEntry, "%q", part) } - sid := raft.ServerID(id) - if _, exists := seen[sid]; exists { + if _, exists := seen[id]; exists { return nil, errors.Wrapf(ErrInvalidRaftBootstrapMembersEntry, "duplicate id %q", id) } - seen[sid] = struct{}{} - servers = append(servers, raft.Server{ - Suffrage: raft.Voter, - ID: sid, - Address: raft.ServerAddress(addr), + seen[id] = struct{}{} + servers = append(servers, raftengine.Server{ + Suffrage: "voter", + ID: id, + Address: addr, }) } return servers, nil diff --git a/shard_config_test.go b/shard_config_test.go index eebe13cf2..ad037a9bf 100644 --- a/shard_config_test.go +++ b/shard_config_test.go @@ -3,7 +3,7 @@ package main import ( "testing" - "github.com/hashicorp/raft" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/stretchr/testify/require" ) @@ -95,7 +95,7 @@ func TestParseShardRanges(t *testing.T) { func TestParseRaftRedisMap(t *testing.T) { m, err := parseRaftRedisMap("a=b, c=d") require.NoError(t, err) - require.Equal(t, map[raft.ServerAddress]string{ + require.Equal(t, map[string]string{ "a": "b", "c": "d", }, m) @@ -103,7 +103,7 @@ func TestParseRaftRedisMap(t *testing.T) { t.Run("trims whitespace", func(t *testing.T) { m, err := parseRaftRedisMap(" a = b , c = d ") require.NoError(t, err) - require.Equal(t, map[raft.ServerAddress]string{ + require.Equal(t, map[string]string{ "a": "b", "c": "d", }, m) @@ -118,7 +118,7 @@ func TestParseRaftRedisMap(t *testing.T) { func TestParseRaftS3Map(t *testing.T) { m, err := parseRaftS3Map("a=b, c=d") require.NoError(t, err) - require.Equal(t, map[raft.ServerAddress]string{ + require.Equal(t, map[string]string{ "a": "b", "c": "d", }, m) @@ -126,7 +126,7 @@ func TestParseRaftS3Map(t *testing.T) { t.Run("trims whitespace", func(t *testing.T) { m, err := parseRaftS3Map(" a = b , c = d ") require.NoError(t, err) - require.Equal(t, map[raft.ServerAddress]string{ + require.Equal(t, map[string]string{ "a": "b", "c": "d", }, m) @@ -142,9 +142,9 @@ func TestParseRaftBootstrapMembers(t *testing.T) { t.Run("parses members", func(t *testing.T) { members, err := parseRaftBootstrapMembers("n1=10.0.0.11:50051, n2=10.0.0.12:50051") require.NoError(t, err) - require.Equal(t, []raft.Server{ - {Suffrage: raft.Voter, ID: raft.ServerID("n1"), Address: raft.ServerAddress("10.0.0.11:50051")}, - {Suffrage: raft.Voter, ID: raft.ServerID("n2"), Address: raft.ServerAddress("10.0.0.12:50051")}, + require.Equal(t, []raftengine.Server{ + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + {Suffrage: "voter", ID: "n2", Address: "10.0.0.12:50051"}, }, members) }) From 05feede2743d1ca49e8ecaa5489f8afe3acb87bb Mon Sep 17 00:00:00 2001 From: "Yoshiaki Ueda (bootjp)" Date: Thu, 23 Apr 2026 03:43:31 +0900 Subject: [PATCH 2/6] fix(lint): address reviewdog/golangci-lint findings - gci: re-format imports in adapter/redis_info_test.go, cmd/server/demo.go, kv/leader_routed_store_test.go. - noctx: replace net.DialTimeout with (*net.Dialer).DialContext bound to a context in TestLeaderProxy_ForwardsWhenFollower so the wait honours both the dial timeout and the surrounding context. - SA9004: annotate raftEngineMarkerFile with an explicit string type so the grouped const declaration does not silently downgrade it to untyped. - unconvert: drop the redundant string(...) wrapper around lis.Addr().String() in kv/leader_routed_store_test.go. - cyclop: extract openRaftEngine from demo.go run() so the raft engine setup (factory, data dir, bootstrap) lives in its own helper and run() drops below the cyclomatic-complexity threshold. --- adapter/redis_info_test.go | 12 +++--- cmd/server/demo.go | 71 ++++++++++++++++++++-------------- kv/leader_proxy_test.go | 5 ++- kv/leader_routed_store_test.go | 3 +- multiraft_runtime.go | 2 +- 5 files changed, 53 insertions(+), 40 deletions(-) diff --git a/adapter/redis_info_test.go b/adapter/redis_info_test.go index c1381e090..51732aa93 100644 --- a/adapter/redis_info_test.go +++ b/adapter/redis_info_test.go @@ -20,12 +20,12 @@ type infoTestCoordinator struct { func (c *infoTestCoordinator) Dispatch(context.Context, *kv.OperationGroup[kv.OP]) (*kv.CoordinateResponse, error) { return &kv.CoordinateResponse{}, nil } -func (c *infoTestCoordinator) IsLeader() bool { return c.isLeader } -func (c *infoTestCoordinator) VerifyLeader() error { return nil } -func (c *infoTestCoordinator) RaftLeader() string { return c.raftLeader } -func (c *infoTestCoordinator) IsLeaderForKey([]byte) bool { return c.isLeader } -func (c *infoTestCoordinator) VerifyLeaderForKey([]byte) error { return nil } -func (c *infoTestCoordinator) RaftLeaderForKey([]byte) string { return c.raftLeader } +func (c *infoTestCoordinator) IsLeader() bool { return c.isLeader } +func (c *infoTestCoordinator) VerifyLeader() error { return nil } +func (c *infoTestCoordinator) RaftLeader() string { return c.raftLeader } +func (c *infoTestCoordinator) IsLeaderForKey([]byte) bool { return c.isLeader } +func (c *infoTestCoordinator) VerifyLeaderForKey([]byte) error { return nil } +func (c *infoTestCoordinator) RaftLeaderForKey([]byte) string { return c.raftLeader } func (c *infoTestCoordinator) Clock() *kv.HLC { if c.clock == nil { c.clock = kv.NewHLC() diff --git a/cmd/server/demo.go b/cmd/server/demo.go index 8f2f4d69b..1632c93ed 100644 --- a/cmd/server/demo.go +++ b/cmd/server/demo.go @@ -50,18 +50,18 @@ var ( ) const ( - kvParts = 2 - defaultFileMode = 0755 - joinRetries = 20 - joinWait = 3 * time.Second - joinRetryInterval = 1 * time.Second - joinRPCTimeout = 3 * time.Second - raftObserveInterval = 5 * time.Second - demoTickInterval = 10 * time.Millisecond - demoHeartbeatTick = 1 - demoElectionTick = 10 - demoMaxSizePerMsg = 1 << 20 - demoMaxInflightMsg = 256 + kvParts = 2 + defaultFileMode = 0755 + joinRetries = 20 + joinWait = 3 * time.Second + joinRetryInterval = 1 * time.Second + joinRPCTimeout = 3 * time.Second + raftObserveInterval = 5 * time.Second + demoTickInterval = 10 * time.Millisecond + demoHeartbeatTick = 1 + demoElectionTick = 10 + demoMaxSizePerMsg = 1 << 20 + demoMaxInflightMsg = 256 ) func init() { @@ -483,20 +483,11 @@ func setupDynamo(ctx context.Context, lc net.ListenConfig, st store.MVCCStore, c ), nil } -func run(ctx context.Context, eg *errgroup.Group, cfg config) error { - var lc net.ListenConfig - cleanup := internalutil.CleanupStack{} - defer cleanup.Run() - - st, err := setupFSMStore(cfg.raftDataDir, &cleanup) - if err != nil { - return err - } - cleanup.Add(func() { st.Close() }) - hlc := kv.NewHLC() - fsm := kv.NewKvFSMWithHLC(st, hlc) - readTracker := kv.NewActiveTimestampTracker() - +// openRaftEngine creates the etcd-backed raft engine for a demo node. It +// resolves the data dir (using a temp dir when cfg.raftDataDir is empty) +// and registers cleanup callbacks for the data dir, engine and factory +// resources. +func openRaftEngine(cfg config, fsm raftengine.StateMachine, cleanup *internalutil.CleanupStack) (*raftengine.FactoryResult, error) { factory := etcdraftengine.NewFactory(etcdraftengine.FactoryConfig{ TickInterval: demoTickInterval, HeartbeatTick: demoHeartbeatTick, @@ -509,12 +500,12 @@ func run(ctx context.Context, eg *errgroup.Group, cfg config) error { if raftDir == "" { tmp, err := os.MkdirTemp("", "elastickv-raft-*") if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } cleanup.Add(func() { os.RemoveAll(tmp) }) raftDir = tmp } else if err := os.MkdirAll(raftDir, defaultFileMode); err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } result, err := factory.Create(raftengine.FactoryConfig{ @@ -525,7 +516,7 @@ func run(ctx context.Context, eg *errgroup.Group, cfg config) error { StateMachine: fsm, }) if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } cleanup.Add(func() { _ = result.Engine.Close() @@ -533,6 +524,27 @@ func run(ctx context.Context, eg *errgroup.Group, cfg config) error { _ = result.Close() } }) + return result, nil +} + +func run(ctx context.Context, eg *errgroup.Group, cfg config) error { + var lc net.ListenConfig + cleanup := internalutil.CleanupStack{} + defer cleanup.Run() + + st, err := setupFSMStore(cfg.raftDataDir, &cleanup) + if err != nil { + return err + } + cleanup.Add(func() { st.Close() }) + hlc := kv.NewHLC() + fsm := kv.NewKvFSMWithHLC(st, hlc) + readTracker := kv.NewActiveTimestampTracker() + + result, err := openRaftEngine(cfg, fsm, &cleanup) + if err != nil { + return err + } metricsRegistry := monitoring.NewRegistry(cfg.raftID, cfg.address) proposalObserver := metricsRegistry.RaftProposalObserver(1) @@ -681,7 +693,6 @@ func setupPprofHTTPServer(ctx context.Context, lc net.ListenConfig, pprofAddress return pprofL, ps, nil } - func catalogWatcherTask(ctx context.Context, distCatalog *distribution.CatalogStore, distEngine *distribution.Engine) func() error { return func() error { if err := distribution.RunCatalogWatcher(ctx, distCatalog, distEngine, slog.Default()); err != nil { diff --git a/kv/leader_proxy_test.go b/kv/leader_proxy_test.go index 4a16fb09c..ac9a668e2 100644 --- a/kv/leader_proxy_test.go +++ b/kv/leader_proxy_test.go @@ -107,8 +107,11 @@ func TestLeaderProxy_ForwardsWhenFollower(t *testing.T) { }) // Wait briefly so the gRPC server is ready to serve. + dialer := &net.Dialer{Timeout: 100 * time.Millisecond} require.Eventually(t, func() bool { - c, err := net.DialTimeout("tcp", lis.Addr().String(), 100*time.Millisecond) + dialCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + c, err := dialer.DialContext(dialCtx, "tcp", lis.Addr().String()) if err != nil { return false } diff --git a/kv/leader_routed_store_test.go b/kv/leader_routed_store_test.go index c1eb62169..b30a9a440 100644 --- a/kv/leader_routed_store_test.go +++ b/kv/leader_routed_store_test.go @@ -8,7 +8,6 @@ import ( pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" - "github.com/stretchr/testify/require" "google.golang.org/grpc" ) @@ -138,7 +137,7 @@ func startRawKVServer(t *testing.T, svc pb.RawKVServer) (string, func()) { grpcServer.Stop() _ = lis.Close() } - return string(lis.Addr().String()), stop + return lis.Addr().String(), stop } func TestLeaderRoutedStore_UsesLocalStoreWhenLeaderVerified(t *testing.T) { diff --git a/multiraft_runtime.go b/multiraft_runtime.go index 5d0dbc66c..754e6e037 100644 --- a/multiraft_runtime.go +++ b/multiraft_runtime.go @@ -28,7 +28,7 @@ type raftEngineType string const ( raftEngineEtcd raftEngineType = "etcd" - raftEngineMarkerFile = "raft-engine" + raftEngineMarkerFile string = "raft-engine" ) var ( From c5dc58a52330156451200653fd0ac22f7fbb0144 Mon Sep 17 00:00:00 2001 From: "Yoshiaki Ueda (bootjp)" Date: Thu, 23 Apr 2026 04:17:22 +0900 Subject: [PATCH 3/6] test(adapter): transfer leadership to node 0 after election MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Existing grpc_test.go helpers (rawKVClient, transactionalKVClient) connect to hosts[0] and assume it is the cluster leader. The previous hashicorp setup enforced this with an electionTimeout=0 on node 0 while others waited 10s. With etcd/raft, election timing is randomised and any node can win, so the scan in Test_grpc_scan could race against a forwarded Put that had not yet applied on the contacted follower — the last Put's LastCommitTS hadn't propagated locally. ensureNodeZeroIsLeader waits for any leader to emerge, then asks the current leader to transfer leadership to node 0 via raftengine.Admin.TransferLeadershipToServer. The transfer is best-effort and the subsequent Eventually retries if a concurrent election races it. --- adapter/test_util.go | 53 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/adapter/test_util.go b/adapter/test_util.go index b8d60d712..41f95361f 100644 --- a/adapter/test_util.go +++ b/adapter/test_util.go @@ -235,6 +235,13 @@ func waitForNodeListeners(t *testing.T, ctx context.Context, nodes []Node, waitT func waitForRaftReadiness(t *testing.T, nodes []Node, peers []raftengine.Server, waitTimeout, waitInterval time.Duration) { t.Helper() + // Existing tests assume hosts[0] (node 0) is the cluster leader — the + // previous hashicorp setup ensured this by giving node 0 an immediate + // election timeout while others waited 10s. etcd/raft elections are + // randomised, so whoever wins the first election is effectively random. + // Nudge leadership onto node 0 if a different node won. + ensureNodeZeroIsLeader(t, nodes, peers, waitTimeout, waitInterval) + assert.Eventually(t, func() bool { var leaderAddr string for _, n := range nodes { @@ -258,6 +265,52 @@ func waitForRaftReadiness(t *testing.T, nodes []Node, peers []raftengine.Server, }, waitTimeout, waitInterval) } +// ensureNodeZeroIsLeader waits for any node to become leader, then (if +// necessary) triggers a leadership transfer to node 0 so downstream +// tests that assume hosts[0] is authoritative keep working. +func ensureNodeZeroIsLeader(t *testing.T, nodes []Node, peers []raftengine.Server, waitTimeout, waitInterval time.Duration) { + t.Helper() + + if len(nodes) == 0 || len(peers) == 0 { + return + } + targetAddr := peers[0].Address + + // Step 1: wait until some node is leader so we know the cluster is live. + assert.Eventually(t, func() bool { + for _, n := range nodes { + if n.engine.State() == raftengine.StateLeader { + return true + } + } + return false + }, waitTimeout, waitInterval, "no node became leader") + + // Step 2: if node 0 isn't already leader, ask the current leader to + // transfer leadership to it. This is best-effort — a transfer can + // race with another election, in which case the Eventually below + // will retry. + assert.Eventually(t, func() bool { + if nodes[0].engine.State() == raftengine.StateLeader { + return true + } + for _, n := range nodes { + if n.engine.State() != raftengine.StateLeader { + continue + } + admin, ok := n.engine.(raftengine.Admin) + if !ok { + return false + } + transferCtx, cancel := context.WithTimeout(context.Background(), waitInterval) + _ = admin.TransferLeadershipToServer(transferCtx, peers[0].ID, targetAddr) + cancel() + break + } + return false + }, waitTimeout, waitInterval, "node 0 did not become leader") +} + func assignPorts(n int) []portsAdress { ports := make([]portsAdress, n) for i := range n { From 02782f08228ce8643709cfcf2f338a7fb29aeeff Mon Sep 17 00:00:00 2001 From: "Yoshiaki Ueda (bootjp)" Date: Thu, 23 Apr 2026 04:40:53 +0900 Subject: [PATCH 4/6] fix(raft): address codex review on peers and legacy-artifact rejection Codex flagged two P1 issues on the initial drop-hashicorp commit: (1) cmd/server/demo.go was calling factory.Create without seeding Peers. In demo mode the non-bootstrap nodes (n2, n3) would hit etcd's "no persisted peers and no peer list was supplied" guard and abort before joinCluster ever ran. Build the full 3-node peer list in main() and pass it as cfg.raftPeers so every node bootstraps with the same membership; raftBootstrap is forced true for all demo nodes so etcd forms the cluster directly. (2) detectRaftEngineFromDataDir no longer noticed hashicorp artifacts (raft.db, logs.dat, stable.dat), so on upgrade an existing hashicorp data dir would silently be rewritten as an etcd cluster on top of the committed bolt state. Add hasHashicorpRaftArtifacts plus a new ErrLegacyHashicorpDataDir sentinel, and call the probe from ensureRaftEngineDataDir before any marker is written so production restarts fail fast with a migration-required error. demo.go's openRaftEngine runs the same check on its own data dir for the P2 follow-up codex raised there. --- cmd/server/demo.go | 37 ++++++++++++++++++++++++++++++++++--- multiraft_runtime.go | 24 ++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/cmd/server/demo.go b/cmd/server/demo.go index 1632c93ed..bfd84ddd3 100644 --- a/cmd/server/demo.go +++ b/cmd/server/demo.go @@ -88,6 +88,11 @@ type config struct { raftRedisMap string raftS3Map string raftDynamoMap string + // raftPeers, when non-empty, seeds the etcd raft factory with the + // full cluster membership. Demo mode populates it for every node so + // non-bootstrap nodes start with a known peer list instead of + // failing with "no persisted peers and no peer list was supplied". + raftPeers []raftengine.Server } func main() { @@ -177,10 +182,16 @@ func main() { var redisMapParts []string var s3MapParts []string var dynamoMapParts []string + peers := make([]raftengine.Server, 0, len(nodes)) for _, n := range nodes { redisMapParts = append(redisMapParts, n.address+"="+n.redisAddress) s3MapParts = append(s3MapParts, n.address+"="+n.s3Address) dynamoMapParts = append(dynamoMapParts, n.address+"="+n.dynamoAddress) + peers = append(peers, raftengine.Server{ + Suffrage: "voter", + ID: n.raftID, + Address: n.address, + }) } raftRedisMapStr := strings.Join(redisMapParts, ",") raftS3MapStr := strings.Join(s3MapParts, ",") @@ -190,6 +201,12 @@ func main() { n.raftRedisMap = raftRedisMapStr n.raftS3Map = raftS3MapStr n.raftDynamoMap = raftDynamoMapStr + n.raftPeers = peers + // etcd raft requires every member of a fresh cluster to + // bootstrap with the same peer list. Override the per-node + // raftBootstrap flag in demo mode so n2/n3 don't fail with + // "no persisted peers and no peer list was supplied". + n.raftBootstrap = true cfg := n // capture loop variable if err := run(runCtx, eg, cfg); err != nil { slog.Error(err.Error()) @@ -484,9 +501,9 @@ func setupDynamo(ctx context.Context, lc net.ListenConfig, st store.MVCCStore, c } // openRaftEngine creates the etcd-backed raft engine for a demo node. It -// resolves the data dir (using a temp dir when cfg.raftDataDir is empty) -// and registers cleanup callbacks for the data dir, engine and factory -// resources. +// resolves the data dir (using a temp dir when cfg.raftDataDir is empty), +// refuses to start on top of legacy hashicorp/raft state, and registers +// cleanup callbacks for the data dir, engine and factory resources. func openRaftEngine(cfg config, fsm raftengine.StateMachine, cleanup *internalutil.CleanupStack) (*raftengine.FactoryResult, error) { factory := etcdraftengine.NewFactory(etcdraftengine.FactoryConfig{ TickInterval: demoTickInterval, @@ -508,10 +525,24 @@ func openRaftEngine(cfg config, fsm raftengine.StateMachine, cleanup *internalut return nil, errors.WithStack(err) } + // Refuse to start on top of hashicorp/raft artifacts from a previous + // deployment. The backend has been removed and silently overwriting + // its state with etcd markers could commit to an incompatible engine + // over committed data. Fail fast so operators have to migrate + // explicitly. + for _, legacy := range []string{"raft.db", "logs.dat", "stable.dat"} { + if _, err := os.Stat(filepath.Join(raftDir, legacy)); err == nil { + return nil, errors.WithStack(errors.Newf("legacy hashicorp/raft artifact %q found in %s; hashicorp backend has been removed, manual migration required", legacy, raftDir)) + } else if !os.IsNotExist(err) { + return nil, errors.WithStack(err) + } + } + result, err := factory.Create(raftengine.FactoryConfig{ LocalID: cfg.raftID, LocalAddress: cfg.address, DataDir: raftDir, + Peers: cfg.raftPeers, Bootstrap: cfg.raftBootstrap, StateMachine: fsm, }) diff --git a/multiraft_runtime.go b/multiraft_runtime.go index 754e6e037..30b4a0e52 100644 --- a/multiraft_runtime.go +++ b/multiraft_runtime.go @@ -32,8 +32,9 @@ const ( ) var ( - ErrUnsupportedRaftEngine = errors.New("unsupported raft engine") - ErrRaftEngineDataDir = errors.New("raft data dir belongs to a different raft engine") + ErrUnsupportedRaftEngine = errors.New("unsupported raft engine") + ErrRaftEngineDataDir = errors.New("raft data dir belongs to a different raft engine") + ErrLegacyHashicorpDataDir = errors.New("raft data dir contains legacy hashicorp/raft artifacts; hashicorp backend has been removed, manual migration to etcd is required") ) func parseRaftEngineType(raw string) (raftEngineType, error) { @@ -90,6 +91,17 @@ func ensureRaftEngineDataDir(dir string, engineType raftEngineType) error { return errors.WithStack(err) } + // Refuse to start on top of a dir that still holds hashicorp/raft + // artifacts. The hashicorp backend has been removed and silently + // overwriting its state with etcd markers would commit to an + // incompatible engine over committed data. Fail fast; operators must + // migrate the dir explicitly before restarting. + if hashicorpArtifacts, err := hasHashicorpRaftArtifacts(dir); err != nil { + return err + } else if hashicorpArtifacts { + return errors.Wrapf(ErrLegacyHashicorpDataDir, "%s", dir) + } + markerPath := filepath.Join(dir, raftEngineMarkerFile) if current, ok, err := readRaftEngineMarker(markerPath); err != nil { return err @@ -110,6 +122,14 @@ func ensureRaftEngineDataDir(dir string, engineType raftEngineType) error { return writeRaftEngineMarker(markerPath, engineType) } +// hasHashicorpRaftArtifacts reports whether dir contains any files that +// were produced by the removed hashicorp/raft backend (raft.db plus the +// boltdb log/stable files). Used to refuse startup on a legacy data dir +// rather than silently overwriting it with an etcd-shaped cluster. +func hasHashicorpRaftArtifacts(dir string) (bool, error) { + return hasRaftArtifacts(dir, "raft.db", "logs.dat", "stable.dat") +} + func readRaftEngineMarker(path string) (raftEngineType, bool, error) { data, err := os.ReadFile(path) if err != nil { From a8a47f6df6cbb1b3fdf82866b3495275033321e3 Mon Sep 17 00:00:00 2001 From: "Yoshiaki Ueda (bootjp)" Date: Thu, 23 Apr 2026 04:53:25 +0900 Subject: [PATCH 5/6] fix(tests): address coderabbit review on cleanup + assertion severity - adapter/distribution_milestone1_e2e_test.go: register the engine/factory cleanup with t.Cleanup BEFORE the require.Eventually leader-wait so a timeout no longer leaks the etcd engine, WAL and transport. The stop closure is now sync.Once-protected so the explicit defer-stop in test bodies plus the t.Cleanup fallback are both safe. - adapter/test_util.go: promote every assert.Eventually in waitForRaftReadiness / ensureNodeZeroIsLeader to require.Eventually so a missing leader or a failed leadership transfer fails the test immediately instead of letting subsequent RPCs panic on nil leader. The lingering assert.NoError calls in background goroutines stay because require.FailNow there would leak the goroutine. - multiraft_runtime_test.go: add three subtests (raft.db, logs.dat, stable.dat) that assert ensureRaftEngineDataDir returns ErrLegacyHashicorpDataDir, covering the no-silent-migration guard added in the previous commit. --- adapter/distribution_milestone1_e2e_test.go | 20 ++++++++++++++------ adapter/test_util.go | 8 ++++---- multiraft_runtime_test.go | 21 +++++++++++++++++++++ 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/adapter/distribution_milestone1_e2e_test.go b/adapter/distribution_milestone1_e2e_test.go index e6af8b151..d85549402 100644 --- a/adapter/distribution_milestone1_e2e_test.go +++ b/adapter/distribution_milestone1_e2e_test.go @@ -3,6 +3,7 @@ package adapter import ( "context" "path/filepath" + "sync" "testing" "time" @@ -246,15 +247,22 @@ func newSingleRaftForDistributionE2E(t *testing.T, id string, fsm raftengine.Sta }) require.NoError(t, err) + // Register cleanup before the leader-wait so an Eventually timeout + // still closes the engine and the factory-owned transport / WAL. + var stopOnce sync.Once + stop := func() { + stopOnce.Do(func() { + _ = result.Engine.Close() + if result.Close != nil { + _ = result.Close() + } + }) + } + t.Cleanup(stop) + require.Eventually(t, func() bool { return result.Engine.State() == raftengine.StateLeader }, 5*time.Second, 10*time.Millisecond) - stop := func() { - require.NoError(t, result.Engine.Close()) - if result.Close != nil { - require.NoError(t, result.Close()) - } - } return result.Engine, stop } diff --git a/adapter/test_util.go b/adapter/test_util.go index 41f95361f..2a4805b22 100644 --- a/adapter/test_util.go +++ b/adapter/test_util.go @@ -216,7 +216,7 @@ func waitForNodeListeners(t *testing.T, ctx context.Context, nodes []Node, waitT t.Helper() d := &net.Dialer{Timeout: time.Second} for _, n := range nodes { - assert.Eventually(t, func() bool { + require.Eventually(t, func() bool { conn, err := d.DialContext(ctx, "tcp", n.grpcAddress) if err != nil { return false @@ -242,7 +242,7 @@ func waitForRaftReadiness(t *testing.T, nodes []Node, peers []raftengine.Server, // Nudge leadership onto node 0 if a different node won. ensureNodeZeroIsLeader(t, nodes, peers, waitTimeout, waitInterval) - assert.Eventually(t, func() bool { + require.Eventually(t, func() bool { var leaderAddr string for _, n := range nodes { leader := n.engine.Leader().Address @@ -277,7 +277,7 @@ func ensureNodeZeroIsLeader(t *testing.T, nodes []Node, peers []raftengine.Serve targetAddr := peers[0].Address // Step 1: wait until some node is leader so we know the cluster is live. - assert.Eventually(t, func() bool { + require.Eventually(t, func() bool { for _, n := range nodes { if n.engine.State() == raftengine.StateLeader { return true @@ -290,7 +290,7 @@ func ensureNodeZeroIsLeader(t *testing.T, nodes []Node, peers []raftengine.Serve // transfer leadership to it. This is best-effort — a transfer can // race with another election, in which case the Eventually below // will retry. - assert.Eventually(t, func() bool { + require.Eventually(t, func() bool { if nodes[0].engine.State() == raftengine.StateLeader { return true } diff --git a/multiraft_runtime_test.go b/multiraft_runtime_test.go index b5270d96b..7b8e156fc 100644 --- a/multiraft_runtime_test.go +++ b/multiraft_runtime_test.go @@ -194,4 +194,25 @@ func TestEnsureRaftEngineDataDir(t *testing.T) { require.Equal(t, raftEngineEtcd, engineType) }) + t.Run("rejects legacy hashicorp raft.db", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "raft.db"), []byte("legacy"), 0o600)) + err := ensureRaftEngineDataDir(dir, raftEngineEtcd) + require.ErrorIs(t, err, ErrLegacyHashicorpDataDir) + }) + + t.Run("rejects legacy hashicorp logs.dat", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "logs.dat"), []byte("legacy"), 0o600)) + err := ensureRaftEngineDataDir(dir, raftEngineEtcd) + require.ErrorIs(t, err, ErrLegacyHashicorpDataDir) + }) + + t.Run("rejects legacy hashicorp stable.dat", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "stable.dat"), []byte("legacy"), 0o600)) + err := ensureRaftEngineDataDir(dir, raftEngineEtcd) + require.ErrorIs(t, err, ErrLegacyHashicorpDataDir) + }) + } From 83e5e8c2bddf63b1a99644789c0de9e7cd058930 Mon Sep 17 00:00:00 2001 From: "Yoshiaki Ueda (bootjp)" Date: Thu, 23 Apr 2026 04:56:16 +0900 Subject: [PATCH 6/6] fix(demo): drop AddVoter-based joinCluster after prebootstrap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex P1: once every demo node starts with raftPeers=[n1,n2,n3] and raftBootstrap=true (the previous commit's fix for the 'no persisted peers' abort), the 3-node cluster is already formed at startup. The lingering joinCluster goroutine then ran AddVoter against nodes[0], which under etcd's randomised elections is often a follower — the RPC would be rejected, retried for the full 20-attempt window, and either leak cancellation errors into the errgroup or attempt to re-add members that were already voters. Remove joinCluster plus its helpers (joinNodeWithRetry, joinRetryCancelResult, tryJoinNode, waitForJoinRetry, joinClusterWaitError) and the now-dead joinRetries / joinWait / joinRetryInterval / joinRPCTimeout constants. The unused fmt import is dropped with them. --- cmd/server/demo.go | 135 +++------------------------------------------ 1 file changed, 8 insertions(+), 127 deletions(-) diff --git a/cmd/server/demo.go b/cmd/server/demo.go index bfd84ddd3..a7ed51ae3 100644 --- a/cmd/server/demo.go +++ b/cmd/server/demo.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "flag" - "fmt" "io" "log/slog" "net" @@ -52,10 +51,6 @@ var ( const ( kvParts = 2 defaultFileMode = 0755 - joinRetries = 20 - joinWait = 3 * time.Second - joinRetryInterval = 1 * time.Second - joinRPCTimeout = 3 * time.Second raftObserveInterval = 5 * time.Second demoTickInterval = 10 * time.Millisecond demoHeartbeatTick = 1 @@ -214,39 +209,14 @@ func main() { } } - // Wait for n1 to be ready then join others? - // Actually, standard bootstrap expects a configuration. - // If we only bootstrap n1, we need to join n2 and n3. - // For simplicity in this demo, let's bootstrap n1 with just n1, and have n2/n3 join. - // Or better: bootstrap n1 with {n1, n2, n3}. - // But run() logic for bootstrap only adds *raftID to configuration. - - // Let's modify bootstrapping logic in run() slightly or just rely on manual join? - // The original demo likely used raftadmin to join or predefined bootstrap. - // Since we can't easily change run() logic too much without breaking Jepsen, - // let's use a separate goroutine to join n2/n3 to n1 after a delay. - - eg.Go(func() error { - // Wait a bit for n1 to start - // This is hacky but sufficient for a demo - // Better would be to wait for gRPC readiness - // But standard 'sleep' is unavailable here without import time - // We can use a simple retry loop to join. - - // Actually, let's keep it simple: just start them. - // If n1 bootstraps as a single node cluster, n2 and n3 won't be part of it automatically. - // We need to issue 'add_voter' commands. - // Let's rely on an external script or add a helper here? - - // For this specific demo restoration, we'll assume the external script might handle joins - // OR we check if the CI script does it. - // The CI script just waits for ports. It runs `lein run ...` which assumes a cluster. - // If the cluster isn't formed, the tests might fail. - // BUT, looking at the previous demo.go (if I could), it probably did the joins. - - // Let's add a joiner goroutine. - return joinCluster(runCtx, nodes) - }) + // All three nodes were started with the same raftPeers list and + // raftBootstrap=true above, so the etcd cluster is fully formed + // at startup. joinCluster (AddVoter via raftadmin against + // nodes[0]) is no longer needed and would actively misbehave + // under etcd's randomised elections — nodes[0] is not guaranteed + // to be leader, so the AddVoter RPC would either hit a follower + // and be rejected, or add duplicates to an already-complete + // configuration. } if err := eg.Wait(); err != nil { @@ -263,95 +233,6 @@ func effectiveDemoMetricsToken(token string) string { return "demo-metrics-token" } -func joinCluster(ctx context.Context, nodes []config) error { - leader := nodes[0] - // Give servers some time to start - if err := waitForJoinRetry(ctx, joinWait); err != nil { - return joinClusterWaitError(err) - } - - // Connect to leader - conn, err := grpc.NewClient(leader.address, internalutil.GRPCDialOptions()...) - if err != nil { - return fmt.Errorf("failed to dial leader: %w", err) - } - defer conn.Close() - client := pb.NewRaftAdminClient(conn) - - for _, n := range nodes[1:] { - if err := joinNodeWithRetry(ctx, client, n); err != nil { - return err - } - } - return nil -} - -func joinNodeWithRetry(ctx context.Context, client pb.RaftAdminClient, n config) error { - for i := range joinRetries { - if err := tryJoinNode(ctx, client, n); err == nil { - return nil - } else { - if ctx.Err() != nil { - // Retry loop should stop immediately once the parent context is canceled. - return joinRetryCancelResult(ctx) - } - slog.Warn("Failed to join node, retrying...", "id", n.raftID, "err", err) - } - if i == joinRetries-1 { - break - } - if err := waitForJoinRetry(ctx, joinRetryInterval); err != nil { - return joinRetryCancelResult(ctx) - } - } - if ctx.Err() != nil { - return joinRetryCancelResult(ctx) - } - return fmt.Errorf("failed to join node %s after retries", n.raftID) -} - -func joinRetryCancelResult(ctx context.Context) error { - if ctx == nil || ctx.Err() == nil { - return nil - } - return joinClusterWaitError(errors.WithStack(ctx.Err())) -} - -func tryJoinNode(ctx context.Context, client pb.RaftAdminClient, n config) error { - slog.Info("Attempting to join node", "id", n.raftID, "address", n.address) - addCtx, cancelAdd := context.WithTimeout(ctx, joinRPCTimeout) - defer cancelAdd() - _, err := client.AddVoter(addCtx, &pb.RaftAdminAddVoterRequest{ - Id: n.raftID, - Address: n.address, - PreviousIndex: 0, - }) - if err != nil { - return errors.WithStack(err) - } - slog.Info("Successfully joined node", "id", n.raftID) - return nil -} - -func waitForJoinRetry(ctx context.Context, delay time.Duration) error { - timer := time.NewTimer(delay) - defer timer.Stop() - select { - case <-ctx.Done(): - return errors.WithStack(ctx.Err()) - case <-timer.C: - return nil - } -} - -func joinClusterWaitError(err error) error { - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - // Do not override the original errgroup cause with cancellation. - return nil - } - return err -} - // setupFSMStore creates and returns the MVCCStore for the Raft FSM. // When raftDataDir is non-empty the store is persisted under that directory; // otherwise a temporary directory is used and registered for cleanup on exit.