Complete centralized TSO runtime operations

adapter/internal.go

@@ -101,11 +101,17 @@ func (i *Internal) stampTimestamps(ctx context.Context, req *pb.ForwardRequest)
 func (i *Internal) nextTimestamp(ctx context.Context, label string) (uint64, error) {
 	if i.tsAllocator != nil {
 		ts, err := i.tsAllocator.Next(ctx)
-		if err != nil {
+		if err == nil {
+			return ts, nil
+		}
+		if !errors.Is(err, kv.ErrTSOAllocatorRequired) {
 			return 0, errors.Wrap(err, label)
 		}
-		return ts, nil
 	}
+	return i.nextLegacyTimestamp(label)
+}
+
+func (i *Internal) nextLegacyTimestamp(label string) (uint64, error) {
 	if i.clock == nil {
 		return 1, nil
 	}
@@ -121,15 +127,23 @@ func (i *Internal) nextTimestampAfter(ctx context.Context, min uint64, label str
 		return 0, errors.Wrap(kv.ErrTxnCommitTSRequired, label)
 	}
 	if i.tsAllocator != nil {
-		return i.nextTimestampAfterFromAllocator(ctx, min, label)
+		ts, err := i.nextTimestampAfterFromAllocator(ctx, min, label)
+		if err == nil {
+			return ts, nil
+		}
+		if !errors.Is(err, kv.ErrTSOAllocatorRequired) {
+			return 0, err
+		}
 	}
+	return i.nextLegacyTimestampAfter(min, label)
+}
+
+func (i *Internal) nextLegacyTimestampAfter(min uint64, label string) (uint64, error) {
 	if i.clock == nil {
 		return min + 1, nil
 	}
-	if i.clock != nil {
-		i.clock.Observe(min)
-	}
-	ts, err := i.nextTimestamp(ctx, label)
+	i.clock.Observe(min)
+	ts, err := i.nextLegacyTimestamp(label)
 	if err != nil {
 		return 0, err
 	}

adapter/internal_test.go

@@ -4,9 +4,11 @@ import (
 	"context"
 	"encoding/binary"
 	"testing"
+	"time"
 
 	"github.com/bootjp/elastickv/kv"
 	pb "github.com/bootjp/elastickv/proto"
+	"github.com/cockroachdb/errors"
 	"github.com/stretchr/testify/require"
 )
 
@@ -83,6 +85,35 @@ func TestFillForwardedTxnCommitTS_AssignsCommitTS(t *testing.T) {
 	require.Equal(t, meta.CommitTS, commitTS)
 }
 
+func TestFillForwardedTxnCommitTS_FallsBackWhenRuntimeAllocatorLegacy(t *testing.T) {
+	t.Parallel()
+
+	clock := kv.NewHLC()
+	clock.SetPhysicalCeiling(time.Now().Add(time.Minute).UnixMilli())
+	i := &Internal{
+		clock:       clock,
+		tsAllocator: internalLegacyRuntimeAllocator{},
+	}
+	startTS := uint64(10)
+	reqs := []*pb.Request{
+		{
+			IsTxn: true,
+			Phase: pb.Phase_COMMIT,
+			Mutations: []*pb.Mutation{
+				{
+					Op:    pb.Op_PUT,
+					Key:   []byte(kv.TxnMetaPrefix),
+					Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("k"), CommitTS: 0}),
+				},
+			},
+		},
+	}
+
+	commitTS, err := i.fillForwardedTxnCommitTS(context.Background(), reqs, startTS)
+	require.NoError(t, err)
+	require.Greater(t, commitTS, startTS)
+}
+
 func TestFillForwardedTxnCommitTS_PreservesExistingCommitTS(t *testing.T) {
 	t.Parallel()
 
@@ -175,6 +206,21 @@ func TestFillForwardedTxnCommitTS_StampsCommitTSValueOffset(t *testing.T) {
 	require.Zero(t, reqs[0].Mutations[1].CommitTsValueOffset)
 }
 
+func TestStampRawTimestamps_FallsBackWhenRuntimeAllocatorLegacy(t *testing.T) {
+	t.Parallel()
+
+	clock := kv.NewHLC()
+	clock.SetPhysicalCeiling(time.Now().Add(time.Minute).UnixMilli())
+	i := &Internal{
+		clock:       clock,
+		tsAllocator: internalLegacyRuntimeAllocator{},
+	}
+	reqs := []*pb.Request{{Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("k"), Value: []byte("v")}}}}
+
+	require.NoError(t, i.stampRawTimestamps(context.Background(), reqs))
+	require.NotZero(t, reqs[0].Ts)
+}
+
 func TestFillForwardedTxnCommitTS_PrepareAllowsAlreadyStampedOffsets(t *testing.T) {
 	t.Parallel()
 
@@ -241,3 +287,9 @@ func TestStampTxnTimestamps_UsesSingleTxnStartTS(t *testing.T) {
 	require.Greater(t, meta.CommitTS, uint64(9))
 	require.Equal(t, meta.CommitTS, commitTS)
 }
+
+type internalLegacyRuntimeAllocator struct{}
+
+func (internalLegacyRuntimeAllocator) Next(context.Context) (uint64, error) {
+	return 0, errors.WithStack(kv.ErrTSOAllocatorRequired)
+}

docs/centralized_tso_operations.md

@@ -0,0 +1,115 @@
+# Centralized TSO Operations
+
+This runbook covers runtime mode changes, production signals, and the rollout
+gates for the dedicated group-0 timestamp oracle. The implementation design is
+in `docs/design/2026_04_16_implemented_centralized_tso.md`.
+
+## Runtime configuration
+
+Start every node with the same atomically replaceable mode file:
+
+```text
+--tsoModeFile=/etc/elastickv/tso-mode
+--tsoModeReloadInterval=5s
+--tsoBatchSize=256
+```
+
+The file contains exactly one mode: `legacy`, `shadow`, `cutover`, or
+`phase-d`. `--tsoModeFile` cannot be combined with `--tsoEnabled`,
+`--tsoShadowEnabled`, or `--tsoPhaseDEnabled`; those startup-only flags remain
+for backward compatibility. Runtime mode reload requires the dedicated group 0.
+
+Replace the file atomically so a poll cannot observe a truncated value:
+
+```sh
+printf '%s\n' shadow > /etc/elastickv/tso-mode.tmp
+mv /etc/elastickv/tso-mode.tmp /etc/elastickv/tso-mode
+```
+
+An unreadable file, unknown value, backward transition, or skipped phase is
+rejected without changing the live allocator. Runtime transitions must be
+adjacent and one-way:
+
+```text
+legacy -> shadow -> cutover -> phase-d
+```
+
+The durable group-0 markers override stale local configuration. Once cutover
+or Phase D is committed, a node cannot return to legacy or shadow issuance even
+if its mode file moves backward. An allocation already in flight during a
+reload may finish under the preceding process mode, but cutover-side requests
+still use group 0 and the durable markers never move backward.
+
+## Rollout gates
+
+1. Deploy the binary and group 0 while every mode file remains `legacy`.
+2. Change every node to `shadow`. Confirm all nodes expose
+   `elastickv_tso_mode{mode="shadow"} == 1`.
+3. Hold shadow mode until `legacy_overlap` remains zero for a full 15-minute
+   window, allocation errors remain below 1 percent, and allocation p99 remains
+   below 50 ms.
+4. Change nodes to `cutover`. The first production reservation commits the
+   one-way cutover marker. A node still in shadow observes that marker and
+   returns the dedicated TSO value.
+5. Confirm every binary supports Phase D and every node is on the dedicated
+   path. Then change nodes to `phase-d`. The first Phase-D reservation commits
+   its floor marker before returning a new window.
+
+After the cutover marker commits, rollback means restoring service around the
+dedicated TSO path; it never means returning to legacy issuance. Phase D has the
+same one-way rule and additionally keeps data-group HLC renewal retired.
+
+## Metrics and alerts
+
+| Signal | Meaning | Gate or threshold |
+|---|---|---|
+| `elastickv_tso_request_duration_seconds` | Local/remote reserve and validation attempt latency, split by outcome | Warning at reserve p99 > 50 ms for 10m; critical at > 200 ms for 5m |
+| `elastickv_tso_shadow_comparisons_total` | Accepted candidates, overlap discards, cutover bypasses, and errors | `legacy_overlap` must be zero for 15m before cutover |
+| `elastickv_tso_shadow_divergence_timestamps` | Absolute candidate-to-floor distance for accepted and discarded shadow comparisons | Investigate sustained growth before cutover |
+| `elastickv_tso_mode` | One-hot process-local mode | More than one mode for 15m warns about a stalled rollout |
+| `elastickv_tso_mode_reload_total` | Applied and rejected reloads plus file read/parse failures | Any failure in 10m warns |
+| `elastickv_tso_durable_state` | Applied `cutover` and `phase_d` markers | Phase D without cutover is critical |
+
+The checked rules are in `monitoring/prometheus/rules/tso-alerts.yml`. Allocation
+errors warn above 1 percent for 10 minutes and become critical above 10 percent
+for 5 minutes. The expressions require at least 0.1 reserve attempts/second so
+an idle cluster does not alert on an empty denominator.
+
+## Write-fanout benchmark
+
+Run the modeled remote-TSO benchmark with:
+
+```sh
+go test ./kv -run '^$' -bench '^BenchmarkTSOWriteFanout$' -benchmem -benchtime=1s -count=3
+```
+
+The harness uses 16 concurrent write coordinators sharing a `BatchAllocator`.
+Each operation stamps 1, 3, or 8 fan-out legs, and each refill pays a modeled
+1 ms remote-leader plus Raft-commit delay. The following medians were measured
+on Go 1.26.5, darwin/arm64, Apple M1 Max:
+
+| Fan-out | Batch | Wall time/op | p99 | Refills/op | Timestamp writes/s |
+|---:|---:|---:|---:|---:|---:|
+| 1 | 1 | 1.33 ms | 1,249 ms | 1.000 | 753 |
+| 1 | 64 | 24.81 us | 1.323 ms | 0.01563 | 40,309 |
+| 1 | 256 | 8.92 us | 0.000292 ms | 0.00391 | 112,081 |
+| 3 | 1 | 5.70 ms | 1,233 ms | 3.000 | 526 |
+| 3 | 64 | 105.35 us | 3.212 ms | 0.04690 | 28,478 |
+| 3 | 256 | 17.30 us | 1.272 ms | 0.01172 | 173,453 |
+| 8 | 1 | 10.43 ms | 1,104 ms | 8.000 | 767 |
+| 8 | 64 | 157.49 us | 1.316 ms | 0.1251 | 50,796 |
+| 8 | 256 | 39.43 us | 1.302 ms | 0.03127 | 202,870 |
+
+Batch size 1 is intentionally retained as the unamortized baseline and shows
+severe waiter tails under contention. The production default of 256 keeps the
+modeled fan-out p99 below 4 ms in this harness, including the refill-contention
+tail at fan-out 8, and below the 50 ms warning threshold. These are controlled
+local measurements, not a substitute for observing the production histograms
+during rollout.
+
+## Intentional non-goals
+
+This closure does not automate cluster-wide phase orchestration, choose a
+dedicated subset of TSO members, or change the HLC ceiling formula. Operators
+still advance the shared mode file deployment-by-deployment, and the existing
+group membership and clock-floor decisions remain independent future work.

docs/design/2026_04_16_partial_centralized_tso.md → docs/design/2026_04_16_implemented_centralized_tso.md

@@ -1,13 +1,16 @@
 # Centralized Timestamp Oracle (TSO) Design
 
-- Status: Partial — M1-M7 are implemented, including the dedicated group-0
-  FSM, leader-routed durable windows, strict term bootstrap, serialized shadow
-  migration, one-way rolling cutover, durable Phase-D retirement, and
-  cross-shard SSI timestamp validation. Runtime config reload and production
-  latency/alerting work remain open.
-- Author: bootjp
-- Date: 2026-04-16
-- Updated: 2026-07-24
+Status: Implemented
+Author: bootjp
+Date: 2026-04-16
+Updated: 2026-07-24
+
+M1-M8 implement the central subsystem: the dedicated group-0 FSM,
+leader-routed durable windows, strict term bootstrap, serialized shadow
+migration, one-way cutover and Phase-D retirement, validated cross-shard
+timestamps, one-way runtime mode reload, production metrics and alerts, and
+write-fanout benchmark evidence. The topology and clock-policy extensions in
+Section 9 are intentional non-goals and do not leave central scope incomplete.
 
 ---
 
@@ -140,11 +143,20 @@ Implemented:
 19. `BatchAllocator` validates cached candidates once Phase D is required. A
     candidate at/below the Phase-D floor invalidates the entire local window and
     forces a refill above the marker before any timestamp is returned.
-
-Remaining:
-
-1. Runtime config reload for the mode switch; current flags are startup-only.
-2. Production benchmark, divergence metrics, and alert thresholds.
+20. `--tsoModeFile` polls an atomically replaced `legacy`, `shadow`, `cutover`,
+    or `phase-d` mode. Live transitions must be adjacent and one-way. Invalid
+    files, skipped phases, and process-local rollbacks leave the active allocator
+    unchanged. Applied group-0 markers override a stale local mode immediately
+    on allocator resolution, before the next poll can run.
+21. The production registry exports reserve/validation latency by local or
+    remote path and outcome, shadow comparison/divergence, process mode, every
+    reload outcome, and durable marker state. Checked Prometheus rules define
+    warning and critical latency/error thresholds, persistent reload failure,
+    mode divergence, and the 15-minute zero-overlap cutover gate.
+22. `BenchmarkTSOWriteFanout` models 16 concurrent coordinators, 1/3/8-way
+    writes, a 1 ms remote TSO commit, and batch sizes 1/64/256. Three-run
+    evidence supports the existing default batch size 256 and is recorded in
+    `docs/centralized_tso_operations.md`.
 
 ### 1.1 Original Limitation
 
@@ -752,8 +764,10 @@ approach enables a live cutover.
 
 ### 7.3 Phase C — Durable TSO Cutover
 
-- The startup flag `--tsoEnabled` switches coordinator issuance to the
-  leader-routed allocator. Runtime config reload remains future work.
+- The backward-compatible startup flag `--tsoEnabled`, or runtime mode file
+  value `cutover`, switches coordinator issuance to the leader-routed allocator.
+  A live transition is accepted only after `shadow`; restart recovery may start
+  directly in cutover when the durable marker already exists.
 - The first production refill commits the group-0 cutover marker before
   committing and returning its timestamp window.
 - The marker is encoded in a versioned TSO-specific envelope whose leading
@@ -769,12 +783,13 @@ approach enables a live cutover.
 
 - Roll every member to a binary that understands the Phase-D entry and
   `ValidateTimestamp` RPC. Run all members on the dedicated TSO path before
-  enabling `--tsoPhaseDEnabled`; an older member would correctly halt on the
-  unknown control entry, so mixed-version activation is prohibited.
-- The first allocation with `--tsoPhaseDEnabled` commits cutover (if needed),
-  then the Phase-D marker and its pre-Phase-D floor, then a new allocation
-  window strictly above that floor. The switch is one-way and survives restart
-  through the TSO V4 snapshot.
+  enabling `--tsoPhaseDEnabled` or reloading `phase-d`; an older member would
+  correctly halt on the unknown control entry, so mixed-version activation is
+  prohibited.
+- The first allocation with Phase D requested commits cutover (if needed), then
+  the Phase-D marker and its pre-Phase-D floor, then a new allocation window
+  strictly above that floor. The switch is one-way and survives restart through
+  the TSO V4 snapshot.
 - From that marker onward, group 0 reserves contiguous windows from the previous
   committed allocation floor even if wall time or the local HLC mirror has
   advanced. Requests carrying a `min_timestamp` above the committed floor fail
@@ -822,6 +837,29 @@ candidate is returned. Once the cutover marker applies, shadow callers stop
 returning legacy candidates. Independently, every new TSO leader term fences
 its first window above the strict maximum committed data timestamp.
 
+### 7.6 Runtime Reload and Operational Gates
+
+`DynamicTimestampAllocator` publishes the process-selected allocator through
+an atomic pointer. Before resolving that pointer, it checks the consensus-owned
+cutover and Phase-D markers. An applied marker selects the batch/routed TSO path
+and enables matching remote confirmation immediately, so a stale `legacy` or
+`shadow` file cannot mint a legacy timestamp while waiting for the next poll.
+A request that resolved its preceding mode before local marker apply may finish,
+but every subsequent resolution observes the one-way durable override.
+
+`TSORuntimeController` accepts only adjacent one-way live transitions:
+
+```text
+legacy -> shadow -> cutover -> phase-d
+```
+
+The initial process mode may be later in the sequence for restart recovery.
+No-op polls refresh the durable-state gauges, and every failed poll increments
+its bounded reload-result counter even when duplicate log messages are
+suppressed. Exact rollout gates, alert expressions, atomic file replacement,
+and benchmark evidence are maintained in
+`docs/centralized_tso_operations.md`.
+
 ---
 
 ## 8. Milestones
@@ -835,20 +873,24 @@ its first window above the strict maximum committed data timestamp.
 | M5 — shipped | Preserve the default-group `LocalTSOAllocator` compatibility bridge when group 0 is absent; route coordinator-owned timestamp call sites through the allocator abstraction. | Medium |
 | M6 — shipped | Run the dedicated group-0 FSM, fence each new TSO leader term above all authoritative data-group commit floors, redirect follower requests to the TSO leader over gRPC, synchronously serialize fail-closed shadow issuance, and commit the one-way rolling cutover marker before production windows. | Low |
 | M7 — shipped | Commit the durable Phase-D floor marker, preserve V3 snapshots until activation, retire data-shard HLC renewal and legacy/shadow issuance after cutover, activate before read validation while preserving exact applied snapshots through timestamp-bound per-dispatch vouchers, invalidate pre-Phase-D batch windows, and validate unvouched caller-supplied cross-shard SSI timestamps at the group-0 leader from activation onward. | Low |
+| M8 — shipped | Atomically reload one-way runtime modes, make durable markers override stale local mode before allocation, expose production latency/divergence/state metrics, install checked alert thresholds, and validate the default batch size with concurrent write-fanout evidence. | Low |
 
 ---
 
-## 9. Open Questions
+## 9. Resolved Questions and Future Extensions
 
-1. **TSO RTT when TSO leader ≠ write leader:** What batch size minimises tail
-   latency? Needs benchmarking against realistic write fan-out.
+1. **TSO RTT when TSO leader differs from the write leader (resolved):** The
+   concurrent benchmark covers 1/3/8 fan-out legs with a modeled 1 ms remote
+   TSO commit. Batch size 1 exposes serialized refill tails, while default 256
+   keeps the recorded p99 below the 50 ms warning threshold.
 
-2. **TSO group membership:** Should all cluster nodes join the TSO group, or
-   should a dedicated subset (e.g. 3 out of N) be used to reduce Raft traffic?
+2. **TSO group membership (intentional future extension):** Choosing all nodes
+   or a dedicated subset is a topology policy outside the central timestamp
+   subsystem. The implemented routing and fail-closed term fence support either.
 
-3. **Clock floor semantics:** `max(now, ceiling)` vs. `ceiling + 1` — the
-   stricter form (`ceiling + 1`) guarantees no overlap even if wall clocks
-   drift, at the cost of one extra millisecond per renewal window.
+3. **Clock floor semantics (intentional future extension):** The implementation
+   retains the existing floor formula. Changing to `ceiling + 1` is an
+   independent HLC policy decision, not unfinished centralized-TSO scope.
 
 4. **Non-leader TSO requests (resolved):** Followers redirect to the current
    group-0 leader through `Distribution.GetTimestamp`. Timestamp allocation is