diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5cfa8409b6..905b89fab7 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -176,12 +176,20 @@ jobs: # the release-cut 4-node evidence is HG#2a-CI or HG#2a-EXT (external). - { name: stage5-beta-chaos-soak, ranges: "344-345", unit: false, regress: false } # spec-6.12/6.15 cross-node perf + correctness family (t/346-350: - # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash) and - # the renumbered t/351-356 family + t/357 multi-xmax alias floor. + # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash), + # the renumbered t/351-356 family, t/357 multi-xmax alias floor, and + # t/361 XID authority native-era adoption. # Closes the 346+ shard hole (every new t/ file must land in a shard # the same commit; see docs/code-review-checklist.md). - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false } - - { name: stage6-crossnode-b, ranges: "351-357", unit: false, regress: false } + - { name: stage6-crossnode-b, ranges: "351-357 361", unit: false, regress: false } + # t/362 spec-4.6a 4-node shared_catalog kill self-heal (t/358-360 + # reserved by the parallel spec-7.2 lane; L464 occupancy verified + # against origin branches 2026-07-08). Own shard: 4-node formation + # + kill + convergence needs the wall clock. + - { name: stage7-reconfig-liveness, ranges: "362-362", unit: false, regress: false } + # t/363 spec-2.29a marker-async LMON liveness. + - { name: stage7-marker-async, ranges: "363-363", unit: false, regress: false } steps: - name: Checkout uses: actions/checkout@v4 diff --git a/docs/reference/system-views.md b/docs/reference/system-views.md index 649aa954c8..9ceab9b723 100644 --- a/docs/reference/system-views.md +++ b/docs/reference/system-views.md @@ -210,6 +210,9 @@ must not be read as evidence that early transfer is active. | `smart_fusion` | `origin_suspect_count` | Pending dependencies associated with an origin suspected dead or unavailable. | | `smart_fusion` | `dep_lost_failclosed_count` | Missing or malformed dependency evidence rejected fail-closed. | | `smart_fusion` | `retry_failclosed_count` | Retryable Smart Fusion fail-closed outcomes, primarily commit-brake timeout. | +| `catalog` | `xid_authority_native_hw` | Sealed native-era xid high-water for shared-catalog formation; `0` when the authority is inactive or unreadable. | +| `catalog` | `xid_authority_sealed` | `t` when the seed completed a clean native-era shutdown and published adoptable XID prehistory. | +| `catalog` | `xid_prehistory_adopted` | Process-local marker showing whether this postmaster boot adopted native-era pg_xact prehistory. | Example: diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 007c2cb90e..5165f233d9 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -776,6 +776,9 @@ Requirements and behaviour: * `cluster.node_id` must be between 0 and 15; startup is refused otherwise. Striped clusters are therefore limited to **16 nodes**. * `cluster.online_join` must be `on`; startup is refused otherwise. +* `cluster.shared_catalog = on` with more than one declared node requires + `cluster.xid_striping = on`; startup fails closed with SQLSTATE `53RB5` + if a multi-node shared-catalog cluster is formed without striping. * All nodes must run with the same `cluster.xid_striping` setting. The cluster records its striping state durably on the voting disk; a node whose configuration disagrees with that recorded state (or @@ -791,6 +794,50 @@ cluster.xid_striping = on cluster.online_join = on ``` +### Shared-catalog native-era XID authority + +A shared-catalog cluster that is formed from a single seed node has a +short native era: the seed runs with `cluster.enabled = off` while it +creates initial catalog rows, roles, schemas, and seed data. Those +transactions consume the seed node's local xids before other nodes have +joined. During the seed node's clean shutdown, pgrac publishes two +formation files under `cluster.shared_data_dir/global`: +`pgrac_xid_authority` (the sealed native xid high-water) and +`pgrac_xid_prehistory` (the seed node's pg_xact truth for those xids). + +Provisioning sequence: + +1. Create or clone joiner data directories before the seed load if they + must share the seed node's system identifier. +2. Start the seed with `cluster.shared_catalog = on`, + `cluster.controlfile_shared_authority = on`, `cluster.enabled = off`, + and a fixed `cluster.node_id`; perform the seed load. +3. Stop the seed cleanly. Do not use immediate shutdown for the final + native-era stop; joiners refuse an unsealed authority with SQLSTATE + `53RB5`. +4. Start all nodes with `cluster.enabled = on`, `cluster.online_join = on`, + and `cluster.xid_striping = on`. Joiners adopt the seed's transaction + history at startup, so seed tables, roles, and data are visible on + every node. + +Do not delete or re-create the authority files to recover a failed join. +A missing, unsealed, or corrupt authority/prehistory is intentionally +fail-closed (`53RB5`); restore the shared tree files from backup or +re-provision the cluster. After the first `cluster.enabled = on` boot, +the native era is permanently closed and a later `cluster.enabled = off` +boot against the same shared tree is refused. + +Observe formation state with: + +```sql +SELECT key, value + FROM pg_cluster_state + WHERE category = 'catalog' + AND key IN ('xid_authority_native_hw', + 'xid_authority_sealed', + 'xid_prehistory_adopted'); +``` + ### `cluster.xid_herding_slack` | | | diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670985611f..850c468df5 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -201,6 +201,7 @@ #include "cluster/cluster_guc.h" /* PGRAC: spec-5.6 cluster_controlfile_shared_authority */ #include "cluster/cluster_hw_snapshot.h" /* PGRAC: spec-5.7 D3 HW authority checkpoint snapshot */ #include "cluster/cluster_xid_stripe_xlog.h" /* PGRAC: spec-6.15 D5d checkpoint re-emit */ +#include "cluster/cluster_xid_authority.h" /* PGRAC: spec-6.15b native-era XID authority */ #include "cluster/cluster_recovery_anchor.h" /* PGRAC: spec-5.6a per-node recovery anchor */ #include "cluster/cluster_lms.h" /* PGRAC: spec-5.6 GES-ready boundary for CF X */ #endif @@ -5593,6 +5594,42 @@ StartupXLOG(void) */ if (cluster_recovery_anchor_active() && !InRecovery) checkPoint = cluster_recovery_anchor_get()->checkPointCopy; + + /* + * PGRAC (spec-6.15b D5): shared_catalog nodes must reconcile their + * transaction horizon with the sealed native-era XID authority before + * ShmemVariableCache is seeded and before StartupCLOG computes its latest + * page. This deliberately consumes a dedicated never-lowered authority, + * not the shared pg_control CheckPoint copy (spec-5.6a keeps that + * per-writer). + */ + if (cluster_shared_catalog && cluster_enabled) + { + ClusterXidAuthorityHeader auth; + bool auth_from_bak = false; + bool auth_ok; + + auth_ok = cluster_xid_authority_read_checked(&auth, &auth_from_bak); + if (auth_ok && auth_from_bak) + elog(LOG, "cluster shared_catalog: XID authority served from .bak fallback; " + "the primary copy failed validation"); + + if (!auth_ok || + (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is unavailable during WAL startup"), + errdetail("StartupXLOG cannot seed nextXid without a sealed native-era high-water."), + errhint("Complete a clean native-era seed shutdown before starting shared_catalog nodes."))); + + if (U64FromFullTransactionId(checkPoint.nextXid) < auth.native_hw_full) + { + uint64 own_next = U64FromFullTransactionId(checkPoint.nextXid); + + checkPoint.nextXid = FullTransactionIdFromU64(auth.native_hw_full); + elog(LOG, "cluster shared_catalog: merged nextXid with XID authority native high-water %llu (own was %llu)", + (unsigned long long)auth.native_hw_full, (unsigned long long)own_next); + } + } #endif /* initialize shared memory variables from the checkpoint record */ @@ -7549,6 +7586,59 @@ CreateCheckPoint(int flags) */ END_CRIT_SECTION(); +#ifdef USE_PGRAC_CLUSTER + /* + * PGRAC (spec-6.15b D3): after the shutdown checkpoint record, + * control-file update, CLOG flush, and per-node recovery anchor publish + * are complete, publish the native-era pg_xact prehistory and seal the + * shared XID authority. This does durable file I/O and may fail closed, + * so it must stay outside the checkpoint critical section. + * + * Two deliberate exclusions (review P2-1): + * + * - END_OF_RECOVERY checkpoints carry `shutdown` but are NOT a clean + * close of the native run: the seed keeps consuming xids afterwards, + * so sealing here would let a joiner adopt a mid-run stale high-water + * (the same 8.A window the begin_native_run unseal closes for the + * multi-pass case). Only a true shutdown checkpoint seals. + * + * - An unsupported native era (MultiXacts created, or past the + * prehistory cap) SKIPS the seal with a WARNING instead of FATALing: + * the refusal still lands fail-closed on every joiner (53RB5, + * authority unsealed), but the seed itself can shut down cleanly + * instead of wedging every later shutdown/boot in a FATAL loop. + */ + if (shutdown && !(flags & CHECKPOINT_END_OF_RECOVERY) + && cluster_shared_catalog && !cluster_enabled) + { + uint64 native_hw = U64FromFullTransactionId(checkPoint.nextXid); + + if (checkPoint.nextMulti > FirstMultiXactId) + ereport(WARNING, + (errmsg("native-era seed loads that create MultiXacts are not supported; " + "leaving the shared XID authority unsealed"), + errdetail("Shutdown checkpoint nextMulti is %u, but only %u is supported.", + checkPoint.nextMulti, FirstMultiXactId), + errhint("Joiners will fail closed (53RB5); recreate the seed without " + "MultiXact-producing operations, or move the load in-protocol."))); + else if (cluster_xid_prehistory_payload_bytes(native_hw) == 0) + ereport(WARNING, + (errmsg("native-era xid high-water %llu is outside the prehistory publish " + "range; leaving the shared XID authority unsealed", + (unsigned long long) native_hw), + errhint("Joiners will fail closed (53RB5); split the seed load or move " + "it in-protocol."))); + else + { + cluster_xid_prehistory_publish(DataDir, native_hw); + cluster_xid_authority_publish_native(native_hw, checkPoint.nextMulti, true); + elog(LOG, "cluster shared_catalog: published and sealed native-era XID authority high-water %llu", + (unsigned long long) native_hw); + } + } + +#endif + /* * Let smgr do post-checkpoint cleanup (eg, deleting old files). */ diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 7c3daf86a5..c0b7a009c7 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -3273,8 +3273,16 @@ ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record, TimeLineID *repl /* * ShmemVariableCache->nextXid must be beyond record's xid. + * + * PGRAC: a merged-recovery FOREIGN record belongs to a peer's xid + * authority, not this node's allocator. Advancing the local nextXid here + * would make StartupCLOG/TrimCLOG expect local pg_xact pages for peer + * striped xids, while the peer outcome truth lives in pg_xact_remote. */ - AdvanceNextFullTransactionIdPastXid(record->xl_xid); +#ifdef USE_PGRAC_CLUSTER + if (!(cluster_recmerge_window_active && cluster_recmerge_apply_foreign)) +#endif + AdvanceNextFullTransactionIdPastXid(record->xl_xid); /* * Before replaying this record, check if this record causes the current diff --git a/src/backend/cluster/Makefile b/src/backend/cluster/Makefile index 5ea41237f7..1a21622e4d 100644 --- a/src/backend/cluster/Makefile +++ b/src/backend/cluster/Makefile @@ -220,6 +220,8 @@ OBJS = \ cluster_views.o \ cluster_wal_state.o \ cluster_wal_thread.o \ + cluster_xid_authority.o \ + cluster_xid_prehistory.o \ cluster_xid_stripe.o \ cluster_xid_stripe_boot.o \ cluster_xid_stripe_xlog.o \ diff --git a/src/backend/cluster/cluster_catalog_bootstrap.c b/src/backend/cluster/cluster_catalog_bootstrap.c index 9fee3e481c..413af44ebb 100644 --- a/src/backend/cluster/cluster_catalog_bootstrap.c +++ b/src/backend/cluster/cluster_catalog_bootstrap.c @@ -35,6 +35,10 @@ */ #include "postgres.h" +#include +#include + +#include "access/transam.h" #include "access/xlog.h" #include "catalog/pg_control.h" #include "cluster/cluster_catalog_bootstrap.h" @@ -44,13 +48,263 @@ #include "cluster/cluster_inject.h" #include "cluster/cluster_mode.h" #include "cluster/cluster_oid_lease.h" +#include "cluster/cluster_recovery_anchor.h" #include "cluster/cluster_startup_phase.h" +#include "cluster/cluster_xid_authority.h" #include "miscadmin.h" +#include "storage/fd.h" + +static bool +cluster_catalog_backup_label_present(void) +{ + char label_path[MAXPGPATH]; + struct stat st; + + snprintf(label_path, sizeof(label_path), "%s/%s", DataDir, BACKUP_LABEL_FILE); + return stat(label_path, &st) == 0; +} + +static void +cluster_catalog_xid_authority_corrupt_fatal(const char *detail) +{ + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is unavailable at catalog bootstrap"), + errdetail("%s", detail), + errhint("Restore \"%s/%s\" (or its .bak) from a backup of the shared tree; " + "do not delete or re-seed it from a stale xid high-water.", + cluster_shared_data_dir, CLUSTER_XID_AUTHORITY_REL_PATH))); +} + + +/* + * Early D6 topology sniff: cluster_conf_load() runs later when shmem exists, + * but shared_catalog bootstrap must reject multi-node xid_striping=off before + * it seeds/adopts any authority. This deliberately counts only [node.N] + * section headers; the real parser remains the startup SSOT and will still + * validate syntax, required fields, and node_id consistency later. + */ +static int +cluster_catalog_declared_node_count_early(void) +{ + const char *path = (cluster_config_file != NULL && cluster_config_file[0] != '\0') + ? cluster_config_file + : "pgrac.conf"; + FILE *f; + char line[1024]; + int nodes = 0; + + f = AllocateFile(path, "r"); + if (f == NULL) + return 1; + + while (fgets(line, sizeof(line), f) != NULL) { + const char *p = line; + + while (*p != '\0' && isspace((unsigned char)*p)) + p++; + if (strncmp(p, "[node.", 6) == 0) + nodes++; + } + FreeFile(f); + + return nodes > 0 ? nodes : 1; +} + +static void +cluster_catalog_vet_xid_striping_for_shared_catalog(void) +{ + int nodes; + + if (!cluster_enabled || cluster_xid_striping) + return; + + nodes = cluster_catalog_declared_node_count_early(); + if (nodes > 1) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("cluster.shared_catalog on a multi-node cluster requires cluster.xid_striping"), + errdetail("The configured cluster declares %d nodes.", nodes), + errhint("Set cluster.xid_striping=on (and cluster.online_join=on) on every " + "shared_catalog node."))); +} + +static void +cluster_catalog_prepare_xid_authority(const ControlFileData *cf, + ClusterCatalogMigrateResult migrate_result) +{ + ClusterXidAuthorityHeader auth; + bool have_auth; + + { + bool auth_from_bak = false; + + have_auth = cluster_xid_authority_read_checked(&auth, &auth_from_bak); + if (have_auth && auth_from_bak) + elog(LOG, "cluster shared_catalog: XID authority served from .bak fallback; " + "the primary copy failed validation"); + } + if (!have_auth && cluster_xid_authority_present()) + cluster_catalog_xid_authority_corrupt_fatal( + "Neither the primary shared XID authority nor its .bak fallback passes validation."); + + if (!have_auth) { + if (migrate_result != CLUSTER_CATALOG_MIGRATE_SEEDED) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is absent for an existing shared catalog tree"), + errdetail("The shared catalog marker exists, but \"%s/%s\" does not.", + cluster_shared_data_dir, CLUSTER_XID_AUTHORITY_REL_PATH), + errhint("Re-seed the shared tree with a build carrying spec-6.15b."))); + + if (cluster_xid_authority_seed_if_absent( + U64FromFullTransactionId(cf->checkPointCopy.nextXid))) + elog(LOG, "cluster shared_catalog: seeded XID authority native high-water at %llu", + (unsigned long long)U64FromFullTransactionId(cf->checkPointCopy.nextXid)); + + if (!cluster_xid_authority_read(&auth)) + cluster_catalog_xid_authority_corrupt_fatal( + "The shared XID authority was just seeded but cannot be read back."); + } + + if (!cluster_enabled) { + if (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("native seed era cannot be re-entered on a formed shared catalog tree"), + errdetail("The shared XID authority is already marked cluster-era started."), + errhint( + "Keep cluster.enabled=on for this shared tree, or destroy and re-form it."))); + + /* + * A follow-up native run on a SEALED authority re-opens the era: the + * seal is cleared up front so a crash of this run leaves joiners + * fail-closed (unsealed) instead of adopting the previous pass's + * stale high-water (spec-6.15b §3.1 multi-pass arm, rule 8.A). The + * clean shutdown of this run re-publishes and re-seals. + * + * Called unconditionally (not gated on the primary's SEALED flag): + * the transition re-asserts BOTH on-disk copies, so a crash between + * a previous unseal's two renames -- a stale SEALED .bak behind an + * already-unsealed primary, which a later primary read failure + * would resurrect -- is repaired here instead of surviving the run + * (review r3-X1). Once both copies are open this is a no-write + * no-op. + */ + cluster_xid_authority_begin_native_run(); + if (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) + elog(LOG, "cluster shared_catalog: re-opened native seed era " + "(XID authority unsealed for this run)"); + return; + } + + if ((auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is not sealed"), + errdetail("The seed node has not completed a clean native-era shutdown."), + errhint( + "Start the seed with cluster.enabled=off, stop it cleanly, then start joiners."))); + + if (cluster_catalog_backup_label_present()) + elog(LOG, "cluster shared_catalog: skipped XID prehistory adopt on backup_label boot"); + else { + ClusterRecoveryAnchor ra; + uint64 own_next; + + if (!cluster_recovery_anchor_read(cf->system_identifier, &ra, NULL)) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("per-node recovery anchor is unavailable for XID prehistory adopt"), + errdetail("The joiner cannot determine its own pre-adopt nextXid without \"%s\".", + cluster_recovery_anchor_path() != NULL ? cluster_recovery_anchor_path() + : "(unset)"), + errhint( + "Re-provision this node or verify cluster.shared_data_dir before startup."))); + + own_next = U64FromFullTransactionId(ra.checkPointCopy.nextXid); + + /* + * Review F2 (spec Q6 amendment): the sysid gate cannot tell a true + * pre-seed clone from a same-sysid clone that ran STANDALONE after + * cloning -- such a node's pg_xact holds its own outcomes in the + * native range, so skipping would trust divergent local truth and + * adopting would overwrite the node's own outcomes; both are + * silently wrong (8.A). Byte-compare the comparable prefix + * [oldestXid, min(own_next, native_hw)) against the sealed blob and + * fail closed on any contradiction. Bypass only when the local + * oldestXid has advanced past the native range (frozen+truncated: + * those bits are never consulted again). Below that bypass the + * check starts at oldestXid itself (review r3-X2): segments frozen + * and truncated away are no alibi for the surviving range, and a + * missing local page inside the comparable range fails closed + * (UNAVAILABLE) instead of passing as a shorter clone. + */ + if ((uint64)ra.checkPointCopy.oldestXid <= auth.native_hw_full) { + ClusterXidPrefixVerdict pv; + + pv = cluster_xid_prehistory_prefix_check(DataDir, auth.native_hw_full, + Min(own_next, auth.native_hw_full), + (uint64)ra.checkPointCopy.oldestXid); + if (pv == CLUSTER_XID_PREFIX_DIVERGED) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("local pg_xact contradicts the sealed native-era prehistory"), + errdetail("divergent native-era lineage: this node's transaction " + "history overlaps the seed's native xid range with " + "different outcomes."), + errhint("This node is not a pre-seed clone of the shared tree; " + "destroy and re-provision it from the seed lineage."))); + if (pv == CLUSTER_XID_PREFIX_UNAVAILABLE) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID prehistory is unavailable for the lineage check"), + errdetail("Either no sealed prehistory blob passes validation, or the " + "local pg_xact has a hole inside the comparable native " + "range."), + errhint("Restore \"%s\" (or its .bak) from the shared tree's backup, " + "or re-provision this node if its local pg_xact is damaged.", + CLUSTER_XID_PREHISTORY_REL_PATH))); + } + + if (own_next < auth.native_hw_full) { + cluster_xid_prehistory_adopt(DataDir, auth.native_hw_full); + elog(LOG, + "cluster shared_catalog: adopted XID prehistory through native high-water %llu " + "(own nextXid was %llu)", + (unsigned long long)auth.native_hw_full, (unsigned long long)own_next); + } else + elog(LOG, + "cluster shared_catalog: skipped XID prehistory adopt; own nextXid %llu >= native " + "high-water %llu", + (unsigned long long)own_next, (unsigned long long)auth.native_hw_full); + } + + /* + * One-way CLUSTER_ERA stamp, re-run unconditionally on every cluster + * boot: the transition re-asserts BOTH on-disk copies, so a crash + * between a previous stamp's two renames -- a stale pre-CLUSTER_ERA + * .bak behind an already-stamped primary, which a later primary read + * failure would hand to an enabled=off boot as a re-entry bypass -- is + * repaired here instead of surviving the run (review r3-X1). Once + * both copies are stamped this is a no-write no-op. + */ + cluster_xid_authority_mark_cluster_era(); + if ((auth.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) == 0) + elog(LOG, + "cluster shared_catalog: marked XID authority cluster era started at native " + "high-water %llu", + (unsigned long long)auth.native_hw_full); +} void cluster_catalog_startup_prepare(void) { ControlFileData cf; + ClusterCatalogMigrateResult migrate_result; /* Postmaster-once: only the postmaster seeds; forked backends inherit. */ if (IsUnderPostmaster) @@ -66,6 +320,8 @@ cluster_catalog_startup_prepare(void) return; /* off: stock per-node catalog */ } + cluster_catalog_vet_xid_striping_for_shared_catalog(); + /* * shared_catalog=on requires the shared pg_control authority (D1 vet), so * it is the single source of the cluster-wide next-OID high-water. Reading @@ -113,7 +369,8 @@ cluster_catalog_startup_prepare(void) * the (D3-flipped) shared smgr route. system_identifier is the shared * pg_control's cluster-wide value both seed and join agree on. */ - cluster_catalog_migrate_tree(DataDir, cf.system_identifier); + migrate_result = cluster_catalog_migrate_tree(DataDir, cf.system_identifier); + cluster_catalog_prepare_xid_authority(&cf, migrate_result); } /* diff --git a/src/backend/cluster/cluster_catalog_migrate.c b/src/backend/cluster/cluster_catalog_migrate.c index 961179b008..bb17838229 100644 --- a/src/backend/cluster/cluster_catalog_migrate.c +++ b/src/backend/cluster/cluster_catalog_migrate.c @@ -588,7 +588,7 @@ vet_seed_no_tablespaces(const char *local_pgdata) FreeDir(dir); } -void +ClusterCatalogMigrateResult cluster_catalog_migrate_tree(const char *local_pgdata, uint64 system_identifier) { ClusterCatalogAuthorityMarker existing; @@ -624,7 +624,7 @@ cluster_catalog_migrate_tree(const char *local_pgdata, uint64 system_identifier) elog(LOG, "cluster shared_catalog: adopted shared catalog authority (sysid " UINT64_FORMAT ")", system_identifier); - return; + return CLUSTER_CATALOG_MIGRATE_ADOPTED; } /* @@ -645,6 +645,7 @@ cluster_catalog_migrate_tree(const char *local_pgdata, uint64 system_identifier) "cluster shared_catalog: seeded shared catalog authority under \"%s\" " "(sysid " UINT64_FORMAT ")", cluster_shared_data_dir, system_identifier); + return CLUSTER_CATALOG_MIGRATE_SEEDED; } /* diff --git a/src/backend/cluster/cluster_clean_leave.c b/src/backend/cluster/cluster_clean_leave.c index d6152a4e28..24be2345d8 100644 --- a/src/backend/cluster/cluster_clean_leave.c +++ b/src/backend/cluster/cluster_clean_leave.c @@ -58,7 +58,8 @@ #include "cluster/cluster_guc.h" #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_ic_router.h" -#include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT (D12) */ +#include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT (D12) */ +#include "cluster/cluster_lmon.h" #include "cluster/cluster_pcm_lock.h" /* PCM release-all-self + no-leftover verify (D5) */ #include "cluster/cluster_qvotec.h" /* cluster_qvotec_in_quorum (request gate) */ #include "cluster/cluster_reconfig.h" /* apply_clean_leave + record/is_clean_departed + join_in_progress */ @@ -121,6 +122,8 @@ cluster_clean_leave_shmem_init(void) pg_atomic_init_u32(&cl_state->commit_point_observed, 0); pg_atomic_init_u32(&cl_state->committed_durable_confirmed, 0); pg_atomic_init_u32(&cl_state->committed_marker_durable, 0); + /* spec-2.29a r3 (evidence latch). */ + cl_state->committed_confirmed_epoch = 0; /* Hardening v1.0.2 (P1 preflight / P2 nonce). */ pg_atomic_init_u64(&cl_state->leave_attempt_nonce, 0); pg_atomic_init_u32(&cl_state->preflight_pending, 0); @@ -381,6 +384,11 @@ cluster_clean_leave_check_pending_in_proc_interrupts(void) * disabled survivor MUST reply LEAVE_DRAIN_NAK rather than go silent, else the * leaving node waits for an ACK that never comes and false-times-out. */ + +/* spec-2.29a ②b: forward decls — the bind sites (below) snapshot the + * others-dead set that the coherence gate (defined near drive_drain) compares. */ +static void cl_others_dead_snapshot(int32 leaving, uint8 *out); + static void cl_announce_handler(const ClusterICEnvelope *env, const void *payload) { @@ -509,6 +517,9 @@ cl_announce_handler(const ClusterICEnvelope *env, const void *payload) * fails the commit closed instead of committing on a stale membership view. */ cl_state->leave_baseline_dead_gen = cluster_cssd_get_dead_generation(); + /* spec-2.29a ②b: snapshot the others-dead set (excludes the leaving node) + * the coherence gate compares against. */ + cl_others_dead_snapshot(leaving, cl_state->leave_baseline_others_dead); pg_atomic_write_u32(&cl_state->survivor_acked, 0); pg_atomic_write_u32(&cl_state->commit_ready_received, 0); pg_atomic_write_u32(&cl_state->committed_marker_durable, 0); @@ -546,11 +557,15 @@ cl_commit_ready_handler(const ClusterICEnvelope *env, const void *payload) } /* - * cl_committed_handler -- leaving node side (Hardening v1.0.1, P1-V0.7 exit gate): - * the survivor coordinator has made the COMMITTED marker majority-durable and - * signals that the durable truth-source now exists, so this leaving node may - * proceed to COMMITTED and exit. Only acted on if it is about OUR leave; idem- - * potent (the coordinator re-sends each tick until we are gone). + * cl_committed_handler -- leaving node side (Hardening v1.0.1, P1-V0.7 exit gate; + * spec-2.29a r3 evidence latch): the survivor coordinator has made the COMMITTED + * marker majority-durable and attests that the durable truth-source now exists. + * This confirmation is the leaver's own-commit MARKER EVIDENCE: the barrier tick + * latches on it (and only on it) and proceeds to COMMITTED/exit. Identity is + * checked by the pure evidence gate (self-addressed + currently leaving + + * per-attempt nonce + committed epoch past the bound baseline) — any mismatch is + * dropped fail-closed. Idempotent (the coordinator re-sends each tick until we + * are gone). */ static void cl_committed_handler(const ClusterICEnvelope *env, const void *payload) @@ -562,17 +577,19 @@ cl_committed_handler(const ClusterICEnvelope *env, const void *payload) return; if (!cluster_clean_leave_announce_payload_valid(p)) return; - if (p->leaving_node_id != cluster_node_id) - return; /* only the leaving node consumes its own COMMITTED confirmation */ - /* Hardening v1.0.2 (P2): bind to THIS attempt — a stale LEAVE_COMMITTED from a - * prior same-epoch attempt must not prematurely confirm the current one. Only - * meaningful once we are the leaver actively awaiting confirmation. */ - if (p->leave_nonce != pg_atomic_read_u64(&cl_state->leave_attempt_nonce)) + LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); + if (!cluster_clean_leave_committed_evidence_matches( + p->leaving_node_id, p->leave_nonce, p->leave_epoch, cluster_node_id, + cl_state->leaving_node_id, pg_atomic_read_u64(&cl_state->leave_attempt_nonce), + cl_state->leave_epoch)) { + LWLockRelease(&cl_state->lock); return; - if (cl_state->leaving_node_id != cluster_node_id) - return; /* we are not currently leaving */ - + } + /* the committed epoch E is recorded BEFORE the evidence flag is published, + * inside the same critical section the barrier tick reads it back under. */ + cl_state->committed_confirmed_epoch = p->leave_epoch; pg_atomic_write_u32(&cl_state->committed_durable_confirmed, 1); + LWLockRelease(&cl_state->lock); } /* @@ -745,6 +762,125 @@ cluster_clean_leave_ic_send_ack(int32 dest_node_id, int32 leaving_node_id, uint6 /* qvotec-side per-process handshake cursors (mirror the fence-marker ones). */ static uint64 cl_qvotec_last_processed_marker_seq = 0; static uint64 cl_qvotec_inflight_marker_seq = 0; +static ClusterMarkerAsync cl_lmon_marker_async; +static ClusterLeaveIntentMarker cl_lmon_marker; +static int cl_lmon_marker_phase = 0; +static int32 cl_lmon_marker_leaving = -1; +static uint64 cl_lmon_marker_epoch = 0; +static bool cl_lmon_marker_submitted = false; + +static void cl_build_marker(ClusterLeaveIntentMarker *m, uint8 marker_phase, int32 leaving, + uint64 epoch); + +typedef enum ClusterLeaveAsyncMarkerResult { + CL_LEAVE_ASYNC_PENDING = 0, + CL_LEAVE_ASYNC_ACKED, + CL_LEAVE_ASYNC_FAILED +} ClusterLeaveAsyncMarkerResult; + +static ClusterMarkerAsyncKind +cl_marker_phase_kind(int phase) +{ + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTING) + return CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTING; + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTED) + return CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED; + return CLUSTER_MARKER_KIND_UNKNOWN; +} + +static void +cl_release_lmon_marker_stage(void) +{ + cluster_marker_async_release_stage(&cl_lmon_marker_async); + memset(&cl_lmon_marker, 0, sizeof(cl_lmon_marker)); + cl_lmon_marker_phase = 0; + cl_lmon_marker_leaving = -1; + cl_lmon_marker_epoch = 0; + cl_lmon_marker_submitted = false; +} + +static bool +cl_start_lmon_marker_stage(const ClusterLeaveIntentMarker *m, int phase, int32 leaving, + uint64 epoch) +{ + if (cl_lmon_marker_async.has_staged_event) + return false; + cl_lmon_marker = *m; + cl_lmon_marker_phase = phase; + cl_lmon_marker_leaving = leaving; + cl_lmon_marker_epoch = epoch; + cl_lmon_marker_async.has_staged_event = true; + cl_lmon_marker_submitted = false; + return true; +} + +static ClusterLeaveAsyncMarkerResult +cl_poll_lmon_marker_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_LEAVE_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + ClusterMarkerAsyncKind kind; + + if (!cl_lmon_marker_async.has_staged_event) + return CL_LEAVE_ASYNC_FAILED; + + now = GetCurrentTimestamp(); + kind = cl_marker_phase_kind(cl_lmon_marker_phase); + if (!cl_lmon_marker_submitted) { + if (!cluster_clean_leave_submit_marker_async(&cl_lmon_marker_async, &cl_lmon_marker, kind, + cl_lmon_marker_leaving, now)) + return CL_LEAVE_ASYNC_PENDING; + cl_lmon_marker_submitted = true; + return CL_LEAVE_ASYNC_PENDING; + } + + pr = cluster_clean_leave_poll_marker_async(&cl_lmon_marker_async, now, &result, &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return CL_LEAVE_ASYNC_PENDING; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(kind, cl_lmon_marker_leaving, elapsed_us); + cl_release_lmon_marker_stage(); + return CL_LEAVE_ASYNC_FAILED; + } + + cluster_reconfig_note_marker_slow_ack(kind, cl_lmon_marker_leaving, elapsed_us); + if (result != CLUSTER_LEAVE_MARKER_SUBMIT_ACK) { + cl_release_lmon_marker_stage(); + return CL_LEAVE_ASYNC_FAILED; + } + return CL_LEAVE_ASYNC_ACKED; +} + +static void +cl_drive_committed_marker_stage(int32 leaving, uint64 committed_epoch) +{ + ClusterLeaveIntentMarker cm; + ClusterLeaveAsyncMarkerResult ar; + + if (cl_lmon_marker_async.has_staged_event) { + if (cl_lmon_marker_phase != CLUSTER_LEAVE_MARKER_PHASE_COMMITTED + || cl_lmon_marker_leaving != leaving || cl_lmon_marker_epoch != committed_epoch) + return; + } else { + cl_build_marker(&cm, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, committed_epoch); + (void)cl_start_lmon_marker_stage(&cm, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, + committed_epoch); + } + + ar = cl_poll_lmon_marker_stage(); + if (ar == CL_LEAVE_ASYNC_ACKED) { + pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); + cl_send_committed(leaving, committed_epoch); + cl_release_lmon_marker_stage(); + } else if (ar == CL_LEAVE_ASYNC_FAILED) { + ereport(LOG, (errmsg("cluster clean-leave: committed node %d at epoch %llu but the " + "COMMITTED marker is not yet majority-durable; retrying each tick " + "(leaving node waits)", + leaving, (unsigned long long)committed_epoch))); + } +} ClusterLeaveMarkerSubmitResult cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m) @@ -787,6 +923,38 @@ cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m) } } +bool +cluster_clean_leave_submit_marker_async(ClusterMarkerAsync *a, const ClusterLeaveIntentMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + int wait_ms; + + if (cl_state == NULL || m == NULL || a == NULL) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&cl_state->marker_request_seq, + &cl_state->marker_completion_seq)) + return false; + + cl_state->pending_marker = *m; + wait_ms = cluster_quorum_poll_interval_ms * 3 + 2000; + return cluster_marker_async_submit(a, &cl_state->marker_request_seq, + &cl_state->marker_completion_seq, cl_state->qvotec_latch, + now, (uint64)wait_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_clean_leave_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) +{ + if (cl_state == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &cl_state->marker_completion_seq, &cl_state->marker_result, + now, out_result, out_elapsed_us); +} + bool cluster_clean_leave_qvotec_poll_pending(void *out_slot512) { @@ -818,6 +986,7 @@ cluster_clean_leave_qvotec_complete(bool acked) pg_write_barrier(); pg_atomic_write_u64(&cl_state->marker_completion_seq, cl_qvotec_inflight_marker_seq); cl_qvotec_last_processed_marker_seq = cl_qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } static void @@ -1277,6 +1446,8 @@ cl_request_body(void) cl_state->leaving_node_id = cluster_node_id; cl_state->leave_epoch = baseline_epoch; cl_state->leave_baseline_dead_gen = cluster_cssd_get_dead_generation(); + /* spec-2.29a ②b: snapshot the others-dead set (excludes self, the leaver). */ + cl_others_dead_snapshot(cluster_node_id, cl_state->leave_baseline_others_dead); cl_state->barrier_deadline_us = (uint64)GetCurrentTimestamp() + (uint64)cluster_clean_leave_drain_timeout_ms * 1000ULL; /* @@ -1304,6 +1475,7 @@ cl_request_body(void) pg_atomic_write_u32(&cl_state->commit_point_observed, 0); pg_atomic_write_u32(&cl_state->committed_durable_confirmed, 0); pg_atomic_write_u32(&cl_state->committed_marker_durable, 0); + cl_state->committed_confirmed_epoch = 0; /* spec-2.29a r3: per-attempt evidence */ LWLockRelease(&cl_state->lock); cl_set_phase(CLUSTER_LEAVE_REQUESTED); @@ -1410,6 +1582,49 @@ cluster_clean_leave_request(void) * re-checks version coherence (CL-I3: an external epoch bump = a real death * intruded -> escalate) and the mixed-mode NAK (clean abort). */ + +/* + * spec-2.29a ②b: snapshot the CSSD dead set EXCLUDING the leaving node. The + * coherence gate compares this "others-dead" set rather than the scalar global + * dead_generation, so the leaving node's own expected alive->DEAD transition + * (it stops heart-beating once its drain finishes) never falsely escalates the + * leave. A third-party death still changes this set and escalates (CL-I3). + */ +static void +cl_others_dead_snapshot(int32 leaving, uint8 *out /* CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES */) +{ + int i; + + memset(out, 0, CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES); + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + if (i == leaving) + continue; /* the leaving node's own DEAD is expected, not incoherence */ + if (i >= CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES * 8) + break; + if (cluster_conf_lookup_node(i) == NULL) + continue; + if (cluster_cssd_get_peer_state(i) == CLUSTER_CSSD_PEER_DEAD) + out[i / 8] |= (uint8)(1u << (i % 8)); + } +} + +/* + * spec-2.29a ②b: the coherence gate used by every clean-leave step. Coherent + * iff the epoch has not moved AND no THIRD-PARTY death changed the others-dead + * set since the leave was bound. cl_state->leaving_node_id names the node + * whose own DEAD is excluded. + */ +static bool +cl_coherent(uint64 baseline_epoch) +{ + uint8 now_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; + + cl_others_dead_snapshot(cl_state->leaving_node_id, now_others_dead); + return cluster_clean_leave_version_coherent( + baseline_epoch, cluster_epoch_get_current(), cl_state->leave_baseline_others_dead, + now_others_dead, CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES); +} + void cluster_clean_leave_drive_drain(void) { @@ -1470,16 +1685,14 @@ cluster_clean_leave_drive_drain(void) /* * CL-I3 coherence gate, re-checked before every drain step. Uses the * dead_gen-aware helper, not an epoch-only compare: CSSD increments the - * dead_generation the moment it declares a peer dead, which is STRICTLY - * BEFORE the reconfig coordinator bumps the membership epoch. Checking - * dead_gen too lets us escalate in that earlier window instead of draining - * into a death that has not yet reached the epoch. At commit the guarded - * CAS in apply_clean_leave_as_coordinator is the final authority. + * others-dead set the moment it declares a THIRD-PARTY peer dead, which is + * STRICTLY BEFORE the reconfig coordinator bumps the membership epoch. + * Checking that set too lets us escalate in that earlier window instead of + * draining into a death that has not yet reached the epoch. The leaving + * node's OWN DEAD is excluded (spec-2.29a ②b). At commit the guarded CAS + * in apply_clean_leave_as_coordinator is the final authority. */ -#define CL_COHERENT() \ - cluster_clean_leave_version_coherent(baseline_epoch, cluster_epoch_get_current(), \ - cl_state->leave_baseline_dead_gen, \ - cluster_cssd_get_dead_generation()) +#define CL_COHERENT() cl_coherent(baseline_epoch) /* REQUESTED -> QUIESCING: abort local writable backends (53R62). */ if (!CL_COHERENT()) { @@ -1551,60 +1764,72 @@ cl_leaving_barrier_tick(void) { uint64 baseline_epoch = cl_state->leave_epoch; int32 coordinator; + uint8 now_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; + bool committed_evidence; + bool others_dead_unchanged; + bool dead_gen_unchanged; /* - * 1. observe the commit. The survivor coordinator runs the actual two-phase - * commit and publishes the CLEAN_LEAVE event into ITS OWN reconfig state — - * the leaving node's last_applied never carries it. What DOES propagate to - * the leaving node is the membership epoch (every IC envelope piggybacks it). - * So the leaving node detects its own commit by the epoch advancing past the - * bound baseline. Discriminate from a real death intruding (CL-I3) by the - * CSSD dead_generation: a cooperative leave does NOT mark anyone CSSD-dead, so - * an unchanged dead_generation at the bump means OUR clean-leave committed; a - * changed one means a real death (escalate). + * 1. observe the commit — EVIDENCE over inference (spec-2.29a r3). The + * survivor coordinator runs the actual two-phase commit, publishes the + * CLEAN_LEAVE event into ITS OWN reconfig state, drives the COMMITTED marker + * to voting-disk majority-durability, and only then sends the nonce-bound + * LEAVE_COMMITTED (re-sent each tick while we are alive). That confirmation + * — validated by the pure identity gate in cl_committed_handler — is the ONLY + * basis on which this node latches its own commit: latch <=> a valid + * COMMITTED marker for THIS leave attempt exists. + * + * Two inference generations preceded this and each failed one way (see the + * predicate comment in cluster_clean_leave_policy.c): "epoch advanced + + * others-dead bitmap unchanged" could mis-latch a REFUSED leave after a + * third-party false-DEAD rebound (r2 P2-1 wedge); adding the monotone scalar + * dead_generation conjunct then false-escalated a healthy committed leave + * whenever a transient third-party flap advanced the leaver's local + * dead_generation during the leave window (nightly t/331 C1/C4). Marker + * evidence is immune to both: flaps cannot erase a durable marker, and a + * refused leave never produces one — no latch, so the barrier deadline below + * still bounds the wait (fail-closed escalation, unchanged semantics). * - * Hardening v1.0.4 (P2): this "epoch bump + dead_gen unchanged == OUR commit" - * inference is sound ONLY because there is no OTHER dead_gen-unchanged reconfig - * in flight. A spec-5.15 online JOIN also bumps the epoch with dead_gen - * unchanged and would be mis-observed here as the leave's commit (then the node - * stops escalating but never gets the real LEAVE_COMMITTED -> BARRIER_WAIT - * hang). That collision is removed STRUCTURALLY by the one-membership-reconfig- - * at-a-time serialization: the join driver does not bump the epoch while a clean - * leave is active (cluster_clean_leave_in_progress gates drive_joins + - * commit_member), and the leave does not start while a join is pending - * (cluster_reconfig_join_in_progress gates the request). Under that invariant a - * dead_gen-unchanged bump during a leave can only be the leave's own commit. + * The coherence observations are still taken, but ONLY for the flap-noise + * LOG at the latch (the predicate contract pins that they never affect the + * verdict). A real third-party death intruding mid-leave is refused on the + * survivor side (cl_coherent pre-check + guarded CAS, CL-I3), so no evidence + * arrives here and the deadline escalates this leaver — same outcome as the + * old immediate escalate arm, now bounded by the barrier deadline instead. + * + * Latching collapses the old two-step gate: the evidence IS the P1-V0.7 + * durable-truth-source confirmation, so the leave can no longer be + * un-committed (deadline must not escalate past here, Hardening v1.0.1 + * P1-1) AND the node may exit (COMMITTED) in the same tick. */ - if (cluster_epoch_get_current() > baseline_epoch) { - if (cluster_cssd_get_dead_generation() == cl_state->leave_baseline_dead_gen) { - /* - * Commit point observed (our clean-leave epoch was published; no real - * death intruded). The leave can no longer be un-committed, so from - * here the barrier deadline must NOT escalate (Hardening v1.0.1 P1-1). - */ - pg_atomic_write_u32(&cl_state->commit_point_observed, 1); - LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); - cl_state->leave_epoch = cluster_epoch_get_current(); /* the committed epoch E */ - LWLockRelease(&cl_state->lock); + committed_evidence = (pg_atomic_read_u32(&cl_state->committed_durable_confirmed) != 0); + cl_others_dead_snapshot(cl_state->leaving_node_id, now_others_dead); + others_dead_unchanged = (memcmp(now_others_dead, cl_state->leave_baseline_others_dead, + CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES) + == 0); + dead_gen_unchanged = (cluster_cssd_get_dead_generation() == cl_state->leave_baseline_dead_gen); + if (cluster_clean_leave_own_commit_latched(committed_evidence, others_dead_unchanged, + dead_gen_unchanged)) { + uint64 committed_epoch; + + pg_atomic_write_u32(&cl_state->commit_point_observed, 1); + LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); + committed_epoch = cl_state->committed_confirmed_epoch; /* the committed epoch E */ + cl_state->leave_epoch = committed_epoch; + LWLockRelease(&cl_state->lock); - /* - * P1-V0.7 exit gate: reach COMMITTED ("may exit") ONLY after the - * coordinator confirms the COMMITTED marker is majority-durable - * (LEAVE_COMMITTED). Until then stay in BARRIER_WAIT and re-tick — the - * durable truth-source must exist before this node departs, else a - * survivor restart could not rebuild the clean-departed fact. - */ - if (pg_atomic_read_u32(&cl_state->committed_durable_confirmed)) { - cl_set_phase(CLUSTER_LEAVE_COMMITTED); - CLUSTER_INJECTION_POINT("cluster-clean-leave-barrier-complete"); - ereport(LOG, - (errmsg("cluster clean-leave: committed at epoch %llu + COMMITTED marker " - "majority-durable; this node has drained and may exit", - (unsigned long long)cl_state->leave_epoch))); - } - } else { - cl_escalate(); /* a real death changed the version mid-leave (CL-I3) */ - } + if (!others_dead_unchanged || !dead_gen_unchanged) + ereport(LOG, (errmsg("cluster clean-leave: third-party liveness flap observed at the " + "commit latch (others-dead %s, dead_generation %s); durable " + "COMMITTED marker evidence overrides it", + others_dead_unchanged ? "unchanged" : "changed", + dead_gen_unchanged ? "unchanged" : "advanced"))); + + cl_set_phase(CLUSTER_LEAVE_COMMITTED); + CLUSTER_INJECTION_POINT("cluster-clean-leave-barrier-complete"); + ereport(LOG, (errmsg("cluster clean-leave: committed at epoch %llu + COMMITTED marker " + "majority-durable; this node has drained and may exit", + (unsigned long long)committed_epoch))); return; } @@ -1615,8 +1840,12 @@ cl_leaving_barrier_tick(void) } /* 3. fail-closed deadline — ONLY before the commit point (P1-1: a committed - * leave is never un-committed; post-commit we wait for the durable marker, - * bounded by disk health, and never escalate). */ + * leave is never un-committed). With the r3 evidence latch this arm is what + * bounds EVERY no-evidence outcome: a refused leave, a foreign death that + * moved the version (the coordinator then refuses, CL-I3), or a lost/never- + * sent confirmation all end here instead of hanging in BARRIER_WAIT. The + * commit_point_observed guard is belt-and-braces: the latch above transitions + * out of BARRIER_WAIT in the same tick it sets the flag. */ if (!pg_atomic_read_u32(&cl_state->commit_point_observed) && (uint64)GetCurrentTimestamp() > cl_state->barrier_deadline_us) { cl_escalate(); @@ -1640,79 +1869,97 @@ cl_coordinator_commit(int32 leaving) uint64 new_epoch; ClusterLeaveIntentMarker m; + if (cl_lmon_marker_async.has_staged_event) { + ClusterLeaveAsyncMarkerResult ar = cl_poll_lmon_marker_stage(); + int phase = cl_lmon_marker_phase; + uint64 staged_epoch = cl_lmon_marker_epoch; + + if (ar == CL_LEAVE_ASYNC_PENDING) + return; + if (ar == CL_LEAVE_ASYNC_FAILED) + return; + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTING) { + cl_release_lmon_marker_stage(); + + /* + * spec-2.29a review r1 P1 — CL-I3 re-check at the staged-ACK + * handoff. The COMMITTING marker wait now spans LMON ticks, so a + * real death can bump CSSD dead_generation INSIDE the wait window + * while the fail-stop epoch has not yet advanced (the >=3-node + * window of Hardening v1.0.1 P1-2). The guarded CAS inside + * apply_clean_leave_as_coordinator only catches the epoch move; + * re-run the same dead_gen-aware coherence check the non-staged + * pre-check uses, at the last observable point before the commit + * applies. On failure the leave does not commit and the leaving + * node escalates to fail-stop (identical to the pre-check path). + */ + if (!cl_coherent(baseline_epoch)) { + ereport(LOG, + (errmsg("cluster clean-leave: version moved (epoch or third-party death) " + "across the COMMITTING marker wait for node %d; not committing " + "(escalate to fail-stop, CL-I3)", + leaving))); + return; + } + + new_epoch = cluster_reconfig_apply_clean_leave_as_coordinator(leaving, baseline_epoch); + if (new_epoch == 0) { + ereport( + LOG, + (errmsg("cluster clean-leave: epoch moved off baseline %llu before commit " + "for node %d; not committing (the leaving node escalates to fail-stop)", + (unsigned long long)baseline_epoch, leaving))); + return; + } + cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, new_epoch); + (void)cl_start_lmon_marker_stage(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, + new_epoch); + (void)cl_poll_lmon_marker_stage(); + ereport(LOG, + (errmsg("cluster clean-leave: committed departure of node %d at epoch %llu", + leaving, (unsigned long long)new_epoch))); + return; + } + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTED) { + pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); + cl_send_committed(leaving, staged_epoch); + cl_release_lmon_marker_stage(); + return; + } + cl_release_lmon_marker_stage(); + return; + } + /* CL-I3 pre-check: refuse to commit a clean leave on a version a real death * already bumped — the leaving node will then observe a non-CLEAN_LEAVE event * and escalate. This is a cheap early-out; the authoritative guard is the * guarded CAS inside apply_clean_leave_as_coordinator below, which closes the * check-then-bump TOCTOU at >=3 nodes. */ /* - * CL-I3 commit-handoff coherence (epoch AND dead_gen): refuse to commit if a - * real death intruded since this survivor started tracking the leave — either - * the death's fail-stop already bumped the epoch, OR (the >=3-node window, - * Hardening v1.0.1 P1-2) CSSD bumped its dead_generation but the fail-stop - * epoch has NOT yet advanced, which an epoch-only check (and the guarded CAS - * below) would miss and commit on a stale membership view. Uses the same - * unit-tested version_coherent helper as drive_drain so the dead_gen-aware - * coherence holds through the commit handoff; the leaving node then observes - * the eventual foreign event and escalates. + * CL-I3 commit-handoff coherence (epoch AND others-dead): refuse to commit if + * a THIRD-PARTY death intruded since this survivor started tracking the leave + * — either the death's fail-stop already bumped the epoch, OR (the >=3-node + * window, Hardening v1.0.1 P1-2) CSSD added it to the others-dead set but the + * fail-stop epoch has NOT yet advanced, which an epoch-only check (and the + * guarded CAS below) would miss and commit on a stale membership view. Uses + * the same others-dead coherence helper as drive_drain (spec-2.29a ②b: the + * leaving node's own expected DEAD is excluded so it never falsely escalates); + * the leaving node then observes the eventual foreign event and escalates. */ - if (!cluster_clean_leave_version_coherent(baseline_epoch, cluster_epoch_get_current(), - cl_state->leave_baseline_dead_gen, - cluster_cssd_get_dead_generation())) { - ereport(LOG, (errmsg("cluster clean-leave: version moved (epoch or dead_generation) before " - "committing node %d; not committing (escalate to fail-stop, CL-I3)", - leaving))); + if (!cl_coherent(baseline_epoch)) { + ereport(LOG, + (errmsg("cluster clean-leave: version moved (epoch or third-party death) before " + "committing node %d; not committing (escalate to fail-stop, CL-I3)", + leaving))); return; } /* (1) COMMITTING(E) marker (coordinator's own slot, before the bump; NOT a * trust basis). Not durable -> do not commit. */ cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTING, leaving, baseline_epoch + 1); - if (cluster_clean_leave_submit_marker(&m) != CLUSTER_LEAVE_MARKER_SUBMIT_ACK) { - ereport(LOG, (errmsg("cluster clean-leave: COMMITTING marker not durable for node %d; " - "not committing (the leaving node will retry/escalate)", - leaving))); - return; - } - - /* (2) the real commit point: guarded epoch bump (only if still baseline) + - * publish CLEAN_LEAVE + record clean-departed. Returns 0 if a concurrent - * reconfig moved the epoch off the baseline after the COMMITTING marker — - * fail-closed, do NOT write the COMMITTED marker (CL-I3). */ - new_epoch = cluster_reconfig_apply_clean_leave_as_coordinator(leaving, baseline_epoch); - if (new_epoch == 0) { - ereport(LOG, - (errmsg("cluster clean-leave: epoch moved off baseline %llu before commit " - "for node %d; not committing (the leaving node escalates to fail-stop)", - (unsigned long long)baseline_epoch, leaving))); - return; /* CL-I3: stale baseline (or cluster disabled / bad id) */ - } - - /* - * (3) COMMITTED(E) marker (after the bump; the ONLY rebuild trust basis). - * Post-commit it MUST become majority-durable — the durable truth-source must - * exist BEFORE the leaving node departs (P1-V0.7 exit gate; §2.5). Try once - * here. If it reaches majority, mark durable + tell the leaving node it may - * exit (LEAVE_COMMITTED). If NOT, do not give up: committed_marker_durable - * stays 0, cl_survivor_tick retries the marker every tick, and the leaving - * node stays in BARRIER_WAIT (no LEAVE_COMMITTED) until it is durable — it - * never departs without a durable truth-source. The leave is already - * committed (epoch bumped); we never revert. - */ - cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, new_epoch); - if (cluster_clean_leave_submit_marker(&m) == CLUSTER_LEAVE_MARKER_SUBMIT_ACK) { - pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); - cl_send_committed(leaving, new_epoch); - } else { - ereport( - LOG, - (errmsg("cluster clean-leave: committed node %d at epoch %llu but the COMMITTED " - "marker is not yet majority-durable; retrying each tick (leaving node waits)", - leaving, (unsigned long long)new_epoch))); - } - - ereport(LOG, (errmsg("cluster clean-leave: committed departure of node %d at epoch %llu", - leaving, (unsigned long long)new_epoch))); + (void)cl_start_lmon_marker_stage(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTING, leaving, + baseline_epoch + 1); + (void)cl_poll_lmon_marker_stage(); } /* survivor (incl. coordinator) side of another node's leave. */ @@ -1773,13 +2020,9 @@ cl_survivor_tick(int32 leaving) if (coordinator == cluster_node_id && cluster_reconfig_is_clean_departed(leaving)) { uint64 committed_epoch = cluster_reconfig_get_clean_departed_epoch(leaving); - if (!pg_atomic_read_u32(&cl_state->committed_marker_durable)) { - ClusterLeaveIntentMarker cm; + if (!pg_atomic_read_u32(&cl_state->committed_marker_durable)) + cl_drive_committed_marker_stage(leaving, committed_epoch); - cl_build_marker(&cm, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, committed_epoch); - if (cluster_clean_leave_submit_marker(&cm) == CLUSTER_LEAVE_MARKER_SUBMIT_ACK) - pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); - } /* Re-send LEAVE_COMMITTED every tick once durable until the leaver is gone * (best-effort IC): the leaving node will not depart until it receives one, * and step 3 holds the slot until it is CSSD-dead, so delivery is assured. */ diff --git a/src/backend/cluster/cluster_clean_leave_policy.c b/src/backend/cluster/cluster_clean_leave_policy.c index 0a4eeebd04..91a2c4d768 100644 --- a/src/backend/cluster/cluster_clean_leave_policy.c +++ b/src/backend/cluster/cluster_clean_leave_policy.c @@ -136,17 +136,112 @@ cluster_clean_leave_should_abort_writable(bool in_transaction, bool has_top_xid) /* * cluster_clean_leave_version_coherent -- the bound leave is still coherent - * only if neither the cluster epoch nor the CSSD dead_generation moved since - * the leave was bound. Any external bump (a real death intruding mid-drain) - * makes it incoherent → caller must ABORTED_ESCALATE (never complete a clean - * leave on a stale version, which would let a destructive sweep run on a newer - * version and double-grant). + * only if neither the cluster epoch nor the OTHERS-dead set (every dead node + * except the leaving node itself) moved since the leave was bound. Any + * external membership change — a third-party death intruding mid-drain, or a + * third-party fail-stop that already bumped the epoch — makes it incoherent → + * caller must ABORTED_ESCALATE (never complete a clean leave on a stale + * membership view, which would let a destructive sweep run on a newer view + * and double-grant, CL-I3). + * + * spec-2.29a ②b fix: the pre-fix predicate compared the SCALAR global CSSD + * dead_generation, which also counts the leaving node's OWN alive→DEAD + * transition (it stops heart-beating once its drain finishes — the EXPECTED + * terminal state of a clean leave). Under the async COMMITTING marker that + * spans several LMON ticks, a slow environment lets that transition land + * inside the wait window and the scalar check wrongly escalated an otherwise + * healthy leave. Comparing the others-dead bitmap (which excludes the + * leaving node) removes exactly that false positive while keeping every + * third-party incoherence escalation (see the reflexive-case matrix in + * spec-2.29a §②b 8.A argument). */ bool cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 current_epoch, - uint64 bound_dead_gen, uint64 current_dead_gen) + const uint8 *bound_others_dead, + const uint8 *current_others_dead, int nbytes) +{ + if (bound_epoch != current_epoch) + return false; + if (bound_others_dead == NULL || current_others_dead == NULL) + return false; /* fail-closed: cannot prove coherence without both views */ + return memcmp(bound_others_dead, current_others_dead, (size_t)nbytes) == 0; +} + +/* + * cluster_clean_leave_own_commit_latched -- the LEAVING node's barrier tick + * latches "my clean-leave committed" on direct EVIDENCE, never on inference + * (spec-2.29a r3). The evidence is committed_marker_evidence: the survivor + * coordinator made the COMMITTED marker for THIS leave attempt majority- + * durable on the voting disk and attested it with a nonce-bound + * LEAVE_COMMITTED (validated by cluster_clean_leave_committed_evidence_ + * matches before the flag this argument mirrors is ever set). Latching + * suppresses the barrier-deadline escalation and lets the leaver exit, so + * the verdict must be exactly: latch <=> evidence. + * + * History of the two inference failure modes this replaces: + * - r2 P2-1 (mis-latch wedge): the pre-r2 "epoch advanced + others-dead + * bitmap unchanged" inference could mis-latch a REFUSED leave after a + * third-party false-DEAD -> ALIVE rebound restored the (non-monotone) + * bitmap, suppressing the deadline escalation forever. + * - r3 (t/331 C1/C4 false-escalation): the r2 scalar dead_generation + * conjunct is monotone the other way — a third-party transient flap on + * the leaver's local CSSD view advances it forever, so a healthy + * committed leave was refused until the deadline escalated it. + * Marker evidence is immune to both: a refused leave never gets a COMMITTED + * marker (no latch -> bounded deadline escalation), and a flap cannot make + * durable evidence disappear (latch -> no false escalation). + * + * others_dead_unchanged / dead_gen_unchanged are the leaver's live coherence + * observations. They are deliberately kept in the signature as contract + * inputs that MUST NOT affect the verdict — the U3b unit matrix pins the + * flap-immunity on them — and the runtime uses them only for the flap-noise + * LOG at the latch point (observability, never control flow). + */ +bool +cluster_clean_leave_own_commit_latched(bool committed_marker_evidence, bool others_dead_unchanged, + bool dead_gen_unchanged) +{ + (void)others_dead_unchanged; /* observability-only input (see above) */ + (void)dead_gen_unchanged; /* observability-only input (see above) */ + return committed_marker_evidence; +} + +/* + * cluster_clean_leave_committed_evidence_matches -- may the leaving node + * accept a LEAVE_COMMITTED confirmation as marker evidence for THIS leave + * attempt? (spec-2.29a r3; the payload's magic/version/CRC were already + * checked by cluster_clean_leave_announce_payload_valid.) + * + * Identity is bound three ways, all fail-closed: + * - payload_leaving_node == self_node: the confirmation is addressed to + * this node's own leave (LEAVE_COMMITTED is point-to-point, but a + * misrouted frame must still not latch). + * - current_leaving_node == self_node: this node IS currently the leaver + * (not idle, not a survivor of someone else's leave). + * - payload_nonce == current_attempt_nonce: the per-attempt nonce + * (Hardening v1.0.2) pins the confirmation to THIS attempt, so a stale + * LEAVE_COMMITTED — and through it a stale COMMITTED marker — from a + * PREVIOUS leave of the same node can never false-latch a new attempt. + * - payload_epoch > bound_leave_epoch: the committed epoch E the + * coordinator attests must lie past the baseline this leave bound + * (sanity; the commit is a guarded CAS off that baseline). + */ +bool +cluster_clean_leave_committed_evidence_matches(int32 payload_leaving_node, uint64 payload_nonce, + uint64 payload_epoch, int32 self_node, + int32 current_leaving_node, + uint64 current_attempt_nonce, + uint64 bound_leave_epoch) { - return bound_epoch == current_epoch && bound_dead_gen == current_dead_gen; + if (payload_leaving_node != self_node) + return false; + if (current_leaving_node != self_node) + return false; + if (payload_nonce != current_attempt_nonce) + return false; + if (payload_epoch <= bound_leave_epoch) + return false; + return true; } diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 7d966746e6..161085e6be 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -130,6 +130,7 @@ PG_FUNCTION_INFO_V1(cluster_dump_state); #include "cluster/cluster_catalog_stats.h" /* catalog counters (spec-6.14 D10b) */ #include "cluster/cluster_oid_lease.h" /* catalog category (spec-6.14 D10) */ #include "cluster/cluster_relmap_authority.h" /* relmap authority state (spec-6.14 D5) */ +#include "cluster/cluster_xid_authority.h" /* XID authority state (spec-6.15b D7) */ #include "cluster/cluster_remote_xact.h" /* remote outcome counters (spec-4.5a D11) */ #include "cluster/cluster_ic.h" /* ClusterICOps_Active, ClusterICTier */ #include "cluster/cluster_ic_tier1.h" /* listener metadata accessors (Hardening v1.0.1 F3) */ @@ -544,6 +545,10 @@ dump_lmon(ReturnSetInfo *rsinfo) iters = cluster_lmon_main_loop_iters(); emit_row(rsinfo, "lmon", "lmon_main_loop_iters", fmt_int64(iters)); + emit_row(rsinfo, "lmon", "lmon_last_iter_us", fmt_int64((int64)cluster_lmon_last_iter_us())); + emit_row(rsinfo, "lmon", "lmon_max_iter_us", fmt_int64((int64)cluster_lmon_max_iter_us())); + emit_row(rsinfo, "lmon", "lmon_slow_iter_count", + fmt_int64((int64)cluster_lmon_slow_iter_count())); } /* @@ -1195,8 +1200,30 @@ static void dump_grd_recovery(ReturnSetInfo *rsinfo) { ClusterGrdRecoveryCounters c; + uint32 state; cluster_grd_recovery_counters_snapshot(&c); + state = cluster_grd_recovery_state_value(); + emit_row(rsinfo, "grd_recovery", "state", cluster_grd_recovery_state_name(state)); + emit_row(rsinfo, "grd_recovery", "state_enum_value", fmt_int32((int32)state)); + emit_row(rsinfo, "grd_recovery", "last_event_id", + fmt_int64((int64)cluster_grd_recovery_last_event_id())); + emit_row(rsinfo, "grd_recovery", "event_old_epoch", + fmt_int64((int64)cluster_grd_recovery_event_old_epoch())); + emit_row(rsinfo, "grd_recovery", "episode_epoch", + fmt_int64((int64)cluster_grd_recovery_episode_epoch_value())); + emit_row(rsinfo, "grd_recovery", "event_coordinator", + fmt_int32((int32)cluster_grd_recovery_event_coordinator())); + emit_row(rsinfo, "grd_recovery", "done_self_epoch", + fmt_int64((int64)cluster_grd_recovery_done_epoch_for(cluster_node_id))); + emit_row(rsinfo, "grd_recovery", "done_self_bitmap_hash", + fmt_int64((int64)cluster_grd_recovery_done_bitmap_hash_for(cluster_node_id))); + emit_row(rsinfo, "grd_recovery", "block_redeclare_cursor", + fmt_int32((int32)cluster_grd_recovery_block_redeclare_cursor())); + emit_row(rsinfo, "grd_recovery", "block_redeclare_epoch", + fmt_int64((int64)cluster_grd_recovery_block_redeclare_epoch())); + emit_row(rsinfo, "grd_recovery", "block_redeclare_done", + fmt_bool(cluster_grd_recovery_block_redeclare_done())); emit_row(rsinfo, "grd_recovery", "remaster_started", fmt_int64((int64)c.remaster_started)); emit_row(rsinfo, "grd_recovery", "remaster_done", fmt_int64((int64)c.remaster_done)); emit_row(rsinfo, "grd_recovery", "remaster_failed", fmt_int64((int64)c.remaster_failed)); @@ -1212,6 +1239,11 @@ dump_grd_recovery(ReturnSetInfo *rsinfo) emit_row(rsinfo, "grd_recovery", "unaffected_holder_survived", fmt_int64((int64)c.unaffected_holder_survived)); emit_row(rsinfo, "grd_recovery", "stale_holder_swept", fmt_int64((int64)c.stale_holder_swept)); + emit_row(rsinfo, "grd_recovery", "cluster_gate_timeout", + fmt_int64((int64)c.cluster_gate_timeout)); + emit_row(rsinfo, "grd_recovery", "wait_epoch_escape", fmt_int64((int64)c.wait_epoch_escape)); + /* spec-4.6a D12 — dead-node PCM residue cleanup (categorized under pcm). */ + emit_row(rsinfo, "pcm", "dead_cleanup_entries", fmt_int64((int64)c.pcm_dead_cleanup_entries)); /* spec-5.16 D5 — join-direction remaster counters (same grd_recovery * category; no new dump category, §8 Q6-A). */ emit_row(rsinfo, "grd_recovery", "join_remaster_started", @@ -1475,6 +1507,10 @@ dump_reconfig_join(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_reconfig_get_join_timeout_count())); emit_row(rsinfo, "reconfig_join", "clean_departed_cleared_count", fmt_int64((int64)cluster_reconfig_get_clean_departed_cleared_count())); + emit_row(rsinfo, "reconfig", "marker_slow_ack_count", + fmt_int64((int64)cluster_reconfig_get_marker_slow_ack_count())); + emit_row(rsinfo, "reconfig", "marker_timeout_count", + fmt_int64((int64)cluster_reconfig_get_marker_timeout_count())); } /* @@ -2810,6 +2846,12 @@ dump_hw(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_hw_remaster_done_count())); emit_row(rsinfo, "hw", "remaster_blocked_count", fmt_int64((int64)cluster_hw_remaster_blocked_count())); + emit_row(rsinfo, "hw", "remaster_retry_count", + fmt_int64((int64)cluster_hw_remaster_retry_count())); + emit_row(rsinfo, "hw", "remaster_retry_exhausted_count", + fmt_int64((int64)cluster_hw_remaster_retry_exhausted_count())); + emit_row(rsinfo, "hw", "remaster_recoverable", + fmt_int64((int64)(cluster_hw_remaster_recoverable() ? 1 : 0))); /* * spec-6.12d D-obs: space-lease counters. bloat_ratio itself is a @@ -3035,6 +3077,19 @@ dump_catalog(ReturnSetInfo *rsinfo) emit_row(rsinfo, "catalog", "relmap_shared_pending_owner_node", fmt_int64(rmok ? (int64)rmhdr.owner_node : -1)); + /* spec-6.15b D7: native-era XID authority and this-boot adopt marker. */ + { + ClusterXidAuthorityHeader xahdr; + bool xaok = cluster_shared_catalog && cluster_xid_authority_read(&xahdr); + + emit_row(rsinfo, "catalog", "xid_authority_native_hw", + fmt_int64(xaok ? (int64)xahdr.native_hw_full : 0)); + emit_row(rsinfo, "catalog", "xid_authority_sealed", + fmt_bool(xaok && (xahdr.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) != 0)); + emit_row(rsinfo, "catalog", "xid_prehistory_adopted", + fmt_bool(cluster_xid_prehistory_was_adopted())); + } + /* * D10b shared-catalog data-plane counters: cluster-path visibility * resolutions of catalog-page tuples, their fail-closed (53R97-family) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index fd3f3b7817..15d0643910 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -51,11 +51,12 @@ #include "cluster/cluster_gcs_block_dedup.h" /* spec-2.34 D1 — counter forward */ #include "cluster/cluster_grd.h" /* spec-4.6 D4 — block_path_failclosed counter */ #include "cluster/cluster_grd_outbound.h" -#include "cluster/cluster_membership.h" /* spec-5.16 D3b — is_member master-side gate */ -#include "cluster/cluster_qvotec.h" /* spec-5.16 D3b — in_quorum master-side gate */ -#include "cluster/cluster_recovery_merge.h" /* spec-4.7 D5 — recovered_through redo gate */ -#include "cluster/cluster_xnode_profile.h" /* spec-5.59 D2/D3/D4 profiling buckets */ -#include "cluster/cluster_xnode_lever.h" /* spec-6.12a — downgrade counters */ +#include "cluster/cluster_membership.h" /* spec-5.16 D3b — is_member master-side gate */ +#include "cluster/cluster_qvotec.h" /* spec-5.16 D3b — in_quorum master-side gate */ +#include "cluster/cluster_recovery_merge.h" /* spec-4.7 D5 — recovered_through redo gate */ +#include "cluster/cluster_thread_recovery.h" /* spec-4.11 scope gate for online replay */ +#include "cluster/cluster_xnode_profile.h" /* spec-5.59 D2/D3/D4 profiling buckets */ +#include "cluster/cluster_xnode_lever.h" /* spec-6.12a — downgrade counters */ #include "cluster/cluster_guc.h" #include "cluster/cluster_inject.h" #include "cluster/cluster_itl.h" /* spec-5.2 D11 — active-ITL writer-transfer guard */ @@ -65,6 +66,7 @@ #include "cluster/cluster_lmon.h" #include "cluster/cluster_pcm_lock.h" #include "cluster/cluster_shmem.h" +#include "cluster/storage/cluster_shared_fs.h" #include "cluster/cluster_sf_dep.h" #include "cluster/cluster_touched_peers.h" /* spec-5.14 D2 class 2 */ #include "common/hashfn.h" @@ -1199,6 +1201,39 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) return GCS_BLOCK_RECOVERING; } + /* + * r3-P2-1 unseal-safety proof — this predicate is heartbeat LIVENESS + * (CSSD hysteresis flips DEAD->ALIVE on heartbeat receipt alone, + * cluster_cssd.c deadband scan), NOT a direct "instance recovery + * complete" signal. It is nevertheless safe to return NORMAL here, + * because on a crash-restarted master the heartbeat source itself is + * recovery-gated: + * (1) CSSD — the only heartbeat sender — is spawned by the cluster + * phase-4 driver, which the postmaster reaper invokes only at the + * PM_RUN transition, i.e. after the startup process exited 0 and + * the node's crash recovery fully replayed its WAL thread to shared + * storage (the ServerLoop respawn is equally PM_RUN-gated). A + * still-recovering node sends NO heartbeats, so a survivor's DEAD + * verdict cannot flip back early. + * (2) Belt-and-suspenders: even under a stale-ALIVE view (fast restart + * inside the deadband), a fetch cannot complete against a + * still-recovering node. The master-side handler + * (cluster_gcs_handle_block_request_envelope) default-denies unless + * the node is an in-quorum MEMBER, and cluster_qvotec_in_quorum() + * demands QUORUM_OK plus a live lease — state only the QVOTEC + * process (phase-4 / post-PM_RUN as well) can establish after a + * restart wiped shmem. The deny replies map to bounded 53R9L; an + * unresponsive endpoint exhausts the retransmit budget into bounded + * 53R90. Neither path ever falls back to a silent local storage + * read (STORAGE_FALLBACK is a master REPLY status, not a local + * fallback). + * (3) For online_join rejoin, MEMBER additionally requires coordinator + * admission, which vets the joiner's post-recovery voting slot + * (the slot_generation != 0 readiness sub-gate). + * So heartbeat-ALIVE implies the returned master completed its own + * instance recovery: its committed WAL is on shared storage and no + * merged-materialization proof is needed for its blocks. + */ if (static_master == cluster_node_id || cluster_cssd_get_peer_state(static_master) != CLUSTER_CSSD_PEER_DEAD) return GCS_BLOCK_NORMAL; @@ -1221,12 +1256,8 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) /* * spec-4.7 D7 + D5 — static master is DEAD; the block is remastered to a - * live survivor (recovery-aware routing). The survivor may SERVE only - * after the dead origin's merged WAL recovery passes the redo-before- - * unfreeze gate; before that the shared-storage / re-declared version may - * be stale → fail-closed RECOVERING (never a stale page). - * - * TWO conditions, both required (Q5): + * live survivor (recovery-aware routing). Two conditions are both + * required before the on-disk version may be served (Q5): * (a) is_materialized(origin): the dead origin's merged replay completed * (publish is atomic at end-of-replay with the max EndRecPtr). This * is the cold-block safety door — a block NO survivor observed has no @@ -1236,12 +1267,20 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) * survivor DID observe (rebuilt pi_watermark_lsn > 0), the dead * origin's recovered_lsn must reach that observed page_lsn — else the * dead node wrote a version a survivor saw but whose WAL never durably - * reached us → lost-write → fail-closed. This is the LSN comparison - * (NOT a bool), live in the serve path; required_lsn == 0 (cold) is - * trivially covered and (a) carries the safety. + * reached us → lost-write → fail-closed. * - * Once both hold → NORMAL → the re-routed survivor serves (rebuilt-from- - * redeclare for held blocks, lazy minimal view for cold blocks). + * spec-4.6a Amendment v1.2 (R1): this proof is UNCONDITIONAL. Where + * online thread recovery cannot run (GUC off — the default — or any + * >2-node deployment) the materialization authority is never published, + * so a dead master's blocks stay RECOVERING until the failed node + * restarts and completes its own instance recovery (the unseal above is + * heartbeat liveness; the r3-P2-1 note explains why heartbeats imply + * recovery completion): a bounded, retryable + * ERROR on the request path (53R9L), never an unproven serve. A scope + * predicate must never gate a correctness proof — a committed write on + * a cold block that only the dead node saw has NO other guard on this + * read path (GRD freeze ends with the episode; the HW gate covers only + * extend high-water marks; pd_block_scn checks ride the ship path). */ if (!cluster_merged_instance_is_materialized(static_master)) { if (ClusterGcsBlock != NULL) diff --git a/src/backend/cluster/cluster_ges.c b/src/backend/cluster/cluster_ges.c index a81f80ee2b..f68e407239 100644 --- a/src/backend/cluster/cluster_ges.c +++ b/src/backend/cluster/cluster_ges.c @@ -550,9 +550,12 @@ cluster_ges_request_handler(const ClusterICEnvelope *env, const void *payload) /* spec-4.6 P0#3 cluster gate — fire-and-forget barrier announcement: * record and return (no work queue, no reply, no dedup; the sender - * re-announces each tick, the receiver write is a monotonic max). */ + * re-announces each tick, the receiver write is a monotonic max). + * Amendment v1.2 (R2): the request-id field pair carries the sender's + * dead_bitmap hash (composite convergence key), not an event_id. */ if (req->opcode == GES_REQ_OPCODE_REDECLARE_DONE) { - cluster_grd_recovery_mark_peer_done((int32)env->source_node_id, holder_epoch); + cluster_grd_recovery_mark_peer_done((int32)env->source_node_id, holder_epoch, + ges_request_holder_request_id(req)); return; } diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index 33c5b2b7c9..2088c113ec 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -46,7 +46,8 @@ #include "cluster/cluster_pcm_lock.h" /* spec-2.36 HC124 pending_x node-dead cleanup */ #include "cluster/cluster_gcs.h" /* spec-4.7 D2 — cluster_gcs_lookup_master */ #include "cluster/cluster_membership.h" /* spec-5.16 D1 — cluster_membership_is_member */ -#include "cluster/cluster_gcs_block.h" /* spec-4.7 D2 — block re-declare scan + send */ +#include "cluster/cluster_native_lock_probe.h" +#include "cluster/cluster_gcs_block.h" /* spec-4.7 D2 — block re-declare scan + send */ #include "cluster/cluster_signal.h" #include "cluster/cluster_shmem.h" #include "cluster/cluster_cssd.h" /* spec-2.16 D8 newly-dead bitmap diff */ @@ -731,9 +732,14 @@ cluster_grd_shmem_init(void) pg_atomic_init_u64(&cluster_grd_state->recovery_event_old_epoch, 0); pg_atomic_init_u64(&cluster_grd_state->recovery_redeclare_generation, 0); pg_atomic_init_u64(&cluster_grd_state->recovery_barrier_deadline, 0); + pg_atomic_init_u32(&cluster_grd_state->recovery_event_coordinator, 0); + pg_atomic_init_u64(&cluster_grd_state->recovery_done_epoch_at_accept, 0); /* spec-4.6 P0#3 cluster gate — per-node barrier-done epochs. */ - for (i = 0; i < CLUSTER_MAX_NODES; i++) + for (i = 0; i < CLUSTER_MAX_NODES; i++) { pg_atomic_init_u64(&cluster_grd_state->recovery_done_epoch[i], 0); + pg_atomic_init_u64(&cluster_grd_state->recovery_done_bitmap_hash[i], 0); + } + pg_atomic_init_u64(&cluster_grd_state->recovery_event_bitmap_hash, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_started_count, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_done_count, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_failed_count, 0); @@ -747,6 +753,9 @@ cluster_grd_shmem_init(void) pg_atomic_init_u64(&cluster_grd_state->block_path_failclosed_count, 0); pg_atomic_init_u64(&cluster_grd_state->unaffected_holder_survived_count, 0); pg_atomic_init_u64(&cluster_grd_state->stale_holder_swept_count, 0); + pg_atomic_init_u64(&cluster_grd_state->cluster_gate_timeout_count, 0); + pg_atomic_init_u64(&cluster_grd_state->wait_epoch_escape_count, 0); + pg_atomic_init_u64(&cluster_grd_state->pcm_dead_cleanup_entries, 0); /* spec-5.16 D2/D3b/D5 — online-join remaster fence + counters. */ pg_atomic_init_u64(&cluster_grd_state->join_pcm_fence_epoch, 0); @@ -1630,6 +1639,99 @@ cluster_grd_redeclare_episode_epoch(void) * been re-declared to its recovery-aware master — serving it would risk an * 8.A double-grant. Reaching IDLE implies all survivor scans completed. */ +uint32 +cluster_grd_recovery_state_value(void) +{ + if (cluster_grd_state == NULL) + return (uint32)GRD_RECOVERY_IDLE; + return pg_atomic_read_u32(&cluster_grd_state->recovery_state); +} + +const char * +cluster_grd_recovery_state_name(uint32 state) +{ + switch ((ClusterGrdRecoveryState)state) { + case GRD_RECOVERY_IDLE: + return "idle"; + case GRD_RECOVERY_WAIT_EPOCH: + return "wait_epoch"; + case GRD_RECOVERY_WAIT_BARRIER: + return "wait_barrier"; + case GRD_RECOVERY_WAIT_CLUSTER: + return "wait_cluster"; + } + return "unknown"; +} + +uint64 +cluster_grd_recovery_last_event_id(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id) + : 0; +} + +uint64 +cluster_grd_recovery_episode_epoch_value(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u64(&cluster_grd_state->recovery_episode_epoch) + : 0; +} + +uint32 +cluster_grd_recovery_event_coordinator(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u32(&cluster_grd_state->recovery_event_coordinator) + : 0; +} + +uint64 +cluster_grd_recovery_done_epoch_for(int32 node) +{ + if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); +} + +uint64 +cluster_grd_recovery_done_bitmap_hash_for(int32 node) +{ + if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[node]); +} + +uint64 +cluster_grd_recovery_event_bitmap_hash_value(void) +{ + if (cluster_grd_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); +} + +/* + * cluster_grd_dead_bitmap_hash — spec-4.6a Amendment v1.2 (R2). + * + * The cross-node half of the REDECLARE_DONE convergence key: a hash over + * the sender's ACCEPTED dead bitmap ALONE (each node's own local event — + * there is no cross-node ratification of the bitmap; see the P0 stamp-site + * note and the r3-P2-2 mid-episode re-consume guard for how detection skew + * converges). Same kernel as the event_id hash + * (cluster_reconfig_compute_event_id) minus the cssd_dead_generation fold — + * the generation is per-instance observation history and does NOT converge + * across survivors, which is exactly why event_id cannot key the P6 gate. + * A JOIN episode's dead bitmap is all-zero, hashing identically everywhere + * (a fixed NONZERO constant — 0 stays reserved for "unstamped"), so the + * composite key degrades to the epoch-only gate there. + */ +uint64 +cluster_grd_dead_bitmap_hash(const uint8 *dead_bitmap) +{ + return hash_bytes_extended(dead_bitmap, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES, 0); +} + bool cluster_grd_recovery_in_progress(void) { @@ -1638,6 +1740,20 @@ cluster_grd_recovery_in_progress(void) return pg_atomic_read_u32(&cluster_grd_state->recovery_state) != (uint32)GRD_RECOVERY_IDLE; } +/* + * spec-2.29a: the pre-reconfig baseline epoch the WAIT_EPOCH gate compares + * against. Exposed for diagnostics + the IDLE baseline-hold unit test (the + * async pre-bump staging window must not let this re-capture a post-bump + * value; see the IDLE branch of cluster_grd_recovery_lmon_tick). + */ +uint64 +cluster_grd_recovery_event_old_epoch(void) +{ + if (cluster_grd_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_event_old_epoch); +} + /* spec-4.6 D4/D5 — recovery counter bump helpers for out-of-module * call sites (S4 stale mapping in cluster_lock_acquire.c; GCS block * fail-closed guard in cluster_gcs_block.c). */ @@ -1686,6 +1802,10 @@ cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out) out->unaffected_holder_survived = pg_atomic_read_u64(&cluster_grd_state->unaffected_holder_survived_count); out->stale_holder_swept = pg_atomic_read_u64(&cluster_grd_state->stale_holder_swept_count); + out->cluster_gate_timeout = pg_atomic_read_u64(&cluster_grd_state->cluster_gate_timeout_count); + out->wait_epoch_escape = pg_atomic_read_u64(&cluster_grd_state->wait_epoch_escape_count); + out->pcm_dead_cleanup_entries + = pg_atomic_read_u64(&cluster_grd_state->pcm_dead_cleanup_entries); /* spec-5.16 D5 — join-direction remaster counters. */ out->join_remaster_started = pg_atomic_read_u64(&cluster_grd_state->join_remaster_started_count); @@ -1924,6 +2044,39 @@ grd_recovery_barrier_complete(uint64 gen, uint64 episode_epoch) return true; } +static void +grd_recovery_format_waiting_backend(uint64 gen, uint64 episode_epoch, char *buf, Size buflen) +{ + int beid; + pid_t self_pid = MyProcPid; + + if (buflen == 0) + return; + buf[0] = '\0'; + for (beid = 1; beid <= MaxBackends; beid++) { + PGPROC *proc = BackendIdGetProc((BackendId)beid); + uint32 registered; + uint64 acked; + uint64 acked_epoch; + + if (proc == NULL || proc->pid == 0 || proc->pid == self_pid) + continue; + registered = pg_atomic_read_u32(&proc->cluster_grd_registered_count); + if (registered == 0) + continue; + acked = pg_atomic_read_u64(&proc->cluster_grd_redeclare_acked); + acked_epoch = pg_atomic_read_u64(&proc->cluster_grd_redeclare_acked_epoch); + if (acked < gen || acked_epoch != episode_epoch) { + snprintf(buf, buflen, + "beid=%d pid=%d registered=%u acked=" UINT64_FORMAT "/" UINT64_FORMAT + " target=" UINT64_FORMAT "/" UINT64_FORMAT, + beid, (int)proc->pid, registered, acked, acked_epoch, gen, episode_epoch); + return; + } + } + snprintf(buf, buflen, "none"); +} + /* * spec-4.6 P0#3 cluster gate — announce "my local rebind barrier is * complete for `epoch`" to every declared peer (fire-and-forget; the @@ -1934,6 +2087,11 @@ static void grd_recovery_broadcast_done(uint64 epoch) { GesRequestPayload req; + /* Amendment v1.2 (R2): the DONE payload carries the sender's accepted + * dead-bitmap hash (cross-node convergence key), riding the request-id + * field pair — NOT the sender-local event_id (its dead_generation fold + * diverges across survivors' flap observation histories). */ + uint64 bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); uint64 master_gen = cluster_lms_get_shard_master_generation(); int i; @@ -1943,6 +2101,8 @@ grd_recovery_broadcast_done(uint64 epoch) req.holder_procno = 0; req.holder_cluster_epoch_lo = (uint32)(epoch & 0xffffffffu); req.holder_cluster_epoch_hi = (uint32)(epoch >> 32); + req.holder_request_id_lo = (uint32)(bitmap_hash & 0xffffffffu); + req.holder_request_id_hi = (uint32)(bitmap_hash >> 32); req.shard_master_generation_lo = (uint32)(master_gen & 0xffffffffu); req.shard_master_generation_hi = (uint32)(master_gen >> 32); @@ -1961,19 +2121,53 @@ grd_recovery_broadcast_done(uint64 epoch) /* REDECLARE_DONE receiver (cluster_ges.c inbound handler). */ void -cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch) +cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap_hash) { - uint64 prev; + uint64 episode_bitmap_hash; if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) return; - /* Monotonic max: late/duplicate announcements never regress. */ - prev = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); - while (epoch > prev) { - if (pg_atomic_compare_exchange_u64(&cluster_grd_state->recovery_done_epoch[node], &prev, - epoch)) - break; + + /* + * spec-4.6a Amendment v1.2 (R2): REDECLARE_DONE converges on the COMPOSITE + * key (episode_epoch, dead_bitmap_hash). A stale DONE from a previous + * episode can carry the same epoch when cur==old, so epoch monotonicity + * alone is not enough — but the previous episode's dead SET necessarily + * differs (a coordinator re-election adds the old coordinator to it; a + * re-death of the same node rides a higher epoch via the interposed JOIN + * bump), so the bitmap hash is the ABA guard. Flap-history drift changes + * only the per-instance dead_generation, never the sender's accepted dead + * SET, so same-set DONEs match here (the R2 wedge fix). r3-P2-2: the + * bitmap is NOT quorum-ratified — under multi-death detection skew a + * behind survivor transiently stamps (and announces) a SMALLER set's hash + * at the same epoch; those frames are dropped HERE until its own CSSD + * catches up and the mid-episode re-consume guard re-stamps + re-announces + * the full set (grd_recovery_consume_new_event_mid_episode). Per-tick + * re-announce then converges the accounting. + * Pre-accept frames mismatch the previous episode's stamp and are dropped; + * senders re-announce every tick, so accounting lands after our P0 accept + * (which also closes the R4 pre-accept window by construction). + * + * r3-P3(a) — the `== 0` arm treats 0 as "unstamped/pre-accept" (our own + * recovery_event_bitmap_hash is 0 before the first P0 accept). This + * relies on a stamped hash never being 0: JOIN episodes hash an ALL-ZERO + * bitmap and hash_bytes_extended(zero[16], 0) is a fixed nonzero + * constant (asserted at the stamp site). + */ + episode_bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); + if (dead_bitmap_hash == 0 || dead_bitmap_hash != episode_bitmap_hash) + return; + + /* spec-4.6a §2.3: monotonic-max accounting. Under strictly-monotonic + * per-event epochs the incoming value always wins, but never regress the + * published value if a delayed lower-epoch frame ever slips through. */ + { + uint64 prev = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); + + if (epoch > prev) + pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[node], epoch); } + pg_atomic_write_u64(&cluster_grd_state->recovery_done_bitmap_hash[node], dead_bitmap_hash); } /* @@ -2000,6 +2194,199 @@ grd_recovery_abort_to_idle(void) "mid-recovery); shards stay frozen, re-running under the new epoch"))); } +/* + * grd_recovery_consume_new_event_mid_episode — spec-4.6a r3-P2-2. + * + * WAIT_BARRIER / WAIT_CLUSTER guard against a NEWER local reconfig event at + * the SAME epoch. Multi-death detection skew: when two nodes die near- + * simultaneously the coordinator can fold both into ONE epoch bump with dead + * set {A,B}, while a survivor whose CSSD saw only {B} first stamps + * recovery_event_bitmap_hash = H({B}) and sails past WAIT_EPOCH on that + * bump. Its later local detection of A publishes a new event with dead set + * {A,B} but does NOT move the epoch, so the WAIT_BARRIER / WAIT_CLUSTER + * epoch guards never fire, the survivor keeps announcing DONE under H({B}), + * every peer that stamped H({A,B}) keeps dropping it (composite-key + * mismatch), and P6 — which has no timeout by design — wedges permanently. + * + * Disposition (runs in the single-writer LMON tick, after the epoch guard): + * - new event, FAIL direction, SAME dead-bitmap hash: only the sender- + * local dead_generation fold moved (flap drift re-fired the same quorum + * dead set). Absorb the event_id — the episode semantics (epoch, dead + * set) are unchanged, so re-running P1-P7 would be pure churn. + * Absorbing also keeps the IDLE dedup from re-consuming the event after + * this episode completes. + * - new event with a DIFFERENT dead-bitmap hash (grown dead set — the + * skew above — or a JOIN/FAIL interleave): abort to IDLE. The next + * tick re-consumes the current event, re-stamps + * recovery_event_bitmap_hash, re-runs recovery against the fuller dead + * set, and re-announces DONE under the new composite key — closing the + * survivor-behind wedge. The coordinator-behind direction needs no + * handling here: the coordinator's own later detection bumps the epoch + * again and the existing epoch guards abort. + * + * Returns true when the caller must return (episode aborted to IDLE). + */ +static bool +grd_recovery_consume_new_event_mid_episode(void) +{ + ReconfigEvent latest; + + cluster_reconfig_get_last_event(&latest); + if (latest.event_id == 0 + || latest.event_id == pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) + return false; + + if (latest.reconfig_kind == (uint8)RECONFIG_KIND_FAIL_STOP + && pg_atomic_read_u32(&cluster_grd_state->recovery_direction) + == (uint32)GRD_REMASTER_DIR_FAIL + && cluster_grd_dead_bitmap_hash(latest.dead_bitmap) + == pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)) { + pg_atomic_write_u64(&cluster_grd_state->recovery_last_event_id, latest.event_id); + return false; + } + + grd_recovery_abort_to_idle(); + return true; +} + +static void +grd_recovery_appendf(char *buf, Size buflen, int *off, const char *fmt, ...) +{ + va_list ap; + int n; + Size avail; + + if (buflen == 0 || *off >= (int)buflen - 1) + return; + avail = buflen - (Size)*off; + va_start(ap, fmt); + n = vsnprintf(buf + *off, avail, fmt, ap); + va_end(ap); + if (n < 0) + return; + if ((Size)n >= avail) + *off = (int)buflen - 1; + else + *off += n; +} + +static void +grd_recovery_format_missing_survivors(const uint64 *dead, uint64 episode_epoch, bool is_join, + char *buf, Size buflen) +{ + int off = 0; + int i; + bool any = false; + + buf[0] = '\0'; + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + uint64 done_epoch; + + if (cluster_conf_lookup_node(i) == NULL) + continue; + if (grd_dead_bitmap_test(dead, i)) + continue; + if (is_join && join_fence_is_recipient_for(i, episode_epoch)) + continue; + done_epoch = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]); + if (done_epoch >= episode_epoch + && pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[i]) + == pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)) + continue; + grd_recovery_appendf(buf, buflen, &off, + "%s%d(done=" UINT64_FORMAT "/bitmap_hash=" UINT64_FORMAT ")", + any ? "," : "", i, done_epoch, + pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[i])); + any = true; + } + if (!any) + grd_recovery_appendf(buf, buflen, &off, "none"); +} + +static void +grd_recovery_format_hw_dead(const uint64 *dead, char *buf, Size buflen) +{ + int off = 0; + int i; + bool any = false; + + buf[0] = '\0'; + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + ClusterHwRemasterResult result; + + if (!grd_dead_bitmap_test(dead, i) || i == cluster_node_id) + continue; + result = cluster_hw_remaster_result(i); + grd_recovery_appendf(buf, buflen, &off, "%s%d:%s/%u", any ? "," : "", i, + cluster_hw_remaster_result_name(result), + cluster_hw_remaster_attempts(i)); + any = true; + } + if (!any) + grd_recovery_appendf(buf, buflen, &off, "none"); +} + +static bool +grd_recovery_adopt_observed_epoch(const uint64 *dead, uint64 cur_epoch) +{ + uint64 max_epoch = cur_epoch; + int i; + + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + uint64 peer_epoch; + + if (i == cluster_node_id || cluster_conf_lookup_node(i) == NULL) + continue; + if (grd_dead_bitmap_test(dead, i)) + continue; + peer_epoch = cluster_reconfig_get_observed_epoch(i); + if (peer_epoch > max_epoch) + max_epoch = peer_epoch; + } + if (max_epoch <= cur_epoch) + return false; + cluster_epoch_adopt_admitted(max_epoch); + return true; +} + +static void +grd_recovery_wait_cluster_watchdog(const uint64 *dead, uint64 episode_epoch) +{ + TimestampTz now; + TimestampTz deadline; + TimestampTz next_deadline; + char missing[512]; + char hw_dead[512]; + bool thread_gate; + bool hw_gate; + bool is_join; + + deadline = (TimestampTz)pg_atomic_read_u64(&cluster_grd_state->recovery_barrier_deadline); + now = GetCurrentTimestamp(); + if (deadline == 0 || now <= deadline) + return; + + is_join = (pg_atomic_read_u32(&cluster_grd_state->recovery_direction) + == (uint32)GRD_REMASTER_DIR_JOIN); + grd_recovery_format_missing_survivors(dead, episode_epoch, is_join, missing, sizeof(missing)); + thread_gate = cluster_thread_recovery_gate_unfreeze(dead, (CLUSTER_MAX_NODES + 63) / 64); + hw_gate = cluster_hw_remaster_gate_unfreeze(); + grd_recovery_format_hw_dead(dead, hw_dead, sizeof(hw_dead)); + + pg_atomic_fetch_add_u64(&cluster_grd_state->cluster_gate_timeout_count, 1); + ereport( + WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_CLUSTER watchdog fired; affected shards stay frozen"), + errdetail("episode_epoch=" UINT64_FORMAT ", missing_survivors=%s, " + "thread_gate=%s, hw_gate=%s, hw_dead=%s", + episode_epoch, missing, thread_gate ? "held" : "open", hw_gate ? "held" : "open", + hw_dead), + errhint("This watchdog is observational only; it never unfreezes a shard."))); + next_deadline = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, (uint64)next_deadline); +} + /* * spec-4.7 D2 — survivor block re-declare scan (Q6-A' worker-centric). @@ -2078,6 +2465,24 @@ grd_block_redeclare_scan_complete(uint64 episode_epoch) return grd_block_redeclare_epoch == episode_epoch && grd_block_redeclare_done; } +int +cluster_grd_recovery_block_redeclare_cursor(void) +{ + return grd_block_redeclare_cursor; +} + +uint64 +cluster_grd_recovery_block_redeclare_epoch(void) +{ + return grd_block_redeclare_epoch; +} + +bool +cluster_grd_recovery_block_redeclare_done(void) +{ + return grd_block_redeclare_done; +} + void cluster_grd_recovery_lmon_tick(void) { @@ -2108,9 +2513,19 @@ cluster_grd_recovery_lmon_tick(void) * fires, making old_epoch already == the post-bump epoch and * wedging WAIT_EPOCH forever (cur <= old). The last stable * idle epoch is the reliable "before reconfig" value. + * + * spec-2.29a nightly-regression fix: the coordinator's own + * async pre-bump stages (fail-stop / node-remove / join + * Phase-1) bump the epoch several ticks before publishing the + * event. During that window this IDLE tick must NOT re-capture + * the already-bumped epoch as the baseline, or the P0 accept + * that follows the publish reads old == cur and wedges + * WAIT_EPOCH — the coordinator wedging itself, no IC piggyback + * needed. Hold the last stable pre-reconfig baseline instead. */ - pg_atomic_write_u64(&cluster_grd_state->recovery_event_old_epoch, - cluster_epoch_get_current()); + if (!cluster_reconfig_has_pending_prebump_stage()) + pg_atomic_write_u64(&cluster_grd_state->recovery_event_old_epoch, + cluster_epoch_get_current()); return; } @@ -2119,6 +2534,18 @@ cluster_grd_recovery_lmon_tick(void) * epoch (the genuine pre-reconfig baseline) — do NOT overwrite it * with evt.old_epoch here. */ pg_atomic_write_u64(&cluster_grd_state->recovery_last_event_id, ev_id); + pg_atomic_write_u32(&cluster_grd_state->recovery_event_coordinator, + (uint32)evt.coordinator_node_id); + if (evt.coordinator_node_id >= 0 && evt.coordinator_node_id < CLUSTER_MAX_NODES) + pg_atomic_write_u64( + &cluster_grd_state->recovery_done_epoch_at_accept, + pg_atomic_read_u64( + &cluster_grd_state->recovery_done_epoch[evt.coordinator_node_id])); + else + pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch_at_accept, 0); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, + (uint64)TimestampTzPlusMilliseconds(GetCurrentTimestamp(), + cluster_grd_rebuild_timeout_ms)); for (b = 0; b < (CLUSTER_MAX_NODES + 63) / 64; b++) { uint64 word = 0; int j; @@ -2131,6 +2558,28 @@ cluster_grd_recovery_lmon_tick(void) } pg_atomic_write_u64(&cluster_grd_state->recovery_dead_bitmap[b], word); } + /* Amendment v1.2 (R2) + r3-P2-2: stamp the composite convergence key's + * cross-node half. The hash is over THIS node's OWN accepted event's + * dead bitmap — there is NO cross-node ratification of the bitmap (the + * envelope propagates only the epoch). Convergence is eventual, not + * instantaneous: every survivor's CSSD deadband converges on the same + * dead set, and until it does, a survivor that accepted a SMALLER set + * at the same epoch stamps a different hash. The mid-episode + * re-consume guard (grd_recovery_consume_new_event_mid_episode) closes + * that skew: the survivor's own later detection re-stamps the full + * set's hash and re-announces DONE. dead_generation drift alone never + * changes the bitmap, so same-set re-fires hash identically. */ + pg_atomic_write_u64(&cluster_grd_state->recovery_event_bitmap_hash, + cluster_grd_dead_bitmap_hash(evt.dead_bitmap)); + /* r3-P3(a) — 0 is reserved as "unstamped/pre-accept" in + * mark_peer_done's drop test, so a stamped episode hash must never be + * 0. JOIN episodes hash an ALL-ZERO dead bitmap: the invariant is + * that hash_bytes_extended(zero[16], 0) != 0 (a fixed nonzero + * constant). For FAIL bitmaps a zero hash is a 2^-64 collision, the + * same trust level as the event_id hash; the Assert turns the + * otherwise-undebuggable "all DONEs dropped" wedge into a loud stop + * on cassert builds. */ + Assert(pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash) != 0); /* * spec-5.16 D2 — record the remaster direction and, for JOIN, arm the * joiner-home PCM block fence on THIS node. The joiner already armed it @@ -2174,35 +2623,120 @@ cluster_grd_recovery_lmon_tick(void) int i; int signaled; + for (i = 0; i < (CLUSTER_MAX_NODES + 63) / 64; i++) + dead[i] = pg_atomic_read_u64(&cluster_grd_state->recovery_dead_bitmap[i]); + /* - * Gate on the ACCEPTED epoch having advanced past the episode's - * old epoch: the coordinator bumped it earlier this tick; a - * non-coordinator survivor observes it via IC envelope piggyback - * a tick or two later. Running the rebind before the local - * epoch advances would mint holders the new master rejects. - * - * spec-5.16 (P0, Rule 8.A) — direction-aware. A FAIL survivor captures - * old_epoch BEFORE the coordinator bumps, so it must wait for a STRICT - * advance (cur > old) to prove it adopted the new master view. A JOIN - * observer survivor instead adopts the coordinator's JOIN epoch bump (via the - * joiner_self_tick max-peer-epoch adoption, same LMON tick) BEFORE it readmits - * the rejoiner and publishes its own observer JOIN_COMMITTED event, so by the - * time its FSM accepts that event old_epoch == cur_epoch == the join epoch. - * Requiring a strict advance there wedges the survivor in WAIT_EPOCH forever: - * it never runs the re-declare barrier, never broadcasts REDECLARE_DONE, and - * the coordinator's all-members JOIN barrier (Hardening v1.1) hangs → - * join_remaster_done never advances. For JOIN the survivor already holds the - * epoch it rebinds under (that IS the post-bump master view), so equality is - * sufficient; cur_epoch is monotonic so cur < old never occurs for JOIN. + * Gate on the ACCEPTED epoch having advanced past the episode's old epoch. + * FAIL requires proof of a post-bump epoch; JOIN keeps its existing equality + * allowance because the observer already accepted the coordinator's JOIN epoch. + * For FAIL, deadline expiry is not an escape hatch by itself: we either adopt + * a durable observed epoch or require a coordinator REDECLARE_DONE witness for + * cur==old. Without either proof, stay fail-closed in WAIT_EPOCH. */ - if (pg_atomic_read_u32(&cluster_grd_state->recovery_direction) - == (uint32)GRD_REMASTER_DIR_JOIN - ? cur_epoch < old_epoch - : cur_epoch <= old_epoch) - return; + { + ReconfigEvent latest; + bool is_join; + + cluster_reconfig_get_last_event(&latest); + if (latest.event_id != 0 + && latest.event_id + != pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) { + grd_recovery_abort_to_idle(); + return; + } - for (i = 0; i < (CLUSTER_MAX_NODES + 63) / 64; i++) - dead[i] = pg_atomic_read_u64(&cluster_grd_state->recovery_dead_bitmap[i]); + is_join = (pg_atomic_read_u32(&cluster_grd_state->recovery_direction) + == (uint32)GRD_REMASTER_DIR_JOIN); + if (is_join) { + if (cur_epoch < old_epoch) + return; + } else if (cur_epoch <= old_epoch) { + TimestampTz now = GetCurrentTimestamp(); + TimestampTz wait_deadline; + TimestampTz next_deadline; + uint32 coordinator; + uint64 coord_done; + uint64 coord_done_bitmap_hash; + uint64 episode_bitmap_hash; + uint64 coord_done_at_accept; + + wait_deadline = (TimestampTz)pg_atomic_read_u64( + &cluster_grd_state->recovery_barrier_deadline); + if (wait_deadline == 0 || now <= wait_deadline) + return; + + if (grd_recovery_adopt_observed_epoch(dead, cur_epoch)) { + next_deadline + = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, + (uint64)next_deadline); + ereport( + WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH adopted a durable observed epoch; " + "affected shards stay frozen until recovery completes"))); + return; + } + + coordinator = pg_atomic_read_u32(&cluster_grd_state->recovery_event_coordinator); + episode_bitmap_hash + = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); + coord_done + = coordinator < CLUSTER_MAX_NODES + ? pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[coordinator]) + : 0; + coord_done_bitmap_hash + = coordinator < CLUSTER_MAX_NODES + ? pg_atomic_read_u64( + &cluster_grd_state->recovery_done_bitmap_hash[coordinator]) + : 0; + coord_done_at_accept + = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch_at_accept); + /* + * spec-4.6a §2.3 witness, composite-key form (Amendment v1.2 + * R2): the coordinator's DONE must (a) name this episode's + * quorum dead SET (bitmap hash — the cross-node-consistent + * key; the sender-local event_id folds the per-instance + * dead_generation and diverges under flap asymmetry), (b) + * name exactly cur as the episode epoch, and (c) have been + * accounted after our P0 accept snapshot. Under strictly + * monotonic per-event epochs (c) is implied by (a)+(b) — any + * pre-accept residue carries a lower epoch — but it stays as a + * belt-and-suspenders guard against accounting bugs. + */ + if (cur_epoch == old_epoch && coord_done == cur_epoch + && coord_done_bitmap_hash == episode_bitmap_hash + && coord_done > coord_done_at_accept) { + pg_atomic_fetch_add_u64(&cluster_grd_state->wait_epoch_escape_count, 1); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH used coordinator witness for " + "equal-epoch progress"), + errdetail("coordinator=%u, epoch=" UINT64_FORMAT + ", dead_bitmap_hash=" UINT64_FORMAT, + coordinator, cur_epoch, episode_bitmap_hash))); + } else { + next_deadline + = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, + (uint64)next_deadline); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH has no post-bump proof; " + "affected shards stay frozen"), + errdetail("cur_epoch=" UINT64_FORMAT ", old_epoch=" UINT64_FORMAT + ", coordinator=%u, coordinator_done=" UINT64_FORMAT + ", coordinator_done_bitmap_hash=" UINT64_FORMAT + ", episode_bitmap_hash=" UINT64_FORMAT + ", coordinator_done_at_accept=" UINT64_FORMAT, + cur_epoch, old_epoch, coordinator, coord_done, + coord_done_bitmap_hash, episode_bitmap_hash, + coord_done_at_accept))); + return; + } + } + } /* * P1 freeze affected + P3 scoped sweep + P4 remaster. Direction- @@ -2306,6 +2840,7 @@ cluster_grd_recovery_lmon_tick(void) if (state == (uint32)GRD_RECOVERY_WAIT_BARRIER) { uint64 gen = pg_atomic_read_u64(&cluster_grd_state->recovery_redeclare_generation); uint64 episode_epoch = pg_atomic_read_u64(&cluster_grd_state->recovery_episode_epoch); + TimestampTz deadline; /* P0-1 epoch-coherence guard: a SECOND epoch bump landed * mid-episode (e.g. a third node's heartbeat flap re-fired @@ -2317,6 +2852,12 @@ cluster_grd_recovery_lmon_tick(void) return; } + /* r3-P2-2 — a newer local event at the SAME epoch (multi-death + * detection skew grew the dead set) re-consumes; same-set drift is + * absorbed without churn. */ + if (grd_recovery_consume_new_event_mid_episode()) + return; + /* spec-4.7 D2 — advance the survivor block re-declare scan one chunk * while the GES rebind barrier is still pending (worker-centric, runs * in this tick; epoch-coherent via the guard just above). */ @@ -2344,7 +2885,12 @@ cluster_grd_recovery_lmon_tick(void) */ pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[cluster_node_id], episode_epoch); + pg_atomic_write_u64(&cluster_grd_state->recovery_done_bitmap_hash[cluster_node_id], + pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)); grd_recovery_broadcast_done(episode_epoch); + deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), + cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, (uint64)deadline); pg_atomic_write_u32(&cluster_grd_state->recovery_state, (uint32)GRD_RECOVERY_WAIT_CLUSTER); ereport(DEBUG1, @@ -2358,6 +2904,12 @@ cluster_grd_recovery_lmon_tick(void) if (GetCurrentTimestamp() > (TimestampTz)pg_atomic_read_u64(&cluster_grd_state->recovery_barrier_deadline)) { TimestampTz deadline; + char waiting_backend[256]; + bool ges_barrier_complete = grd_recovery_barrier_complete(gen, episode_epoch); + bool block_scan_complete = grd_block_redeclare_scan_complete(episode_epoch); + + grd_recovery_format_waiting_backend(gen, episode_epoch, waiting_backend, + sizeof(waiting_backend)); /* * Barrier deadline expired: fail-closed. Affected shards @@ -2376,6 +2928,12 @@ cluster_grd_recovery_lmon_tick(void) (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), errmsg("cluster GRD holder-rebuild barrier timed out; affected shards " "stay frozen"), + errdetail("gen=" UINT64_FORMAT ", episode_epoch=" UINT64_FORMAT + ", ges_barrier=%s, block_scan=%s, block_cursor=%d" + ", block_epoch=" UINT64_FORMAT ", waiting_backend=%s", + gen, episode_epoch, ges_barrier_complete ? "done" : "waiting", + block_scan_complete ? "done" : "waiting", grd_block_redeclare_cursor, + grd_block_redeclare_epoch, waiting_backend), errhint("A backend has not acked the cooperative rebind within " "cluster.grd_rebuild_timeout_ms; re-broadcasting."))); deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), @@ -2401,6 +2959,13 @@ cluster_grd_recovery_lmon_tick(void) return; } + /* r3-P2-2 — same-epoch dead-set growth re-consumes here too: this + * node may already have announced DONE under the stale bitmap hash; + * the re-run re-announces under the full set's hash so peers' + * composite-key accounting converges (P6 has no timeout). */ + if (grd_recovery_consume_new_event_mid_episode()) + return; + /* spec-4.7 D2 — keep advancing the block re-declare scan after the GES * rebind barrier completed, so a large pool is fully re-declared within * the recovery window (no-op once the cursor reaches NBuffers). */ @@ -2409,6 +2974,8 @@ cluster_grd_recovery_lmon_tick(void) for (i = 0; i < (CLUSTER_MAX_NODES + 63) / 64; i++) dead[i] = pg_atomic_read_u64(&cluster_grd_state->recovery_dead_bitmap[i]); + grd_recovery_wait_cluster_watchdog(dead, episode_epoch); + /* * spec-4.11 D1 (3b-4b Part 3) — launch one per-episode online thread- * recovery worker for each in-scope dead origin (idempotent per tick; @@ -2461,8 +3028,13 @@ cluster_grd_recovery_lmon_tick(void) */ if (is_join && join_fence_is_recipient_for(i, episode_epoch)) continue; - if (pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]) - < episode_epoch) { + /* Amendment v1.2 (R2): composite key — the peer's DONE must name + * this episode's epoch AND the same quorum dead SET. Keying on + * event_id here wedged the gate whenever survivors' private + * dead_generation histories diverged (flap asymmetry). */ + if (pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]) < episode_epoch + || pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[i]) + != pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)) { all_done = false; break; } @@ -5344,13 +5916,28 @@ cluster_grd_cleanup_on_node_dead(int32 dead_node_id) int swept = 0; int nresids; uint64 pending_x_cleared = 0; + uint64 pcm_holders_cleaned = 0; int i; pending_x_cleared = cluster_pcm_lock_clear_pending_x_for_node(dead_node_id); - if (pending_x_cleared > 0) - ereport(DEBUG1, (errmsg_internal("cluster_grd_cleanup_on_node_dead(%d): " - "cleared " UINT64_FORMAT " PCM pending_x entries", - dead_node_id, pending_x_cleared))); + pcm_holders_cleaned = cluster_pcm_lock_cleanup_on_node_dead(dead_node_id); + + /* + * spec-4.6a D12: dead-node PCM residue is part of the reconfig + * convergence chain -- a leftover holder/pending-X record from the dead + * node blocks post-kill DDL with lock-acquire timeouts long after the + * remaster itself finished. Surface the cleanup as a LOG summary plus an + * assertable counter (pcm/dead_cleanup_entries). + */ + if (pending_x_cleared > 0 || pcm_holders_cleaned > 0) { + if (cluster_grd_state != NULL) + pg_atomic_fetch_add_u64(&cluster_grd_state->pcm_dead_cleanup_entries, + pending_x_cleared + pcm_holders_cleaned); + ereport(LOG, (errmsg("cluster GRD dead-node cleanup for node %d: cleared " UINT64_FORMAT + " PCM pending-X waiter(s) and " UINT64_FORMAT + " PCM holder record(s) left by the dead node", + dead_node_id, pending_x_cleared, pcm_holders_cleaned))); + } if (cluster_grd_entry_htab == NULL) { ereport(DEBUG2, (errmsg_internal("cluster_grd_cleanup_on_node_dead(%d): " diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 3334422711..999211a3b4 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -226,6 +226,8 @@ int cluster_tm_convert_mode = CLUSTER_TM_CONVERT_MODE_CONVERT; /* spec-4.6 D4/D1 — failure-driven remaster tunables. */ int cluster_grd_remaster_wait_ms = 200; /* frozen-shard short wait before 53R9I */ int cluster_grd_rebuild_timeout_ms = 5000; /* holder-rebuild barrier deadline */ +int cluster_hw_remaster_retry_backoff_ms = 1000; +int cluster_hw_remaster_retry_max_attempts = 16; /* spec-5.4 D8 — SQ sequence lock tunables. */ int cluster_sequence_default_cache = 100; /* CREATE-time CACHE injection default */ @@ -307,6 +309,7 @@ int cluster_phase4_timeout = 30; * LMON main-loop tick / WaitLatch timeout in milliseconds. */ int cluster_lmon_main_loop_interval = 1000; +int cluster_lmon_slow_iteration_warn_ms = 1000; /* spec-1.12 Sprint B D8: cluster.lck_main_loop_interval (mirror). */ int cluster_lck_main_loop_interval = 1000; @@ -2374,6 +2377,21 @@ cluster_init_guc(void) "request with a fresh deadline."), &cluster_grd_rebuild_timeout_ms, 5000, 100, 600000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, NULL); + DefineCustomIntVariable( + "cluster.hw_remaster_retry_backoff_ms", + gettext_noop("Initial backoff before retrying a BLOCKED HW remaster worker (ms)."), + gettext_noop("Range [100, 60000]. Default 1000. Same-episode HW remaster " + "retries use exponential backoff capped at 60 seconds; the adopted " + "shards stay frozen while retrying."), + &cluster_hw_remaster_retry_backoff_ms, 1000, 100, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, + NULL, NULL); + DefineCustomIntVariable( + "cluster.hw_remaster_retry_max_attempts", + gettext_noop("Maximum same-episode retries for a BLOCKED HW remaster worker."), + gettext_noop("Range [0, 1000]. Default 16. Zero disables same-episode retry. " + "Raising the value with SIGHUP lets an exhausted episode resume retrying " + "without waiting for a new reconfig episode."), + &cluster_hw_remaster_retry_max_attempts, 16, 0, 1000, PGC_SIGHUP, 0, NULL, NULL, NULL); /* spec-2.23 D11 NEW: coordinator REPORT collect deadline. */ DefineCustomIntVariable("cluster.lmd_probe_collect_timeout_ms", @@ -2573,6 +2591,15 @@ cluster_init_guc(void) &cluster_lmon_main_loop_interval, 1000, 100, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, NULL); + DefineCustomIntVariable( + "cluster.lmon_slow_iteration_warn_ms", + gettext_noop("LMON main-loop slow-iteration warning threshold in milliseconds."), + gettext_noop("A value greater than zero logs LMON main-loop iterations above this " + "duration; the lmon_slow_iter_count counter is maintained even when " + "logging is disabled with 0."), + &cluster_lmon_slow_iteration_warn_ms, 1000, 0, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, + NULL); + DefineCustomIntVariable( "cluster.lck_main_loop_interval", gettext_noop("LCK main-loop tick interval in milliseconds."), diff --git a/src/backend/cluster/cluster_hw_remaster.c b/src/backend/cluster/cluster_hw_remaster.c index d1ade2a760..f2231b59b0 100644 --- a/src/backend/cluster/cluster_hw_remaster.c +++ b/src/backend/cluster/cluster_hw_remaster.c @@ -61,6 +61,7 @@ #include "miscadmin.h" #include "postmaster/bgworker.h" #include "storage/fd.h" +#include "storage/ipc.h" #include "utils/wait_event.h" #include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES */ @@ -74,6 +75,84 @@ #include "cluster/cluster_wal_thread.h" /* node id -> thread id */ #include "cluster/storage/cluster_undo_xlog.h" /* xl_hw_reserve, XLOG_HW_RESERVE */ +/* One worker process owns exactly one dead origin / episode. */ +static int hw_worker_dead_node = -1; +static uint64 hw_worker_episode = 0; +static bool hw_worker_armed = false; + +static uint64 +hw_remaster_next_attempt_deadline(uint32 completed_retry_attempts) +{ + uint32 backoff_ms; + TimestampTz deadline; + + backoff_ms = cluster_hw_remaster_compute_backoff_ms(cluster_hw_remaster_retry_backoff_ms, + completed_retry_attempts); + deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), (int)backoff_ms); + return (uint64)deadline; +} + +static void +hw_remaster_record_terminal(int dead_node, ClusterHwRemasterResult res) +{ + /* Amendment v1.2 (R6): the worker stores next_attempt_at BEFORE the + * terminal result, and the LMON relaunch decision reads them in the + * opposite order (result first). Order the two stores so a decider + * that observes the terminal result also observes the matching backoff + * deadline — without the fence a weakly-ordered CPU could let it pair a + * fresh BLOCKED with a stale (zero) deadline and skip one backoff wait + * (bounded self-correcting, but cheap to close outright). r3-P3(b): + * paired with the pg_read_barrier() between the result and deadline + * loads in cluster_hw_remaster_launch_workers (acquire/release pair). */ + if (res == CLUSTER_HW_REMASTER_DONE) { + cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + pg_memory_barrier(); + cluster_hw_remaster_set_result(dead_node, res); + cluster_hw_bump_remaster_done(); + } else if (res == CLUSTER_HW_REMASTER_BLOCKED) { + uint32 attempts = cluster_hw_remaster_attempts(dead_node); + + cluster_hw_remaster_set_next_attempt_at(dead_node, + hw_remaster_next_attempt_deadline(attempts)); + pg_memory_barrier(); + cluster_hw_remaster_set_result(dead_node, res); + cluster_hw_bump_remaster_blocked(); + } else if (res == CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL) { + /* Amendment v1.2 (R7): leave next_attempt_at at 0 (not NO_DEADLINE) + * so the FSM's next tick takes the MARK_STRUCTURAL decision branch + * and emits the episode-once operator WARNING + errhint — covering + * the SIGHUP race where only the WORKER (not the launch-side + * precheck) discovers the structural cause. MARK_STRUCTURAL then + * stamps NO_DEADLINE, making the WARNING once-per-episode. */ + cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + pg_memory_barrier(); + cluster_hw_remaster_set_result(dead_node, res); + cluster_hw_bump_remaster_blocked(); + } else if (res == CLUSTER_HW_REMASTER_NOT_APPLICABLE) { + cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + pg_memory_barrier(); + cluster_hw_remaster_set_result(dead_node, res); + } +} + +static void +hw_remaster_mark_blocked_on_exit(int code pg_attribute_unused(), Datum arg) +{ + int dead_node = DatumGetInt32(arg); + + if (!hw_worker_armed) + return; + if (dead_node != hw_worker_dead_node) + return; + if (cluster_hw_remaster_launched_episode(dead_node) != hw_worker_episode) + return; + if (cluster_hw_remaster_result(dead_node) != CLUSTER_HW_REMASTER_RUNNING) + return; + + cluster_hw_bump_failclosed(); + hw_remaster_record_terminal(dead_node, CLUSTER_HW_REMASTER_BLOCKED); +} + /* ============================================================ * Minimal per-thread WAL reader over a dead origin's WAL stream. @@ -332,10 +411,24 @@ cluster_hw_remaster_rebuild_origin(int dead_node_id, uint64 episode_epoch) return CLUSTER_HW_REMASTER_NOT_APPLICABLE; if (dead_node_id < 0 || dead_node_id == cluster_node_id) return CLUSTER_HW_REMASTER_NOT_APPLICABLE; + if (!cluster_hw_remaster_recoverable()) { + cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: cluster.wal_threads_dir is not configured; " + "adopted shards stay fail-closed"), + errhint("Set cluster.wal_threads_dir to the shared per-thread WAL root and " + "restart, or restart the dead node so a JOIN rebuild can replace " + "the failure-driven remaster."))); + return CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL; + } dead_tid = cluster_wal_thread_id_for(true, dead_node_id); - if (dead_tid < XLP_THREAD_ID_FIRST_REAL || dead_tid > CLUSTER_WAL_THREAD_MAX) + if (dead_tid < XLP_THREAD_ID_FIRST_REAL || dead_tid > CLUSTER_WAL_THREAD_MAX) { + cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: dead node %d maps to invalid WAL thread id %u; " + "adopted shards stay fail-closed", + dead_node_id, (unsigned)dead_tid))); return CLUSTER_HW_REMASTER_BLOCKED; + } entries = (ClusterHwSnapshotEntry *)palloc(sizeof(ClusterHwSnapshotEntry) * CLUSTER_HW_AUTHORITY_MAX); @@ -384,6 +477,9 @@ cluster_hw_remaster_rebuild_origin(int dead_node_id, uint64 episode_epoch) if (cluster_wal_state_read_slot(dead_tid, &slot) != CLUSTER_WAL_SLOT_OK || slot.highest_lsn == 0) { cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: dead node %d WAL state slot is unusable; " + "adopted shards stay fail-closed", + dead_node_id))); return CLUSTER_HW_REMASTER_BLOCKED; } validated_min = (XLogRecPtr)slot.highest_lsn; @@ -391,6 +487,9 @@ cluster_hw_remaster_rebuild_origin(int dead_node_id, uint64 episode_epoch) if (cluster_thread_recovery_validated_end(dead_tid, lower, validated_min, &upper) != CLUSTER_THREADREC_DONE) { cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: dead node %d validated WAL end is unavailable; " + "adopted shards stay fail-closed", + dead_node_id))); return CLUSTER_HW_REMASTER_BLOCKED; } @@ -466,29 +565,26 @@ cluster_hw_remaster_worker_main(Datum main_arg) /* The bgworker framework starts the entry point with signals blocked; unblock * before any I/O so a SIGTERM during a stuck shared-storage read / shutdown is - * delivered (the default die handler FATALs, a clean fail-closed: unmarked). */ + * delivered (the default die handler FATALs into the BLOCKED exit callback). */ BackgroundWorkerUnblockSignals(); /* Capture the live reconfig episode this rebuild is for; rebuild_origin uses * it as the staleness fence before it marks any shard rebuilt. */ episode = cluster_grd_redeclare_episode_epoch(); - res = cluster_hw_remaster_rebuild_origin(dead_node, episode); + hw_worker_dead_node = dead_node; + hw_worker_episode = episode; + hw_worker_armed = true; + before_shmem_exit(hw_remaster_mark_blocked_on_exit, Int32GetDatum(dead_node)); - /* Observability (S5/S7): a DONE rebuilt + adopted + marked the shards; a - * BLOCKED kept them frozen (fail-closed). NOT_APPLICABLE (HW inactive) is - * neither. */ - if (res == CLUSTER_HW_REMASTER_DONE) - cluster_hw_bump_remaster_done(); - else if (res == CLUSTER_HW_REMASTER_BLOCKED) - cluster_hw_bump_remaster_blocked(); + res = cluster_hw_remaster_rebuild_origin(dead_node, episode); + hw_remaster_record_terminal(dead_node, res); + hw_worker_armed = false; ereport(LOG, (errmsg("cluster HW remaster worker: dead node %d -> %s", dead_node, - res == CLUSTER_HW_REMASTER_DONE ? "done" - : res == CLUSTER_HW_REMASTER_BLOCKED ? "blocked (shards kept frozen)" - : "not applicable"))); + cluster_hw_remaster_result_name(res)))); /* Returning is a clean exit(0). On BLOCKED / abnormal exit the adopted shards - * stay unmarked, so the serve gate keeps them fail-closed (8.A) and a later - * episode relaunches the rebuild -- no abnormal-exit handler is needed. */ + * stay unmarked, so the serve gate keeps them fail-closed (8.A); the terminal + * result lets the LMON FSM retry within the same episode when appropriate. */ } /* @@ -522,6 +618,7 @@ cluster_hw_remaster_launch_workers(const uint64 *dead, int nwords, uint64 episod { int node; int max_node; + uint64 now; if (dead == NULL || nwords <= 0) return; @@ -538,31 +635,103 @@ cluster_hw_remaster_launch_workers(const uint64 *dead, int nwords, uint64 episod max_node = nwords * 64; if (max_node > CLUSTER_MAX_NODES) max_node = CLUSTER_MAX_NODES; + now = (uint64)GetCurrentTimestamp(); for (node = 0; node < max_node; node++) { + ClusterHwRemasterResult result; + ClusterHwRemasterRelaunchDecision d; + uint64 launched; + uint64 next_attempt_at; + uint32 attempts; + if ((dead[node / 64] & (UINT64CONST(1) << (node % 64))) == 0) continue; if (node == cluster_node_id) continue; /* self is never its own dead origin */ - /* Idempotent: launch once per episode per dead origin. */ - if (cluster_hw_remaster_launched_episode(node) == episode_epoch) + + launched = cluster_hw_remaster_launched_episode(node); + if (!cluster_hw_remaster_recoverable() && launched != episode_epoch) { + cluster_hw_remaster_set_launched(node, episode_epoch); + cluster_hw_remaster_set_attempts(node, 0); + cluster_hw_remaster_set_next_attempt_at(node, 0); + cluster_hw_remaster_set_result(node, CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL); + launched = episode_epoch; + } + + result = cluster_hw_remaster_result(node); + /* r3-P3(b) — read-side pairing for the hw_remaster_record_terminal + * write fence (deadline stored BEFORE the terminal result, with + * pg_memory_barrier between). Load the result FIRST, fence, then the + * deadline: a decider that observed a terminal result is then + * guaranteed to observe the matching backoff deadline. Without this + * the release fence is one-sided and a weakly-ordered reader could + * still pair a fresh BLOCKED with a stale (zero) deadline. attempts + * needs no fence: it is written only by this FSM (single-writer + * LMON). */ + pg_read_barrier(); + attempts = cluster_hw_remaster_attempts(node); + next_attempt_at = cluster_hw_remaster_next_attempt_at(node); + d = cluster_hw_remaster_relaunch_decide(launched, episode_epoch, result, attempts, + next_attempt_at, now, + cluster_hw_remaster_retry_max_attempts); + + if (d.action == CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL) { + cluster_hw_remaster_set_next_attempt_at(node, d.next_attempt_at); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster HW remaster is structurally blocked for dead node %d; " + "adopted shards stay fail-closed", + node), + errhint("Set cluster.wal_threads_dir to the shared per-thread WAL root " + "and restart, or restart the dead node so a JOIN rebuild can " + "replace the failure-driven remaster."))); + continue; + } + if (d.action == CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED) { + cluster_hw_remaster_set_next_attempt_at(node, d.next_attempt_at); + cluster_hw_bump_remaster_retry_exhausted(); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster HW remaster retries exhausted for dead node %d after %u " + "attempt(s); adopted shards stay fail-closed", + node, attempts), + errhint("Fix the shared HW snapshot/WAL source, then raise " + "cluster.hw_remaster_retry_max_attempts with SIGHUP to resume " + "same-episode retrying."))); continue; - /* - * Claim BEFORE registering so a same-episode re-entry does not double- - * launch; revert on a registration failure so a later tick retries (the - * adopted shards stay fail-closed meanwhile, via the P7 gate). - */ + } + if (d.action != CLUSTER_HW_REMASTER_LAUNCH_INITIAL + && d.action != CLUSTER_HW_REMASTER_LAUNCH_RETRY) + continue; + cluster_hw_remaster_set_launched(node, episode_epoch); + cluster_hw_remaster_set_attempts(node, d.next_attempts); + cluster_hw_remaster_set_next_attempt_at(node, 0); + cluster_hw_remaster_set_result(node, CLUSTER_HW_REMASTER_RUNNING); if (!register_one_worker(node)) { - cluster_hw_remaster_set_launched(node, 0); + cluster_hw_remaster_set_launched( + node, d.action == CLUSTER_HW_REMASTER_LAUNCH_INITIAL ? 0 : launched); + cluster_hw_remaster_set_attempts(node, attempts); + cluster_hw_remaster_set_next_attempt_at(node, next_attempt_at); + cluster_hw_remaster_set_result(node, result); ereport(WARNING, (errmsg("could not register HW remaster worker for dead node %d", node), errhint("Background worker slots are exhausted (max_worker_processes); the " "adopted shards stay fail-closed until the rebuild can run."))); + continue; + } + if (d.action == CLUSTER_HW_REMASTER_LAUNCH_RETRY) { + cluster_hw_bump_remaster_retry(); + ereport(LOG, + (errmsg("cluster HW remaster: retrying dead node %d in episode " UINT64_FORMAT + " (attempt %u/%d)", + node, episode_epoch, d.next_attempts, + cluster_hw_remaster_retry_max_attempts))); } } } + bool cluster_hw_remaster_gate_unfreeze(void) { diff --git a/src/backend/cluster/cluster_hw_shmem.c b/src/backend/cluster/cluster_hw_shmem.c index 14bf3f7586..860241e8e9 100644 --- a/src/backend/cluster/cluster_hw_shmem.c +++ b/src/backend/cluster/cluster_hw_shmem.c @@ -100,6 +100,8 @@ typedef struct ClusterHwShared { pg_atomic_uint64 not_ready_count; /* 53RA6 serve-gate (shard adopted, unrebuilt) */ pg_atomic_uint64 remaster_done_count; /* online-remaster HW rebuild DONE (S5/S7) */ pg_atomic_uint64 remaster_blocked_count; /* online-remaster HW rebuild fail-closed (S5/S7) */ + pg_atomic_uint64 remaster_retry_count; /* same-episode retry launches */ + pg_atomic_uint64 remaster_retry_exhausted_count; /* same-episode retry cap reached */ pg_atomic_uint32 hw_rebuilt_generation[PGRAC_GRD_SHARD_COUNT]; /* §3.1b R4/R9 gate */ /* * remaster_launched_episode[node] -- the reconfig episode for which the GRD @@ -111,6 +113,9 @@ typedef struct ClusterHwShared { * race on this field. */ pg_atomic_uint64 remaster_launched_episode[CLUSTER_MAX_NODES]; + pg_atomic_uint32 remaster_result[CLUSTER_MAX_NODES]; + pg_atomic_uint32 remaster_attempts[CLUSTER_MAX_NODES]; + pg_atomic_uint64 remaster_next_attempt_at[CLUSTER_MAX_NODES]; } ClusterHwShared; /* @@ -196,12 +201,18 @@ cluster_hw_shmem_init(void) pg_atomic_init_u64(&hw_state->not_ready_count, 0); pg_atomic_init_u64(&hw_state->remaster_done_count, 0); pg_atomic_init_u64(&hw_state->remaster_blocked_count, 0); + pg_atomic_init_u64(&hw_state->remaster_retry_count, 0); + pg_atomic_init_u64(&hw_state->remaster_retry_exhausted_count, 0); /* No shard rebuilt yet; 0 matches a never-remastered shard's GRD * master_generation (0), so boot / steady state serves. */ for (s = 0; s < PGRAC_GRD_SHARD_COUNT; s++) pg_atomic_init_u32(&hw_state->hw_rebuilt_generation[s], 0); - for (s = 0; s < CLUSTER_MAX_NODES; s++) + for (s = 0; s < CLUSTER_MAX_NODES; s++) { pg_atomic_init_u64(&hw_state->remaster_launched_episode[s], 0); + pg_atomic_init_u32(&hw_state->remaster_result[s], (uint32)CLUSTER_HW_REMASTER_NONE); + pg_atomic_init_u32(&hw_state->remaster_attempts[s], 0); + pg_atomic_init_u64(&hw_state->remaster_next_attempt_at[s], 0); + } } memset(&hctl, 0, sizeof(hctl)); @@ -408,6 +419,81 @@ cluster_hw_remaster_set_launched(int node_id, uint64 episode) pg_atomic_write_u64(&hw_state->remaster_launched_episode[node_id], episode); } +ClusterHwRemasterResult +cluster_hw_remaster_result(int node_id) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return CLUSTER_HW_REMASTER_NONE; + return (ClusterHwRemasterResult)pg_atomic_read_u32(&hw_state->remaster_result[node_id]); +} + +void +cluster_hw_remaster_set_result(int node_id, ClusterHwRemasterResult result) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return; + pg_atomic_write_u32(&hw_state->remaster_result[node_id], (uint32)result); +} + +uint32 +cluster_hw_remaster_attempts(int node_id) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u32(&hw_state->remaster_attempts[node_id]); +} + +void +cluster_hw_remaster_set_attempts(int node_id, uint32 attempts) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return; + pg_atomic_write_u32(&hw_state->remaster_attempts[node_id], attempts); +} + +uint64 +cluster_hw_remaster_next_attempt_at(int node_id) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u64(&hw_state->remaster_next_attempt_at[node_id]); +} + +void +cluster_hw_remaster_set_next_attempt_at(int node_id, uint64 ts) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return; + pg_atomic_write_u64(&hw_state->remaster_next_attempt_at[node_id], ts); +} + +const char * +cluster_hw_remaster_result_name(ClusterHwRemasterResult result) +{ + switch (result) { + case CLUSTER_HW_REMASTER_NONE: + return "none"; + case CLUSTER_HW_REMASTER_RUNNING: + return "running"; + case CLUSTER_HW_REMASTER_DONE: + return "done"; + case CLUSTER_HW_REMASTER_BLOCKED: + return "blocked"; + case CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL: + return "blocked_structural"; + case CLUSTER_HW_REMASTER_NOT_APPLICABLE: + return "not_applicable"; + } + return "unknown"; +} + +bool +cluster_hw_remaster_recoverable(void) +{ + return !cluster_hw_authority_active() + || (cluster_wal_threads_dir != NULL && cluster_wal_threads_dir[0] != '\0'); +} + /* ============================================================ * Checkpoint write + recovery load (§3.1b R1/R2; PG-core hooked). @@ -747,6 +833,16 @@ cluster_hw_bump_remaster_blocked(void) { HW_BUMP(remaster_blocked_count); } +void +cluster_hw_bump_remaster_retry(void) +{ + HW_BUMP(remaster_retry_count); +} +void +cluster_hw_bump_remaster_retry_exhausted(void) +{ + HW_BUMP(remaster_retry_exhausted_count); +} uint64 cluster_hw_alloc_count(void) @@ -788,3 +884,13 @@ cluster_hw_remaster_blocked_count(void) { return HW_READ(remaster_blocked_count); } +uint64 +cluster_hw_remaster_retry_count(void) +{ + return HW_READ(remaster_retry_count); +} +uint64 +cluster_hw_remaster_retry_exhausted_count(void) +{ + return HW_READ(remaster_retry_exhausted_count); +} diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index 89127fe949..332a4e97ff 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -199,6 +199,7 @@ static ClusterInjectPoint cluster_injection_points[] = { /* spec-2.6 Sprint A Step 4 D14 — 5 qvotec / quorum-lite injects. */ { .name = "cluster-qvotec-poll-pre" }, { .name = "cluster-qvotec-poll-post" }, + { .name = "cluster-qvotec-marker-service-hold" }, { .name = "cluster-voting-disk-write-fail" }, { .name = "cluster-quorum-loss-broadcast" }, { .name = "cluster-collision-detect" }, diff --git a/src/backend/cluster/cluster_lmon.c b/src/backend/cluster/cluster_lmon.c index 73e816a0c7..1af59978cc 100644 --- a/src/backend/cluster/cluster_lmon.c +++ b/src/backend/cluster/cluster_lmon.c @@ -681,6 +681,61 @@ cluster_lmon_main_loop_iters(void) return result; } +uint64 +cluster_lmon_last_iter_us(void) +{ + uint64 result; + + if (cluster_lmon_state == NULL) + return 0; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_SHARED); + result = cluster_lmon_state->last_iter_us; + LWLockRelease(&cluster_lmon_state->lwlock); + return result; +} + +uint64 +cluster_lmon_max_iter_us(void) +{ + uint64 result; + + if (cluster_lmon_state == NULL) + return 0; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_SHARED); + result = cluster_lmon_state->max_iter_us; + LWLockRelease(&cluster_lmon_state->lwlock); + return result; +} + +uint64 +cluster_lmon_slow_iter_count(void) +{ + uint64 result; + + if (cluster_lmon_state == NULL) + return 0; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_SHARED); + result = cluster_lmon_state->slow_iter_count; + LWLockRelease(&cluster_lmon_state->lwlock); + return result; +} + +void +cluster_lmon_marker_complete_wakeup(void) +{ + struct Latch *latch; + + if (cluster_lmon_state == NULL) + return; + + latch = cluster_lmon_state->lmon_latch; + if (latch != NULL) + SetLatch(latch); +} + /* ============================================================ * LMON main entry (AuxiliaryProcessMain dispatch target). @@ -720,12 +775,30 @@ lmon_publish_status(ClusterLmonStatus status) cluster_lmon_state->ready_at = 0; cluster_lmon_state->last_liveness_tick_at = 0; cluster_lmon_state->main_loop_iters = 0; + cluster_lmon_state->last_iter_us = 0; + cluster_lmon_state->max_iter_us = 0; + cluster_lmon_state->slow_iter_count = 0; + cluster_lmon_state->lmon_latch = NULL; } else if (status == CLUSTER_LMON_READY) { cluster_lmon_state->ready_at = now; + cluster_lmon_state->lmon_latch = MyLatch; + } else if (status == CLUSTER_LMON_EXITED) { + cluster_lmon_state->lmon_latch = NULL; } LWLockRelease(&cluster_lmon_state->lwlock); } +static void +lmon_clear_latch(int code, Datum arg) +{ + if (cluster_lmon_state == NULL) + return; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_EXCLUSIVE); + cluster_lmon_state->lmon_latch = NULL; + LWLockRelease(&cluster_lmon_state->lwlock); +} + static bool lmon_shutdown_requested(void) @@ -760,6 +833,44 @@ lmon_advance_liveness_tick(void) LWLockRelease(&cluster_lmon_state->lwlock); } +static void +lmon_record_iteration(TimestampTz iter_started_at) +{ + TimestampTz now = GetCurrentTimestamp(); + uint64 elapsed_us; + bool slow; + bool should_log = false; + static TimestampTz last_slow_log_at = 0; + + if (now <= iter_started_at) + elapsed_us = 0; + else + elapsed_us = (uint64)(now - iter_started_at); + + slow = (cluster_lmon_slow_iteration_warn_ms >= 0) + && elapsed_us >= (uint64)cluster_lmon_slow_iteration_warn_ms * 1000ULL; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_EXCLUSIVE); + cluster_lmon_state->last_iter_us = elapsed_us; + if (elapsed_us > cluster_lmon_state->max_iter_us) + cluster_lmon_state->max_iter_us = elapsed_us; + if (slow) + cluster_lmon_state->slow_iter_count++; + LWLockRelease(&cluster_lmon_state->lwlock); + + if (slow && cluster_lmon_slow_iteration_warn_ms > 0 + && (last_slow_log_at == 0 + || now - last_slow_log_at >= INT64CONST(60) * INT64CONST(1000000))) { + last_slow_log_at = now; + should_log = true; + } + + if (should_log) + ereport(LOG, (errmsg("cluster lmon: main loop iteration took %llu ms (threshold %d ms)", + (unsigned long long)(elapsed_us / 1000ULL), + cluster_lmon_slow_iteration_warn_ms))); +} + void LmonMain(void) @@ -811,6 +922,7 @@ LmonMain(void) /* Publish SPAWNING (records pid + spawned_at). */ lmon_publish_status(CLUSTER_LMON_SPAWNING); + on_shmem_exit(lmon_clear_latch, (Datum)0); /* Sprint B inject: ready-publish (test slow startup / phase 1 wait timeout). */ CLUSTER_INJECTION_POINT("cluster-lmon-ready-publish"); @@ -938,6 +1050,7 @@ LmonMain(void) int n_events; long wait_ms; TimestampTz now; + TimestampTz iter_started_at; int32 i; CHECK_FOR_INTERRUPTS(); @@ -950,6 +1063,7 @@ LmonMain(void) if (ShutdownRequestPending || lmon_shutdown_requested()) break; + iter_started_at = GetCurrentTimestamp(); lmon_advance_liveness_tick(); /* @@ -1414,6 +1528,7 @@ LmonMain(void) wait_ms = 1; } + lmon_record_iteration(iter_started_at); n_events = WaitEventSetWait(wes, wait_ms, ev, lengthof(ev), WAIT_EVENT_CLUSTER_IC_HEARTBEAT_WAIT); @@ -1588,6 +1703,7 @@ LmonMain(void) /* Stub / mock / disabled mode -- preserve spec-1.11 simple loop. */ for (;;) { int rc; + TimestampTz iter_started_at; CHECK_FOR_INTERRUPTS(); @@ -1599,6 +1715,7 @@ LmonMain(void) if (ShutdownRequestPending || lmon_shutdown_requested()) break; + iter_started_at = GetCurrentTimestamp(); lmon_advance_liveness_tick(); /* @@ -1677,6 +1794,7 @@ LmonMain(void) CLUSTER_INJECTION_POINT("cluster-lmon-main-loop-iter"); + lmon_record_iteration(iter_started_at); rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, cluster_lmon_main_loop_interval, WAIT_EVENT_CLUSTER_BGPROC_LMON_MAIN_LOOP); diff --git a/src/backend/cluster/cluster_lms.c b/src/backend/cluster/cluster_lms.c index c3b9a39c24..8bec6ddd71 100644 --- a/src/backend/cluster/cluster_lms.c +++ b/src/backend/cluster/cluster_lms.c @@ -59,6 +59,7 @@ #include "cluster/cluster_cr_server.h" /* spec-6.12b CR work slots */ #include "cluster/cluster_conf.h" +#include "cluster/cluster_cssd.h" #include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ #include "cluster/cluster_ges.h" #include "cluster/cluster_ges_dedup.h" @@ -1044,6 +1045,8 @@ cluster_lms_native_probe_dispatch(uint32 slot_idx) conf_node = cluster_conf_lookup_node(candidate); if (conf_node == NULL) continue; + if (cluster_cssd_get_peer_state(candidate) == CLUSTER_CSSD_PEER_DEAD) + continue; if (!native_probe_node_bit(candidate, &peer_bit)) { pg_atomic_fetch_add_u64(&cluster_lms_state->native_probe_timeout_count, 1); slot->final_status = CLUSTER_NATIVE_PROBE_FINAL_TIMEOUT; diff --git a/src/backend/cluster/cluster_node_remove.c b/src/backend/cluster/cluster_node_remove.c index 8825ac6828..39e31c4b72 100644 --- a/src/backend/cluster/cluster_node_remove.c +++ b/src/backend/cluster/cluster_node_remove.c @@ -57,6 +57,7 @@ #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_ic_router.h" #include "cluster/cluster_inject.h" +#include "cluster/cluster_lmon.h" #include "cluster/cluster_membership.h" #include "cluster/cluster_node_remove.h" #include "cluster/cluster_pcm_lock.h" @@ -162,6 +163,14 @@ cluster_node_remove_get_state(ClusterNodeRemoveState *out) static uint64 nr_qvotec_last_processed_marker_seq = 0; static uint64 nr_qvotec_inflight_marker_seq = 0; +static ClusterMarkerAsync nr_marker_async; +static ClusterRemovalMarker nr_marker_stage; +static int nr_marker_phase = 0; +static int32 nr_marker_node_id = -1; +static uint64 nr_marker_epoch = 0; +static uint64 nr_marker_removed_incarnation = 0; +static uint64 nr_marker_removal_event_id = 0; +static bool nr_marker_submitted = false; ClusterRemovalMarkerSubmitResult cluster_node_remove_submit_marker(const ClusterRemovalMarker *m) @@ -195,6 +204,38 @@ cluster_node_remove_submit_marker(const ClusterRemovalMarker *m) } } +bool +cluster_node_remove_submit_marker_async(ClusterMarkerAsync *a, const ClusterRemovalMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + int wait_ms; + + if (nr_state == NULL || m == NULL || a == NULL) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&nr_state->marker_request_seq, + &nr_state->marker_completion_seq)) + return false; + + nr_state->pending_marker = *m; + wait_ms = cluster_quorum_poll_interval_ms * 3 + 2000; + return cluster_marker_async_submit(a, &nr_state->marker_request_seq, + &nr_state->marker_completion_seq, nr_state->qvotec_latch, + now, (uint64)wait_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_node_remove_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) +{ + if (nr_state == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &nr_state->marker_completion_seq, &nr_state->marker_result, + now, out_result, out_elapsed_us); +} + bool cluster_node_remove_qvotec_poll_pending(ClusterRemovalMarker *out) { @@ -223,6 +264,7 @@ cluster_node_remove_qvotec_complete(bool acked) pg_write_barrier(); pg_atomic_write_u64(&nr_state->marker_completion_seq, nr_qvotec_inflight_marker_seq); nr_qvotec_last_processed_marker_seq = nr_qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } static void @@ -241,24 +283,124 @@ cluster_node_remove_publish_qvotec_latch(struct Latch *latch) on_shmem_exit(nr_clear_qvotec_latch, (Datum)0); } -/* build + submit one durable removal marker; returns true iff majority-durable. */ +static ClusterMarkerAsyncKind +nr_marker_phase_kind(int phase) +{ + switch (phase) { + case CLUSTER_REMOVAL_MARKER_REMOVING: + return CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVING; + case CLUSTER_REMOVAL_MARKER_SHRUNK: + return CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK; + case CLUSTER_REMOVAL_MARKER_REMOVED: + return CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED; + default: + return CLUSTER_MARKER_KIND_UNKNOWN; + } +} + +static void +nr_release_marker_stage(void) +{ + cluster_marker_async_release_stage(&nr_marker_async); + memset(&nr_marker_stage, 0, sizeof(nr_marker_stage)); + nr_marker_phase = 0; + nr_marker_node_id = -1; + nr_marker_epoch = 0; + nr_marker_removed_incarnation = 0; + nr_marker_removal_event_id = 0; + nr_marker_submitted = false; +} + +static bool +nr_marker_stage_matches(int phase, int32 node_id, uint64 remove_epoch, uint64 removed_incarnation, + uint64 removal_event_id) +{ + return nr_marker_async.has_staged_event && nr_marker_phase == phase + && nr_marker_node_id == node_id && nr_marker_epoch == remove_epoch + && nr_marker_removed_incarnation == removed_incarnation + && nr_marker_removal_event_id == removal_event_id; +} + +/* build + poll one durable removal marker; returns true iff majority-durable. */ static bool nr_write_marker(int phase, int32 node_id, uint64 remove_epoch, uint64 removed_incarnation, uint64 removal_event_id) { ClusterRemovalMarker m; + TimestampTz now; + uint32 result = CLUSTER_REMOVAL_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + ClusterMarkerAsyncKind kind; + + now = GetCurrentTimestamp(); + kind = nr_marker_phase_kind(phase); + + if (nr_marker_async.has_staged_event + && !nr_marker_stage_matches(phase, node_id, remove_epoch, removed_incarnation, + removal_event_id)) { + if (!nr_marker_submitted) { + if (cluster_node_remove_submit_marker_async(&nr_marker_async, &nr_marker_stage, + nr_marker_phase_kind(nr_marker_phase), + nr_marker_node_id, now)) + nr_marker_submitted = true; + return false; + } + pr = cluster_node_remove_poll_marker_async(&nr_marker_async, now, &result, &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(nr_marker_async.kind, nr_marker_node_id, + elapsed_us); + nr_release_marker_stage(); + } else if (pr == CLUSTER_MARKER_POLL_ACKED) { + cluster_reconfig_note_marker_slow_ack(nr_marker_async.kind, nr_marker_node_id, + elapsed_us); + nr_release_marker_stage(); + } + return false; + } + + if (!nr_marker_async.has_staged_event) { + memset(&m, 0, sizeof(m)); + m.magic = CLUSTER_REMOVAL_MARKER_MAGIC; + m.version = CLUSTER_REMOVAL_MARKER_VERSION; + m.phase = (uint16)phase; + m.removed_node_id = node_id; + m.remove_epoch = remove_epoch; + m.removed_incarnation = removed_incarnation; + m.removal_event_id = removal_event_id; + cluster_removal_marker_compute_crc(&m); + + nr_marker_stage = m; + nr_marker_phase = phase; + nr_marker_node_id = node_id; + nr_marker_epoch = remove_epoch; + nr_marker_removed_incarnation = removed_incarnation; + nr_marker_removal_event_id = removal_event_id; + nr_marker_async.has_staged_event = true; + nr_marker_async.staged_expect_epoch = remove_epoch; + nr_marker_submitted = false; + } - memset(&m, 0, sizeof(m)); - m.magic = CLUSTER_REMOVAL_MARKER_MAGIC; - m.version = CLUSTER_REMOVAL_MARKER_VERSION; - m.phase = (uint16)phase; - m.removed_node_id = node_id; - m.remove_epoch = remove_epoch; - m.removed_incarnation = removed_incarnation; - m.removal_event_id = removal_event_id; - cluster_removal_marker_compute_crc(&m); - - return cluster_node_remove_submit_marker(&m) == CLUSTER_REMOVAL_MARKER_SUBMIT_ACK; + if (!nr_marker_submitted) { + if (!cluster_node_remove_submit_marker_async(&nr_marker_async, &nr_marker_stage, kind, + node_id, now)) + return false; + nr_marker_submitted = true; + return false; + } + + pr = cluster_node_remove_poll_marker_async(&nr_marker_async, now, &result, &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return false; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(kind, node_id, elapsed_us); + nr_release_marker_stage(); + return false; + } + + cluster_reconfig_note_marker_slow_ack(kind, node_id, elapsed_us); + nr_release_marker_stage(); + return result == CLUSTER_REMOVAL_MARKER_SUBMIT_ACK; } @@ -543,31 +685,43 @@ cluster_node_remove_drive(void) (void)baseline_dead_gen; /* contest is now signalled by out_contest, not derived */ - /* §2.5: durable REMOVING marker (pre-commit; not a trust source). */ - (void)nr_write_marker(CLUSTER_REMOVAL_MARKER_REMOVING, node_id, baseline_epoch, - last_incarnation, removal_event_id); - /* - * spec-6.15 D5c (appendix B.3): durably retire the removed node's - * xid stripe slot BEFORE the removal point of no return below. - * Ordering is the identity-reuse defence: once removal commits, a - * later fresh join of the same node_id (new incarnation = a new - * durable owner) must land on a retired slot and refuse (53RB1), - * never resume the old owner's congruence class. Not durable yet - * -> stay FENCE_ARMING and retry next tick (fail-closed; same - * shape as a transient fence/quorum failure). A never-activated - * cluster returns true (nothing to retire). + * spec-2.29a review r1 P2: while the staged fence marker from a + * previous FENCE_ARMING tick is still in flight, skip the pre-work + * below — the epoch was already self-bumped at stage-entry, so + * re-running it would key a SECOND REMOVING marker to the post-bump + * epoch and re-submit the stripe retire during the very contention + * window the async stage relieves. Fall through straight to + * apply(), which polls the stage. */ - if (!cluster_xid_stripe_submit_retire(node_id, last_incarnation)) { - ereport(LOG, (errmsg("cluster node removal: stripe slot retire for node %d not durable " - "yet — retrying before the removal commit", - node_id))); - return; + if (!cluster_reconfig_node_removed_fence_stage_pending()) { + /* §2.5: durable REMOVING marker (pre-commit; not a trust source). */ + (void)nr_write_marker(CLUSTER_REMOVAL_MARKER_REMOVING, node_id, baseline_epoch, + last_incarnation, removal_event_id); + + /* + * spec-6.15 D5c (appendix B.3): durably retire the removed node's + * xid stripe slot BEFORE the removal point of no return below. + * Ordering is the identity-reuse defence: once removal commits, a + * later fresh join of the same node_id (new incarnation = a new + * durable owner) must land on a retired slot and refuse (53RB1), + * never resume the old owner's congruence class. Not durable yet + * -> stay FENCE_ARMING and retry next tick (fail-closed; same + * shape as a transient fence/quorum failure). A never-activated + * cluster returns true (nothing to retire). + */ + if (!cluster_xid_stripe_submit_retire(node_id, last_incarnation)) { + ereport(LOG, + (errmsg("cluster node removal: stripe slot retire for node %d not durable " + "yet — retrying before the removal commit", + node_id))); + return; + } + /* D5d: carry the retire to WAL consumers (standby stops gap- + * filling the dead class). The voting-disk retire above is the + * correctness anchor; emission failure is tolerated (logged). */ + cluster_xid_stripe_emit_retire_wal(node_id); } - /* D5d: carry the retire to WAL consumers (standby stops gap- - * filling the dead class). The voting-disk retire above is the - * correctness anchor; emission failure is tolerated (logged). */ - cluster_xid_stripe_emit_retire_wal(node_id); /* * The commit point: guarded epoch bump + fence-arm (majority-durable, at diff --git a/src/backend/cluster/cluster_pcm_lock.c b/src/backend/cluster/cluster_pcm_lock.c index 545ac33480..88ddaad4cf 100644 --- a/src/backend/cluster/cluster_pcm_lock.c +++ b/src/backend/cluster/cluster_pcm_lock.c @@ -724,6 +724,96 @@ cluster_pcm_lock_clear_pending_x_for_node(int32 dead_node) return cleared; } +uint64 +cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node) +{ + HASH_SEQ_STATUS scan; + struct GrdEntry *entry; + uint64 cleaned = 0; + uint32 dead_bit; + + if (cluster_pcm_htab == NULL || dead_node < 0 || dead_node >= 32) + return 0; + + dead_bit = (uint32)1u << (uint32)dead_node; + + LWLockAcquire(&ClusterPcm->htab_lock.lock, LW_SHARED); + hash_seq_init(&scan, cluster_pcm_htab); + while ((entry = (struct GrdEntry *)hash_seq_search(&scan)) != NULL) { + bool changed = false; + bool broadcast_needed = false; + bool master_holder_was_dead; + PcmState before_state; + PcmState after_state; + uint32 s_bitmap; + uint32 pi_bitmap; + + /* Cheap unlocked filter; rechecked under entry_lock below. */ + if (entry->x_holder_node != dead_node + && (pg_atomic_read_u32(&entry->s_holders_bitmap) & dead_bit) == 0 + && (pg_atomic_read_u32(&entry->pi_holders_bitmap) & dead_bit) == 0 + && (!pcm_master_holder_is_valid(entry) + || (int32)entry->master_holder.node_id != dead_node)) + continue; + + LWLockAcquire(&entry->entry_lock.lock, LW_EXCLUSIVE); + before_state = (PcmState)pg_atomic_read_u32(&entry->master_state); + master_holder_was_dead + = pcm_master_holder_is_valid(entry) && (int32)entry->master_holder.node_id == dead_node; + + if (entry->x_holder_node == dead_node) { + entry->x_holder_node = -1; + changed = true; + } + s_bitmap = pg_atomic_read_u32(&entry->s_holders_bitmap); + if ((s_bitmap & dead_bit) != 0) { + s_bitmap &= ~dead_bit; + pg_atomic_write_u32(&entry->s_holders_bitmap, s_bitmap); + changed = true; + } + pi_bitmap = pg_atomic_read_u32(&entry->pi_holders_bitmap); + if ((pi_bitmap & dead_bit) != 0) { + pg_atomic_write_u32(&entry->pi_holders_bitmap, pi_bitmap & ~dead_bit); + changed = true; + } + + if (entry->x_holder_node >= 0) { + pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_X); + if (master_holder_was_dead) + pcm_master_holder_set_node(entry, entry->x_holder_node); + } else if (s_bitmap != 0) { + int32 next_holder = pcm_lowest_set_bit_node(s_bitmap); + + pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_S); + if (master_holder_was_dead) { + if (next_holder >= 0) + pcm_master_holder_set_node(entry, next_holder); + else + pcm_master_holder_clear(entry); + } + } else { + pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_N); + if (pcm_master_holder_is_valid(entry)) + pcm_master_holder_clear(entry); + } + if (master_holder_was_dead) + changed = true; + after_state = (PcmState)pg_atomic_read_u32(&entry->master_state); + if (changed && after_state != before_state) + broadcast_needed = true; + + LWLockRelease(&entry->entry_lock.lock); + + if (broadcast_needed) + ConditionVariableBroadcast(&entry->wait_cv); + if (changed) + cleaned++; + } + LWLockRelease(&ClusterPcm->htab_lock.lock); + + return cleaned; +} + /* ======================================================================== * PGRAC MODIFICATIONS by SqlRush — spec-5.13 D5 (clean-leave PCM release). @@ -2131,7 +2221,10 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) errmsg("block-level cache protocol state is being rebuilt " "after reconfiguration"), errhint("The block resource is recovering (survivor re-declare / " - "master rebuild after node failure); retry the transaction."))); + "master rebuild after node failure); retry the transaction. " + "If the failed node stays down, restart it to run its " + "instance recovery, or enable online thread recovery in a " + "supported scope (cluster.online_thread_recovery)."))); CHECK_FOR_INTERRUPTS(); pgstat_report_wait_start(WAIT_EVENT_GCS_BLOCK_RECOVERING); @@ -2199,6 +2292,61 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) if (master_state == PCM_LOCK_MODE_X && holder >= 0 && holder != cluster_node_id) return cluster_gcs_local_master_x_transfer_and_wait(buf, holder, clean_eligible); + + /* + * PGRAC: spec-4.6a BUG-C2 follow-through for shared_catalog DDL after + * fail-stop. Local-master state=S with other live S holders but no local + * S bit used to fall through to the tag-only X acquire, which fail-closed + * as "no local S residency". This entry point is buffer-aware: the + * caller has already read or initialized the BufferDesc before asking for + * X, so first register a local S residency, then reuse the existing + * local S->X invalidate/upgrade path. If the invalidate cannot be proven, + * drop the temporary S claim and rethrow the same fail-closed error. + */ + if (master_state == PCM_LOCK_MODE_S && cluster_node_id >= 0 && cluster_node_id < 32) { + uint32 self_bit = (uint32)1u << (uint32)cluster_node_id; + + if ((cluster_pcm_lock_query_s_holders_bitmap(tag) & self_bit) == 0) { + struct GrdEntry *entry; + bool upgraded = false; + + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + /* Amendment v1.2 (R5): the upgrade waits on remote ACKs with + * CHECK_FOR_INTERRUPTS in the loop, so a cancel can THROW out + * of it — release the temporary S claim on that path too, not + * only on the false return. */ + PG_TRY(); + { + upgraded = cluster_gcs_block_local_x_upgrade(tag); + } + PG_CATCH(); + { + cluster_pcm_lock_release(tag); + PG_RE_THROW(); + } + PG_END_TRY(); + if (!upgraded) { + cluster_pcm_lock_release(tag); + ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), + errmsg("cluster_pcm: S->X upgrade invalidate did not complete"), + errhint("Remote S holders did not all acknowledge in time; " + "retry the statement."))); + } + + entry = pcm_find_entry(tag); + if (entry != NULL) { + pcm_entry_lock_exclusive(entry); + /* Amendment v1.2 (R9): the completed S->X upgrade replaces + * ALL of this node's S declarations for the tag (the X + * grant subsumes them), so the local S refcount is + * intentionally hard-reset rather than decremented — a + * later release of the X does not owe any S releases. */ + entry->s_holder_refcount_local = 0; + LWLockRelease(&entry->entry_lock.lock); + } + return true; + } + } } cluster_pcm_lock_acquire(tag, mode); diff --git a/src/backend/cluster/cluster_qvotec.c b/src/backend/cluster/cluster_qvotec.c index 8eb25379a7..916a629399 100644 --- a/src/backend/cluster/cluster_qvotec.c +++ b/src/backend/cluster/cluster_qvotec.c @@ -94,8 +94,9 @@ #include "cluster/cluster_elog.h" /* CLUSTER_LOG (best-effort logging) */ #include "cluster/cluster_epoch.h" /* spec-4.12b D2/D5: current-epoch upper-bound Assert */ #include "cluster/cluster_guc.h" /* cluster_enabled */ -#include "cluster/cluster_pgstat.h" /* cluster.qvotec.* counters */ -#include "cluster/cluster_reconfig.h" /* spec-4.12b D2: applied-membership snapshot */ +#include "cluster/cluster_inject.h" +#include "cluster/cluster_pgstat.h" /* cluster.qvotec.* counters */ +#include "cluster/cluster_reconfig.h" /* spec-4.12b D2: applied-membership snapshot */ #include "cluster/cluster_xid_stripe_boot.h" /* spec-6.15 D5b: region-5 scan + seed */ #include "cluster/cluster_shmem.h" /* cluster_shmem_register_region */ #include "cluster/cluster_write_fence.h" /* spec-4.12 D2: fence marker scan + token refresh */ @@ -883,6 +884,10 @@ qvotec_poll_once(void) * carried forward every poll like the fence marker (R12), and acked majority- * durable from the self-slot write tally. */ have_removal_submit = cluster_node_remove_qvotec_poll_pending(&removal_submit_marker); + if ((have_submit || have_leave_submit || have_join_submit || have_removal_submit) + && cluster_cr_injection_armed("cluster-qvotec-marker-service-hold", NULL)) + return; + memset(&apply_lease_request, 0, sizeof(apply_lease_request)); memset(&apply_lease_winner, 0, sizeof(apply_lease_winner)); apply_lease_winner.owner_node_id = -1; diff --git a/src/backend/cluster/cluster_reconfig.c b/src/backend/cluster/cluster_reconfig.c index 1f79a6c9b9..ccaf5991c1 100644 --- a/src/backend/cluster/cluster_reconfig.c +++ b/src/backend/cluster/cluster_reconfig.c @@ -77,6 +77,7 @@ #include "cluster/cluster_clean_leave.h" /* v1.0.4 — cluster_clean_leave_in_progress (serialize) */ #include "cluster/cluster_guc.h" /* cluster_enabled, cluster_online_join */ #include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT */ +#include "cluster/cluster_lmon.h" /* cluster_lmon_marker_complete_wakeup */ #include "cluster/cluster_voting_disk_io.h" /* spec-5.15 D4 — region-3 join-marker slot I/O */ #include "cluster/cluster_write_fence.h" /* spec-4.12 D4 — durable fence marker submit */ #include "cluster/cluster_qvotec.h" /* cluster_qvotec_in_quorum */ @@ -114,6 +115,38 @@ StaticAssertDecl(CLUSTER_RECONFIG_TOUCH_KIND_COUNT == CLUSTER_TOUCH_KIND_COUNT, */ static ClusterReconfigState *ReconfigShmem = NULL; +typedef struct ClusterReconfigFenceStage { + ClusterMarkerAsync async; + ReconfigEvent event; + ClusterFenceMarker marker; + int32 node_id; + uint64 last_incarnation; + uint64 removal_event_id; + bool submitted; +} ClusterReconfigFenceStage; + +typedef struct ClusterReconfigJoinPrepareStage { + ClusterMarkerAsync async; + ReconfigEvent event; + uint8 join_bitmap[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint64 joiner_incarnations[CLUSTER_MAX_NODES]; + int next_node; + bool submitted; +} ClusterReconfigJoinPrepareStage; + +typedef struct ClusterReconfigJoinCommitStage { + ClusterMarkerAsync async; + ClusterJoinCommitMarker marker; + int32 node_id; + uint64 admitted_incarnation; + bool submitted; +} ClusterReconfigJoinCommitStage; + +static ClusterReconfigFenceStage failstop_fence_stage; +static ClusterReconfigFenceStage node_removed_fence_stage; +static ClusterReconfigJoinPrepareStage join_prepare_stage; +static ClusterReconfigJoinCommitStage join_commit_stage; + /* ============================================================ * Shmem region lifecycle. @@ -144,6 +177,7 @@ cluster_reconfig_shmem_init(void) pg_atomic_init_u64(&ReconfigShmem->apply_counter, 0); pg_atomic_init_u64(&ReconfigShmem->dedup_skip_counter, 0); pg_atomic_init_u64(&ReconfigShmem->procsig_broadcast_count, 0); + pg_atomic_init_u32(&ReconfigShmem->prebump_sync_active, 0); /* spec-2.29a r2 t/274 */ /* spec-5.14 D6 — touched_peers observability counters. */ pg_atomic_init_u64(&ReconfigShmem->touched_abort_count, 0); pg_atomic_init_u64(&ReconfigShmem->touched_stamp_count, 0); @@ -211,6 +245,8 @@ cluster_reconfig_shmem_init(void) pg_atomic_init_u64(&ReconfigShmem->join_reject_count, 0); pg_atomic_init_u64(&ReconfigShmem->join_timeout_count, 0); pg_atomic_init_u64(&ReconfigShmem->clean_departed_cleared_count, 0); + pg_atomic_init_u64(&ReconfigShmem->marker_slow_ack_count, 0); + pg_atomic_init_u64(&ReconfigShmem->marker_timeout_count, 0); } /* @@ -306,6 +342,28 @@ cluster_reconfig_publish_event(const ReconfigEvent *evt) } +static void +cluster_reconfig_log_failstop_epoch_bump(const ReconfigEvent *evt) +{ + char dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8 * 4 + 1]; + int off = 0; + int n; + + Assert(evt != NULL); + + dead[0] = '\0'; + for (n = 0; n < CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8; n++) { + if (evt->dead_bitmap[n / 8] & (1 << (n % 8))) + off += snprintf(dead + off, sizeof(dead) - off, "%s%d", off ? "," : "", n); + } + + ereport(LOG, (errmsg("cluster reconfig: fail-stop epoch bump %llu -> %llu published " + "(coordinator node %d, dead node(s) {%s})", + (unsigned long long)evt->old_epoch, (unsigned long long)evt->new_epoch, + (int)evt->coordinator_node_id, dead))); +} + + /* ============================================================ * Counter accessors (Step 2 + Step 3 SRF support). * ============================================================ @@ -1032,6 +1090,42 @@ cluster_reconfig_get_clean_departed_cleared_count(void) : pg_atomic_read_u64(&ReconfigShmem->clean_departed_cleared_count); } +void +cluster_reconfig_note_marker_slow_ack(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us) +{ + if (ReconfigShmem == NULL || elapsed_us < 1000000ULL) + return; + + pg_atomic_fetch_add_u64(&ReconfigShmem->marker_slow_ack_count, 1); + ereport(LOG, (errmsg("cluster marker: slow qvotec ACK for %s target node %d took %llu ms", + cluster_marker_async_kind_name(kind), target_node, + (unsigned long long)(elapsed_us / 1000ULL)))); +} + +void +cluster_reconfig_note_marker_timeout(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us) +{ + if (ReconfigShmem != NULL) + pg_atomic_fetch_add_u64(&ReconfigShmem->marker_timeout_count, 1); + ereport(LOG, (errmsg("cluster marker: qvotec marker %s target node %d timed out after %llu ms", + cluster_marker_async_kind_name(kind), target_node, + (unsigned long long)(elapsed_us / 1000ULL)))); +} + +uint64 +cluster_reconfig_get_marker_slow_ack_count(void) +{ + return ReconfigShmem == NULL ? 0 : pg_atomic_read_u64(&ReconfigShmem->marker_slow_ack_count); +} + +uint64 +cluster_reconfig_get_marker_timeout_count(void) +{ + return ReconfigShmem == NULL ? 0 : pg_atomic_read_u64(&ReconfigShmem->marker_timeout_count); +} + /* * cluster_reconfig_join_in_progress -- Hardening v1.0.4 (spec-5.13 clean-leave x * spec-5.15 online-join serialization, P1-1/P2): is a membership JOIN currently in @@ -1148,6 +1242,422 @@ cluster_reconfig_join_publish_proven(uint64 admitted_epoch) return advanced >= ((members / 2u) + 1u); } +static void +cluster_reconfig_release_fence_stage(ClusterReconfigFenceStage *stage) +{ + cluster_marker_async_release_stage(&stage->async); + stage->submitted = false; + memset(&stage->event, 0, sizeof(stage->event)); + memset(&stage->marker, 0, sizeof(stage->marker)); + stage->node_id = -1; + stage->last_incarnation = 0; + stage->removal_event_id = 0; +} + +/* + * spec-2.29a nightly-regression fix — pre-bump staging window guard. + * + * The three pre-bump coordinator paths (fail-stop fence, node-remove, + * join Phase-1) advance the membership epoch at stage-entry but only + * publish the reconfig event once the voting-disk marker ACKs, which now + * spans several LMON ticks instead of the pre-async single-tick spin. + * Between the bump and the publish the GRD recovery IDLE tick would + * re-capture recovery_event_old_epoch as the post-bump value, so its P0 + * accept later reads old == cur and wedges WAIT_EPOCH forever (the + * spec-4.6a section 0 shape, here triggered by the coordinator on ITSELF + * — even in a 2-node cluster with no IC piggyback). While any pre-bump + * stage is live the GRD IDLE tick must hold its last stable (genuine + * pre-reconfig) baseline instead of re-capturing. join Phase-2 COMMITTED + * does not pre-bump, so it is intentionally excluded. + */ +bool +cluster_reconfig_has_pending_prebump_stage(void) +{ + /* spec-2.29a r2 t/274: the shmem bit covers a BACKEND-context coordinator's + * bump→publish window (LMON cannot see that process's local stage); the + * three LMON-local staged flags cover the LMON-driven paths. */ + if (ReconfigShmem != NULL && pg_atomic_read_u32(&ReconfigShmem->prebump_sync_active) != 0) + return true; + return failstop_fence_stage.async.has_staged_event + || node_removed_fence_stage.async.has_staged_event + || join_prepare_stage.async.has_staged_event; +} + +static bool +cluster_reconfig_submit_fence_stage(ClusterReconfigFenceStage *stage, ClusterMarkerAsyncKind kind, + int32 target_node, TimestampTz now) +{ + if (stage->submitted) + return true; + if (!cluster_write_fence_submit_marker_async(&stage->async, &stage->marker, kind, target_node, + now)) + return false; + stage->submitted = true; + return true; +} + +static bool +cluster_reconfig_poll_failstop_fence_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + + if (!failstop_fence_stage.async.has_staged_event) + return false; + + now = GetCurrentTimestamp(); + if (!cluster_reconfig_submit_fence_stage(&failstop_fence_stage, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + failstop_fence_stage.event.coordinator_node_id, now)) + return true; + + pr = cluster_write_fence_poll_marker_async(&failstop_fence_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return true; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + failstop_fence_stage.event.coordinator_node_id, + elapsed_us); + cluster_reconfig_release_fence_stage(&failstop_fence_stage); + return true; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + failstop_fence_stage.event.coordinator_node_id, + elapsed_us); + if (result == CLUSTER_FENCE_MARKER_SUBMIT_ACK) { + cluster_reconfig_publish_event(&failstop_fence_stage.event); + cluster_reconfig_log_failstop_epoch_bump(&failstop_fence_stage.event); + cluster_reconfig_broadcast_local_procsig(); + } else { + ereport(LOG, + (errmsg("cluster reconfig: fence marker did not reach a voting-disk majority " + "for epoch %llu; not publishing reconfig event (write-fenced, will retry)", + (unsigned long long)failstop_fence_stage.event.new_epoch))); + } + cluster_reconfig_release_fence_stage(&failstop_fence_stage); + return true; +} + +static uint64 +cluster_reconfig_poll_node_removed_fence_stage(int32 removed_node_id, uint64 removal_event_id, + uint64 last_incarnation) +{ + TimestampTz now; + uint32 result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + + if (!node_removed_fence_stage.async.has_staged_event) + return 0; + if (node_removed_fence_stage.node_id != removed_node_id + || node_removed_fence_stage.removal_event_id != removal_event_id) + return 0; + + now = GetCurrentTimestamp(); + if (!cluster_reconfig_submit_fence_stage(&node_removed_fence_stage, + CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + removed_node_id, now)) + return 0; + + pr = cluster_write_fence_poll_marker_async(&node_removed_fence_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return 0; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + removed_node_id, elapsed_us); + cluster_reconfig_release_fence_stage(&node_removed_fence_stage); + return 0; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, removed_node_id, + elapsed_us); + if (result != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { + ereport(LOG, (errmsg("cluster node removal: fence marker for node %d did not reach a " + "voting-disk majority for epoch %llu; not publishing removal " + "(write-fenced, will retry)", + removed_node_id, + (unsigned long long)node_removed_fence_stage.event.new_epoch))); + cluster_reconfig_release_fence_stage(&node_removed_fence_stage); + return 0; + } + + CLUSTER_INJECTION_POINT("cluster-node-remove-fence-armed"); + cluster_reconfig_publish_event(&node_removed_fence_stage.event); + cluster_reconfig_record_removed(removed_node_id, node_removed_fence_stage.event.new_epoch, + false); + LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); + cluster_membership_shrink_to_removed(removed_node_id, last_incarnation); + LWLockRelease(&ReconfigShmem->lock); + + { + uint64 new_epoch = node_removed_fence_stage.event.new_epoch; + + cluster_reconfig_release_fence_stage(&node_removed_fence_stage); + return new_epoch; + } +} + +static void +cluster_reconfig_release_join_prepare_stage(void) +{ + cluster_marker_async_release_stage(&join_prepare_stage.async); + memset(&join_prepare_stage.event, 0, sizeof(join_prepare_stage.event)); + memset(join_prepare_stage.join_bitmap, 0, sizeof(join_prepare_stage.join_bitmap)); + memset(join_prepare_stage.joiner_incarnations, 0, + sizeof(join_prepare_stage.joiner_incarnations)); + join_prepare_stage.next_node = 0; + join_prepare_stage.submitted = false; +} + +/* (spec-2.29a review r1: the former abort_join_prepare_stage — revert + * JOINING→DEAD + clear pending on PREPARE failure — was removed. Q5=A makes + * PREPARE strictly best-effort: TIMEOUT / non-ACK advances the queue and the + * staged JOIN_PENDING still publishes, so no revert path exists; reverting + * would let compute_join_bitmap re-detect the joiner and re-bump the epoch + * every deadline period — the P1-1 bump storm.) */ + +static bool +cluster_reconfig_submit_join_prepare_current(TimestampTz now) +{ + ClusterJoinCommitMarker m; + int i; + + for (i = join_prepare_stage.next_node; i < CLUSTER_MAX_NODES; i++) { + if (dead_bitmap_test_bit(join_prepare_stage.join_bitmap, i)) + break; + } + join_prepare_stage.next_node = i; + if (i >= CLUSTER_MAX_NODES) { + cluster_reconfig_publish_event(&join_prepare_stage.event); + pg_atomic_fetch_add_u64(&ReconfigShmem->join_pending_count, 1); + cluster_reconfig_broadcast_local_procsig(); + cluster_reconfig_release_join_prepare_stage(); + return true; + } + + if (join_prepare_stage.submitted) + return true; + + memset(&m, 0, sizeof(m)); + m.magic = CLUSTER_JCMK_MAGIC; + m.version = CLUSTER_JCMK_VERSION; + m.node_id = i; + m.phase = CLUSTER_JCMK_PHASE_PREPARE; + m.admitted_incarnation = join_prepare_stage.joiner_incarnations[i]; + m.generation = join_prepare_stage.joiner_incarnations[i]; + m.admitted_epoch = join_prepare_stage.event.new_epoch; + cluster_join_marker_compute_crc(&m); + + if (!cluster_reconfig_submit_join_marker_async(&join_prepare_stage.async, i, &m, + CLUSTER_MARKER_KIND_JOIN_PREPARE, now)) + return false; + join_prepare_stage.submitted = true; + return true; +} + +/* + * spec-2.29a review r1 P2 — node-remove driver pre-work gate. + * + * While the staged node-removed fence marker from a previous FENCE_ARMING + * tick is still in flight, the driver must not re-run its pre-work + * (REMOVING marker write / stripe retire): the epoch has already been + * self-bumped by the stage-entry, so a re-run would key a SECOND REMOVING + * marker to the post-bump epoch and add voting-disk writes during the + * exact contention window the async stage exists to relieve. + */ +bool +cluster_reconfig_node_removed_fence_stage_pending(void) +{ + return node_removed_fence_stage.async.has_staged_event; +} + +static bool +cluster_reconfig_poll_join_prepare_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_JOIN_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + int target; + + if (!join_prepare_stage.async.has_staged_event) + return false; + + now = GetCurrentTimestamp(); + if (!join_prepare_stage.submitted) { + (void)cluster_reconfig_submit_join_prepare_current(now); + return true; + } + + target = join_prepare_stage.next_node; + pr = cluster_reconfig_poll_join_marker_async(&join_prepare_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return true; + + /* + * spec-2.29a Q5=A (review r1 P1): PREPARE is best-effort in OUTCOME — the + * pre-async code ignored the submit result and always published + * JOIN_PENDING (only the Phase-2 COMMITTED marker is a commit point, P1-r5). + * A TIMEOUT / non-ACK PREPARE therefore advances the queue instead of + * aborting: aborting would revert the joiner JOINING→DEAD, and the next + * tick's compute_join_bitmap would re-detect it CSSD-alive and RE-BUMP the + * epoch — exactly the per-tick epoch-bump storm the P1-1 staged record + * forbids. Draining to publish keeps the joiner on a stable JOINING + * (one Phase-1 bump total, regardless of PREPARE outcomes). + */ + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, elapsed_us); + } else { + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, elapsed_us); + if (result != CLUSTER_JOIN_MARKER_SUBMIT_ACK) + ereport(LOG, + (errmsg("cluster membership: PREPARE join marker for node %d did not reach " + "a voting-disk majority; continuing best-effort (COMMITTED is the " + "commit point)", + target))); + } + + join_prepare_stage.submitted = false; + join_prepare_stage.next_node++; + (void)cluster_reconfig_submit_join_prepare_current(now); + return true; +} + +static void +cluster_reconfig_release_join_commit_stage(void) +{ + cluster_marker_async_release_stage(&join_commit_stage.async); + memset(&join_commit_stage.marker, 0, sizeof(join_commit_stage.marker)); + join_commit_stage.node_id = -1; + join_commit_stage.admitted_incarnation = 0; + join_commit_stage.submitted = false; +} + +static bool +cluster_reconfig_publish_join_commit(int32 node_id, uint64 admitted_incarnation, + uint64 expected_epoch) +{ + uint64 old_epoch, new_epoch; + XLogRecPtr lsn; + ReconfigEvent evt; + uint8 jb[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + uint8 empty_dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + uint64 incs[CLUSTER_MAX_NODES]; + + if (cluster_epoch_get_current() + 1 != expected_epoch) + return false; + + CLUSTER_INJECTION_POINT("cluster-reconfig-join-commit-marker-durable"); + + cluster_epoch_advance_for_reconfig(&old_epoch, &new_epoch); + if (new_epoch != expected_epoch) + return false; + lsn = GetXLogInsertRecPtr(); + cluster_epoch_set_changed_at_lsn((uint64)lsn); + cluster_gcs_block_on_epoch_advance(new_epoch); + cluster_sinval_reset_all_on_reconfig(); + cluster_tt_status_flush_all((uint32)new_epoch); + + LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); + cluster_membership_set_state(node_id, CLUSTER_MEMBER_MEMBER); + cluster_membership_record_admitted(node_id, admitted_incarnation); + ReconfigShmem->pending_join_bitmap[node_id / 8] &= (uint8) ~(1u << (node_id % 8)); + LWLockRelease(&ReconfigShmem->lock); + + if (cluster_reconfig_is_clean_departed(node_id)) + pg_atomic_fetch_add_u64(&ReconfigShmem->clean_departed_cleared_count, 1); + cluster_reconfig_clear_clean_departed(node_id); + + dead_bitmap_set_bit(jb, node_id); + memset(incs, 0, sizeof(incs)); + incs[node_id] = admitted_incarnation; + + memset(&evt, 0, sizeof(evt)); + evt.event_id = cluster_reconfig_compute_event_id_v2( + RECONFIG_KIND_JOIN_COMMITTED, empty_dead, jb, incs, cluster_cssd_get_dead_generation()); + evt.coordinator_node_id = cluster_node_id; + evt.old_epoch = old_epoch; + evt.new_epoch = new_epoch; + memcpy(evt.join_bitmap, jb, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); + evt.applied_at = GetCurrentTimestamp(); + evt.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; + evt.cssd_dead_generation = cluster_cssd_get_dead_generation(); + evt.reconfig_kind = RECONFIG_KIND_JOIN_COMMITTED; + cluster_reconfig_publish_event(&evt); + pg_atomic_fetch_add_u64(&ReconfigShmem->join_apply_count, 1); + + return true; +} + +static bool +cluster_reconfig_poll_join_commit_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_JOIN_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + uint64 admitted_incarnation = 0; + uint64 admitted_generation = 0; + ClusterMarkerPollResult pr; + + if (!join_commit_stage.async.has_staged_event) + return false; + + now = GetCurrentTimestamp(); + if (!join_commit_stage.submitted) { + if (!cluster_reconfig_submit_join_marker_async( + &join_commit_stage.async, join_commit_stage.node_id, &join_commit_stage.marker, + CLUSTER_MARKER_KIND_JOIN_COMMITTED, now)) + return true; + join_commit_stage.submitted = true; + return true; + } + + pr = cluster_reconfig_poll_join_marker_async(&join_commit_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return true; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_JOIN_COMMITTED, + join_commit_stage.node_id, elapsed_us); + cluster_reconfig_release_join_commit_stage(); + return true; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_JOIN_COMMITTED, + join_commit_stage.node_id, elapsed_us); + if (result != CLUSTER_JOIN_MARKER_SUBMIT_ACK) { + ereport(LOG, + (errmsg("cluster membership: COMMITTED join marker for node %d did not reach a " + "voting-disk majority; not committing (will retry)", + join_commit_stage.node_id))); + cluster_reconfig_release_join_commit_stage(); + return true; + } + + if (!cluster_reconfig_get_observed_slot(join_commit_stage.node_id, &admitted_incarnation, + &admitted_generation) + || admitted_incarnation != join_commit_stage.admitted_incarnation + || cluster_membership_vet_joiner(join_commit_stage.node_id, admitted_incarnation, + admitted_generation) + != CLUSTER_JOIN_ACCEPT + || !cluster_reconfig_publish_join_commit(join_commit_stage.node_id, + join_commit_stage.admitted_incarnation, + join_commit_stage.async.staged_expect_epoch)) { + pg_atomic_fetch_add_u64(&ReconfigShmem->join_reject_count, 1); + cluster_reconfig_release_join_commit_stage(); + return true; + } + + cluster_reconfig_release_join_commit_stage(); + return true; +} + /* * spec-5.15 D4 — coordinator-side join driver (called from the tick when * online_join is on and self is the min-MEMBER coordinator). Phase-1: fresh @@ -1166,6 +1676,10 @@ cluster_reconfig_drive_joins(int coordinator) if (state == NULL) return; + if (cluster_reconfig_poll_join_prepare_stage()) + return; + if (cluster_reconfig_poll_join_commit_stage()) + return; /* Phase-1 detection + a snapshot of the current pending set, under the lock * (compute_join_bitmap reads membership_state). */ @@ -1182,8 +1696,7 @@ cluster_reconfig_drive_joins(int coordinator) (void)cluster_reconfig_get_observed_slot(i, &joiner_incarnations[i], NULL); } cluster_reconfig_apply_join_as_coordinator(join_bitmap, coordinator, joiner_incarnations); - /* enter the JOIN_PENDING transition fail-closed on every local backend. */ - cluster_reconfig_broadcast_local_procsig(); + return; } /* Phase-2: commit pending joins that have converged. The just-added Phase-1 @@ -1646,6 +2159,7 @@ cluster_reconfig_lmon_tick(void) uint64 cssd_dead_generation; uint64 event_id; int i; + bool failstop_stage_handled = false; /* L20: runtime feature flag check first line. */ if (!cluster_enabled) @@ -1665,6 +2179,7 @@ cluster_reconfig_lmon_tick(void) self_id = cluster_node_id; if (self_id < 0 || self_id >= CLUSTER_MAX_NODES) return; /* defensive: bad self id, cannot participate */ + failstop_stage_handled = cluster_reconfig_poll_failstop_fence_stage(); /* * §3.1 + F11: build the raw CSSD DEAD bitmap, filtering out un-declared @@ -1886,7 +2401,7 @@ cluster_reconfig_lmon_tick(void) * stabilizing the survivor base), THEN the join edge. Each is an independent * ReconfigEvent; neither early-returns past the other. */ - if (!dead_bitmap_is_zero(dead_bitmap)) { + if (!failstop_stage_handled && !dead_bitmap_is_zero(dead_bitmap)) { CLUSTER_INJECTION_POINT("cluster-reconfig-decide-coordinator"); /* §3.2 P1.2: event_id from dead_bitmap + dead_generation snapshot. */ @@ -2024,6 +2539,16 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( CLUSTER_INJECTION_POINT("cluster-reconfig-epoch-bump-pre"); + /* + * spec-2.29a r2 t/274: mark the pre-bump→publish window BEFORE bumping the + * epoch, so a concurrent LMON GRD IDLE tick holds its WAIT_EPOCH baseline + * while this (possibly backend-context, synchronous) path bumps but has not + * yet published. Cleared at every return below. Harmless on the LMON / + * fence-off paths (they either hand off to the local staged flag or publish + * within this same call with no intervening tick). + */ + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 1); + /* D18: atomic CAS-loop increment. Returns pre/post snapshots. */ cluster_epoch_advance_for_reconfig(&old_epoch, &new_epoch); @@ -2123,13 +2648,44 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( marker.fenced_dead_bitmap[b] |= removed_bitmap[b]; } - if (cluster_write_fence_submit_marker(&marker) != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { - ereport(LOG, (errmsg("cluster reconfig: fence marker did not reach a voting-disk " - "majority for epoch %llu; not publishing reconfig event " - "(write-fenced, will retry)", - (unsigned long long)new_epoch))); - return; /* fail-closed: epoch bumped, event NOT published, recovery NOT started */ + /* + * The async stage is process-local by design and is driven only by LMON + * ticks. Test-only backend callers, such as + * cluster_reconfig_inject_dead_node_test(), would otherwise stage the marker + * in their own process and return with no LMON-visible pending publish + * record. Keep those non-LMON paths on the old bounded wait: they are not + * transport-liveness actors, so this does not reintroduce the BUG-C1 LMON + * park. + */ + if (MyBackendType != B_LMON) { + ClusterFenceMarkerSubmitResult result; + + result = cluster_write_fence_submit_marker(&marker); + if (result != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { + ereport(LOG, (errmsg("cluster reconfig: fence marker did not reach a voting-disk " + "majority for epoch %llu; not publishing reconfig event " + "(write-fenced, will retry)", + (unsigned long long)new_epoch))); + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); + return; + } + + cluster_reconfig_publish_event(&evt); + cluster_reconfig_log_failstop_epoch_bump(&evt); + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); + return; } + + failstop_fence_stage.event = evt; + failstop_fence_stage.marker = marker; + failstop_fence_stage.node_id = coordinator_node_id; + failstop_fence_stage.async.has_staged_event = true; + (void)cluster_reconfig_submit_fence_stage(&failstop_fence_stage, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + coordinator_node_id, GetCurrentTimestamp()); + /* LMON path: the local staged flag now covers the pending window. */ + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); + return; /* fail-closed until the staged marker is majority-durable */ } cluster_reconfig_publish_event(&evt); @@ -2143,21 +2699,10 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( * fixed stack buffer (worst case: 128 node ids x 4 chars) keeps the LMON * tick free of allocator traffic. */ - { - char dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8 * 4 + 1]; - int off = 0; - int n; - - dead[0] = '\0'; - for (n = 0; n < CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8; n++) { - if (dead_bitmap[n / 8] & (1 << (n % 8))) - off += snprintf(dead + off, sizeof(dead) - off, "%s%d", off ? "," : "", n); - } - ereport(LOG, (errmsg("cluster reconfig: fail-stop epoch bump %llu -> %llu published " - "(coordinator node %d, dead node(s) {%s})", - (unsigned long long)old_epoch, (unsigned long long)new_epoch, - (int)coordinator_node_id, dead))); - } + cluster_reconfig_log_failstop_epoch_bump(&evt); + + /* spec-2.29a r2 t/274: fence-off path published within this call. */ + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); } @@ -2287,6 +2832,9 @@ cluster_reconfig_apply_node_removed_as_coordinator(int32 removed_node_id, uint64 return 0; if (removed_node_id < 0 || removed_node_id >= CLUSTER_MAX_NODES) return 0; + if (node_removed_fence_stage.async.has_staged_event) + return cluster_reconfig_poll_node_removed_fence_stage(removed_node_id, removal_event_id, + last_incarnation); CLUSTER_INJECTION_POINT("cluster-node-remove-shrink-committing"); @@ -2340,13 +2888,25 @@ cluster_reconfig_apply_node_removed_as_coordinator(int32 removed_node_id, uint64 * a concurrent dead set, if any, is carried by its own fail-stop fence. */ memcpy(marker.fenced_dead_bitmap, removed_with_n, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); - if (cluster_write_fence_submit_marker(&marker) != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { - ereport(LOG, (errmsg("cluster node removal: fence marker for node %d did not reach a " - "voting-disk majority for epoch %llu; not publishing removal " - "(write-fenced, will retry)", - removed_node_id, (unsigned long long)new_epoch))); - return 0; /* fail-closed: epoch bumped, removal NOT published, driver retries */ - } + node_removed_fence_stage.event.event_id + = cluster_reconfig_compute_removal_event_id(removed_with_n, removal_event_id); + node_removed_fence_stage.event.coordinator_node_id = cluster_node_id; + node_removed_fence_stage.event.old_epoch = old_epoch; + node_removed_fence_stage.event.new_epoch = new_epoch; + memcpy(node_removed_fence_stage.event.dead_bitmap, empty_dead, + CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); + node_removed_fence_stage.event.applied_at = GetCurrentTimestamp(); + node_removed_fence_stage.event.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; + node_removed_fence_stage.event.cssd_dead_generation = cssd_dead_generation; + node_removed_fence_stage.event.reconfig_kind = RECONFIG_KIND_NODE_REMOVED; + node_removed_fence_stage.marker = marker; + node_removed_fence_stage.node_id = removed_node_id; + node_removed_fence_stage.last_incarnation = last_incarnation; + node_removed_fence_stage.removal_event_id = removal_event_id; + node_removed_fence_stage.async.has_staged_event = true; + (void)cluster_reconfig_poll_node_removed_fence_stage(removed_node_id, removal_event_id, + last_incarnation); + return 0; /* fail-closed until the staged fence marker ACKs */ } CLUSTER_INJECTION_POINT("cluster-node-remove-fence-armed"); @@ -2431,6 +2991,42 @@ cluster_reconfig_submit_join_marker(int32 target_node, const ClusterJoinCommitMa } } +bool +cluster_reconfig_submit_join_marker_async(ClusterMarkerAsync *a, int32 target_node, + const ClusterJoinCommitMarker *m, + ClusterMarkerAsyncKind kind, TimestampTz now) +{ + int wait_ms; + + if (ReconfigShmem == NULL || m == NULL || a == NULL) + return false; + if (target_node < 0 || target_node >= CLUSTER_MAX_NODES) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&ReconfigShmem->join_marker_request_seq, + &ReconfigShmem->join_marker_completion_seq)) + return false; + + ReconfigShmem->join_marker_target_node_id = target_node; + ReconfigShmem->join_pending_marker = *m; + wait_ms = cluster_quorum_poll_interval_ms * 3 + 2000; + return cluster_marker_async_submit( + a, &ReconfigShmem->join_marker_request_seq, &ReconfigShmem->join_marker_completion_seq, + ReconfigShmem->join_qvotec_latch, now, (uint64)wait_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_reconfig_poll_join_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) +{ + if (ReconfigShmem == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &ReconfigShmem->join_marker_completion_seq, + &ReconfigShmem->join_marker_result, now, out_result, + out_elapsed_us); +} + bool cluster_reconfig_join_qvotec_poll_pending(int32 *out_target_node, void *out_slot512) { @@ -2464,6 +3060,7 @@ cluster_reconfig_join_qvotec_complete(bool acked) pg_atomic_write_u64(&ReconfigShmem->join_marker_completion_seq, join_qvotec_inflight_marker_seq); join_qvotec_last_processed_marker_seq = join_qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } static void @@ -2614,27 +3211,6 @@ cluster_reconfig_apply_join_as_coordinator( } LWLockRelease(&ReconfigShmem->lock); - /* Durable PREPARE marker per joiner (records the presented incarnation; does - * NOT seed — only COMMITTED is a basis). Best-effort: PREPARE failure does - * not block the JOIN_PENDING publish (the COMMITTED marker in Phase-2 is the - * commit point that must be majority-durable). */ - for (i = 0; i < CLUSTER_MAX_NODES; i++) { - ClusterJoinCommitMarker m; - - if (!dead_bitmap_test_bit(join_bitmap, i)) - continue; - memset(&m, 0, sizeof(m)); - m.magic = CLUSTER_JCMK_MAGIC; - m.version = CLUSTER_JCMK_VERSION; - m.node_id = i; - m.phase = CLUSTER_JCMK_PHASE_PREPARE; - m.admitted_incarnation = joiner_incarnations[i]; - m.generation = joiner_incarnations[i]; /* monotonic per node (read-newest intent) */ - m.admitted_epoch = new_epoch; - cluster_join_marker_compute_crc(&m); - (void)cluster_reconfig_submit_join_marker(i, &m); - } - memset(&evt, 0, sizeof(evt)); evt.event_id = cluster_reconfig_compute_event_id_v2(RECONFIG_KIND_JOIN_PENDING, empty_dead, join_bitmap, @@ -2647,20 +3223,21 @@ cluster_reconfig_apply_join_as_coordinator( evt.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; evt.cssd_dead_generation = cssd_dead_generation; evt.reconfig_kind = RECONFIG_KIND_JOIN_PENDING; - cluster_reconfig_publish_event(&evt); - pg_atomic_fetch_add_u64(&ReconfigShmem->join_pending_count, 1); + + join_prepare_stage.event = evt; + memcpy(join_prepare_stage.join_bitmap, join_bitmap, sizeof(join_prepare_stage.join_bitmap)); + memcpy(join_prepare_stage.joiner_incarnations, joiner_incarnations, + sizeof(join_prepare_stage.joiner_incarnations)); + join_prepare_stage.next_node = 0; + join_prepare_stage.submitted = false; + join_prepare_stage.async.has_staged_event = true; + (void)cluster_reconfig_poll_join_prepare_stage(); } bool cluster_reconfig_commit_member(int32 node_id, uint64 admitted_incarnation) { ClusterJoinCommitMarker m; - uint64 old_epoch, new_epoch; - XLogRecPtr lsn; - ReconfigEvent evt; - uint8 jb[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; - uint8 empty_dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; - uint64 incs[CLUSTER_MAX_NODES]; if (!cluster_enabled || ReconfigShmem == NULL) return false; @@ -2704,60 +3281,17 @@ cluster_reconfig_commit_member(int32 node_id, uint64 admitted_incarnation) m.commit_nonce = ((uint64)cluster_node_id << 56) ^ (uint64)GetCurrentTimestamp(); cluster_join_marker_compute_crc(&m); - if (cluster_reconfig_submit_join_marker(node_id, &m) != CLUSTER_JOIN_MARKER_SUBMIT_ACK) { - ereport(LOG, - (errmsg("cluster membership: COMMITTED join marker for node %d did not reach a " - "voting-disk majority; not committing (will retry)", - node_id))); + if (join_commit_stage.async.has_staged_event) return false; - } - - CLUSTER_INJECTION_POINT("cluster-reconfig-join-commit-marker-durable"); + join_commit_stage.marker = m; + join_commit_stage.node_id = node_id; + join_commit_stage.admitted_incarnation = admitted_incarnation; + join_commit_stage.async.has_staged_event = true; + join_commit_stage.async.staged_expect_epoch = m.admitted_epoch; + join_commit_stage.submitted = false; + (void)cluster_reconfig_poll_join_commit_stage(); - /* - * ② publish: bump JOIN_COMMITTED epoch + state MEMBER + last_admitted + clear - * pending + clear clean_departed[node] (INV-J10). Strict order — the publish - * (epoch + state MEMBER) precedes the joiner opening its write gate (D5). - */ - cluster_epoch_advance_for_reconfig(&old_epoch, &new_epoch); - lsn = GetXLogInsertRecPtr(); - cluster_epoch_set_changed_at_lsn((uint64)lsn); - cluster_gcs_block_on_epoch_advance(new_epoch); - cluster_sinval_reset_all_on_reconfig(); - cluster_tt_status_flush_all((uint32)new_epoch); - - LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); - cluster_membership_set_state(node_id, CLUSTER_MEMBER_MEMBER); - cluster_membership_record_admitted(node_id, admitted_incarnation); - ReconfigShmem->pending_join_bitmap[node_id / 8] &= (uint8) ~(1u << (node_id % 8)); - LWLockRelease(&ReconfigShmem->lock); - - /* INV-J10: clear the in-shmem clean_departed suppression so a clean-left node - * that just rejoined has its later real fail-stop honored again (the durable - * supersede across restart is resolved by the seed, RC-5). */ - if (cluster_reconfig_is_clean_departed(node_id)) - pg_atomic_fetch_add_u64(&ReconfigShmem->clean_departed_cleared_count, 1); - cluster_reconfig_clear_clean_departed(node_id); - - dead_bitmap_set_bit(jb, node_id); - memset(incs, 0, sizeof(incs)); - incs[node_id] = admitted_incarnation; - - memset(&evt, 0, sizeof(evt)); - evt.event_id = cluster_reconfig_compute_event_id_v2( - RECONFIG_KIND_JOIN_COMMITTED, empty_dead, jb, incs, cluster_cssd_get_dead_generation()); - evt.coordinator_node_id = cluster_node_id; - evt.old_epoch = old_epoch; - evt.new_epoch = new_epoch; - memcpy(evt.join_bitmap, jb, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); - evt.applied_at = GetCurrentTimestamp(); - evt.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; - evt.cssd_dead_generation = cluster_cssd_get_dead_generation(); - evt.reconfig_kind = RECONFIG_KIND_JOIN_COMMITTED; - cluster_reconfig_publish_event(&evt); - pg_atomic_fetch_add_u64(&ReconfigShmem->join_apply_count, 1); - - return true; + return false; } diff --git a/src/backend/cluster/cluster_shmem.c b/src/backend/cluster/cluster_shmem.c index fa4b2db725..5444528540 100644 --- a/src/backend/cluster/cluster_shmem.c +++ b/src/backend/cluster/cluster_shmem.c @@ -1077,6 +1077,22 @@ cluster_init_shmem(void) errhint("Enable cluster.online_join on every node, or disable " "cluster.xid_striping."))); + /* + * spec-6.15b D6: shared_catalog puts catalog heap pages on one shared + * storage image, so a multi-node cluster must not allocate dense native + * xids from every node. Without striping, two nodes can reuse the seed + * native-era xid range and poison shared catalog visibility. + */ + if (cluster_enabled && cluster_shared_catalog && cluster_conf_node_count() > 1 + && !cluster_xid_striping) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("cluster.shared_catalog on a multi-node cluster requires cluster.xid_striping"), + errdetail("The configured cluster declares %d nodes.", cluster_conf_node_count()), + errhint("Set cluster.xid_striping=on (and cluster.online_join=on) on every " + "shared_catalog node."))); + /* * Stage 0.18: bind the cluster_ic vtable for the configured tier. * Stub mode allocates no shmem; tier1+ (Stage 2+) will piggyback diff --git a/src/backend/cluster/cluster_wal_thread.c b/src/backend/cluster/cluster_wal_thread.c index 17edd98586..b5f7cab947 100644 --- a/src/backend/cluster/cluster_wal_thread.c +++ b/src/backend/cluster/cluster_wal_thread.c @@ -45,7 +45,9 @@ #include #include "access/xlog_internal.h" /* XLOGDIR */ +#include "cluster/cluster_conf.h" #include "cluster/cluster_guc.h" +#include "cluster/cluster_hw.h" #include "cluster/cluster_inject.h" #include "cluster/cluster_shmem.h" #include "cluster/cluster_wal_state.h" /* spec-4.2 ensure() */ @@ -385,8 +387,23 @@ cluster_wal_thread_init(void) cluster_wal_thread_shmem->dir_configured = dir_set ? 1 : 0; } - if (!dir_set) + if (!dir_set) { + if (cluster_shared_catalog && cluster_conf_has_peers()) + ereport(FATAL, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cluster.shared_catalog requires cluster.wal_threads_dir in a " + "multi-node cluster"), + errhint("Set cluster.wal_threads_dir to the shared per-thread WAL root and " + "initialise each node with pgrac-init --wal-threads-dir."))); + if (!cluster_hw_remaster_recoverable()) + ereport(WARNING, + (errmsg("cluster HW remaster is not recoverable because " + "cluster.wal_threads_dir is unset"), + errhint("A dead node's adopted HW authority cannot be rebuilt from per-thread " + "WAL until cluster.wal_threads_dir is configured and the node is " + "restarted."))); return; /* flat layout: identity stamping only (Q3-A) */ + } /* * Configuration coherence, fail-closed (L58: enumerate the diff --git a/src/backend/cluster/cluster_write_fence.c b/src/backend/cluster/cluster_write_fence.c index 2baebb13bd..8917959359 100644 --- a/src/backend/cluster/cluster_write_fence.c +++ b/src/backend/cluster/cluster_write_fence.c @@ -40,6 +40,7 @@ #include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ #include "cluster/cluster_guc.h" /* cluster_node_id, cluster_voting_disks */ +#include "cluster/cluster_lmon.h" /* cluster_lmon_marker_complete_wakeup */ #include "cluster/cluster_qvotec.h" /* ClusterVotingSlot (marker layout asserts) */ #include "cluster/cluster_shmem.h" /* cluster_shmem_register_region */ #include "cluster/cluster_voting_disk_io.h" /* D6 direct durable marker read */ @@ -769,6 +770,37 @@ cluster_write_fence_submit_marker(const ClusterFenceMarker *m) } } +bool +cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, const ClusterFenceMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + if (cluster_write_fence_shmem == NULL || m == NULL || a == NULL) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&cluster_write_fence_shmem->marker_request_seq, + &cluster_write_fence_shmem->marker_completion_seq)) + return false; + + memcpy(&cluster_write_fence_shmem->pending_marker, m, sizeof(*m)); + return cluster_marker_async_submit( + a, &cluster_write_fence_shmem->marker_request_seq, + &cluster_write_fence_shmem->marker_completion_seq, cluster_write_fence_shmem->qvotec_latch, + now, (uint64)cluster_write_fence_lease_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) +{ + if (cluster_write_fence_shmem == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &cluster_write_fence_shmem->marker_completion_seq, + &cluster_write_fence_shmem->marker_result, now, out_result, + out_elapsed_us); +} + /* * cluster_write_fence_clear_qvotec_latch / _publish_qvotec_latch -- qvotec publishes * its MyLatch at startup so LMON can wake it; auto-cleared at proc_exit so a stale @@ -835,6 +867,7 @@ cluster_write_fence_qvotec_complete(bool acked) pg_atomic_write_u64(&cluster_write_fence_shmem->marker_completion_seq, qvotec_inflight_marker_seq); qvotec_last_processed_marker_seq = qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } #endif /* USE_PGRAC_CLUSTER */ diff --git a/src/backend/cluster/cluster_xid_authority.c b/src/backend/cluster/cluster_xid_authority.c new file mode 100644 index 0000000000..6a16671c57 --- /dev/null +++ b/src/backend/cluster/cluster_xid_authority.c @@ -0,0 +1,494 @@ +/*------------------------------------------------------------------------- + * + * cluster_xid_authority.c + * Shared XID authority: never-lowered native-era xid high-water plus + * one-way lifecycle flags, as a torn-safe file under + * cluster.shared_data_dir (spec-6.15b D1). + * + * The authority records the first xid NOT consumed by the native era + * (the pre-formation cluster.enabled=off single-writer window of the + * seed node) and two one-way flags: SEALED (a clean native-era + * shutdown published a complete high-water together with the pg_xact + * prehistory blob, so joiners may adopt) and CLUSTER_ERA (a + * cluster.enabled=on boot closed the native era forever). Writers are + * single by construction: the seed node's postmaster/checkpointer + * during the native era, and the catalog bootstrap window afterwards. + * + * File discipline mirrors cluster_oid_lease.c: temp + fsync + .bak + * roll + durable_rename on write; primary-then-.bak fail-closed read; + * ENOENT-only-absent presence probe. The pure classify layer is + * standalone-linkable for cluster_unit. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_xid_authority.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-6.15b-xid-authority-native-era.md (D1, §4 U1-U3/U5) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include + +#include "cluster/cluster_guc.h" /* cluster_shared_data_dir */ +#include "cluster/cluster_xid_authority.h" +#include "storage/fd.h" + +/* On-disk image size: the fixed header, exactly. */ +#define CLUSTER_XID_AUTHORITY_FILE_SIZE ((int)sizeof(ClusterXidAuthorityHeader)) + +/* ============================================================ + * Pure layer (no elog/shmem/fd; standalone-linkable for cluster_unit). + * ============================================================ */ + +/* + * cluster_xid_authority_classify -- validate an authority image buffer. + */ +ClusterXidAuthorityValidity +cluster_xid_authority_classify(const char *buf, size_t len) +{ + ClusterXidAuthorityHeader hdr; + pg_crc32c crc; + + if (buf == NULL || len < sizeof(ClusterXidAuthorityHeader)) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + memcpy(&hdr, buf, sizeof(hdr)); + + if (hdr.magic != CLUSTER_XID_AUTHORITY_MAGIC) + return CLUSTER_XID_AUTHORITY_INVALID_MAGIC; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, buf, offsetof(ClusterXidAuthorityHeader, crc)); + FIN_CRC32C(crc); + if (!EQ_CRC32C(crc, hdr.crc)) + return CLUSTER_XID_AUTHORITY_INVALID_CRC; + + return CLUSTER_XID_AUTHORITY_VALID; +} + +/* ============================================================ + * Torn-safe authority file I/O (mirrors cluster_oid_lease.c). + * ============================================================ */ + +/* + * build_path -- join cluster_shared_data_dir + relpath into dst. Returns + * false when the shared root is not configured. + */ +static bool +build_path(char *dst, size_t dstlen, const char *relpath) +{ + if (cluster_shared_data_dir == NULL || cluster_shared_data_dir[0] == '\0') + return false; + snprintf(dst, dstlen, "%s/%s", cluster_shared_data_dir, relpath); + return true; +} + +/* + * write_durable -- write buf into tmp, fsync, then durable_rename over final. + * PANICs on any I/O failure (mirrors cluster_oid_lease write_durable). + */ +static void +write_durable(const char *tmp, const char *final, const char *buf) +{ + int fd; + + fd = OpenTransientFile(tmp, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY); + if (fd < 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", tmp))); + + errno = 0; + if (write(fd, buf, CLUSTER_XID_AUTHORITY_FILE_SIZE) != CLUSTER_XID_AUTHORITY_FILE_SIZE) { + if (errno == 0) + errno = ENOSPC; + ereport(PANIC, (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", tmp))); + } + + if (pg_fsync(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmp))); + + if (CloseTransientFile(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not close file \"%s\": %m", tmp))); + + if (durable_rename(tmp, final, PANIC) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", tmp, final))); +} + +/* + * roll_primary_to_bak -- preserve an existing primary into .bak before a new + * primary write. A missing/short primary (first write) is skipped. + */ +static void +roll_primary_to_bak(const char *primary, const char *bak, const char *baktmp) +{ + char buf[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + int fd; + int r; + + fd = OpenTransientFile(primary, O_RDONLY | PG_BINARY); + if (fd < 0) + return; /* first write: no prior primary */ + + r = read(fd, buf, CLUSTER_XID_AUTHORITY_FILE_SIZE); + CloseTransientFile(fd); + if (r != CLUSTER_XID_AUTHORITY_FILE_SIZE) + return; /* short/odd primary: don't manufacture a .bak */ + + write_durable(baktmp, bak, buf); +} + +/* + * read_image -- read a candidate authority file and classify it. Missing or + * short is INVALID_SHORT. + */ +static ClusterXidAuthorityValidity +read_image(const char *path, char *image) +{ + int fd; + int r; + + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY); + if (fd < 0) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + r = read(fd, image, CLUSTER_XID_AUTHORITY_FILE_SIZE); + CloseTransientFile(fd); + if (r != CLUSTER_XID_AUTHORITY_FILE_SIZE) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + return cluster_xid_authority_classify(image, CLUSTER_XID_AUTHORITY_FILE_SIZE); +} + +/* + * copy_flags_settled -- does the on-disk copy at relpath validate AND carry + * all `must_set` flag bits with all `must_clear` bits clear? + */ +static bool +copy_flags_settled(const char *relpath, uint32 must_set, uint32 must_clear) +{ + ClusterXidAuthorityHeader hdr; + char path[MAXPGPATH]; + char image[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + + if (!build_path(path, sizeof(path), relpath)) + return false; + if (read_image(path, image) != CLUSTER_XID_AUTHORITY_VALID) + return false; + memcpy(&hdr, image, sizeof(hdr)); + return (hdr.flags & must_set) == must_set && (hdr.flags & must_clear) == 0; +} + +/* + * both_copies_flags_settled -- is a one-way flag transition complete in BOTH + * on-disk copies? Flag transitions install the same image as primary and + * .bak (primary first), so a crash between the two durable renames leaves a + * pre-transition .bak behind an already-transitioned primary. Nothing else + * on the restart path rewrites the pair, so without a re-assert the stale + * copy survives the whole run, and any later transient primary read failure + * resurrects the pre-transition flags through the .bak fallback -- a stale + * SEALED .bak hands joiners the previous pass's high-water and a stale + * pre-CLUSTER_ERA .bak re-opens the native-era re-entry guard (review + * r3-X1, 8.A). The transition entry points therefore skip their write only + * when this returns true; a missing or invalid copy is "not settled" too, + * which restores the damaged copy for free. + */ +static bool +both_copies_flags_settled(uint32 must_set, uint32 must_clear) +{ + return copy_flags_settled(CLUSTER_XID_AUTHORITY_REL_PATH, must_set, must_clear) + && copy_flags_settled(CLUSTER_XID_AUTHORITY_BAK_REL_PATH, must_set, must_clear); +} + +/* + * cluster_xid_authority_read -- fail-closed read of the shared authority. + * Tries primary then .bak; returns false when neither is trustworthy. Never + * ereports (safe on the bootstrap early-read path). + */ +bool +cluster_xid_authority_read(ClusterXidAuthorityHeader *out) +{ + return cluster_xid_authority_read_checked(out, NULL); +} + +/* + * cluster_xid_authority_read_checked -- as above, additionally reporting + * whether the image came from the .bak fallback so consumers can surface a + * LOG-once operator signal (review F8; the read itself stays silent for + * the bootstrap early-read path). + */ +bool +cluster_xid_authority_read_checked(ClusterXidAuthorityHeader *out, bool *used_bak) +{ + char primary_path[MAXPGPATH]; + char bak_path[MAXPGPATH]; + char image[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + + if (used_bak != NULL) + *used_bak = false; + + if (!build_path(primary_path, sizeof(primary_path), CLUSTER_XID_AUTHORITY_REL_PATH) + || !build_path(bak_path, sizeof(bak_path), CLUSTER_XID_AUTHORITY_BAK_REL_PATH)) + return false; + + if (read_image(primary_path, image) != CLUSTER_XID_AUTHORITY_VALID) { + if (read_image(bak_path, image) != CLUSTER_XID_AUTHORITY_VALID) + return false; /* fail-closed: neither trustworthy */ + if (used_bak != NULL) + *used_bak = true; + } + + memcpy(out, image, sizeof(*out)); + return true; +} + +/* + * cluster_xid_authority_present -- does any authority image (primary or .bak) + * exist on disk at all, trustworthy or not? Lets callers distinguish + * "absent: genuine first seed" from "present but corrupt: fail-closed" -- a + * corrupt authority must never be silently re-seeded (a re-seed from a + * checkpointed value could put already-consumed xids back "in the future"). + * Never ereports. + * + * Fail-closed errno discipline: only ENOENT proves absence (mirrors + * cluster_oid_authority_present; EIO/EACCES/ESTALE report present). + */ +bool +cluster_xid_authority_present(void) +{ + char primary_path[MAXPGPATH]; + char bak_path[MAXPGPATH]; + struct stat st; + + if (!build_path(primary_path, sizeof(primary_path), CLUSTER_XID_AUTHORITY_REL_PATH) + || !build_path(bak_path, sizeof(bak_path), CLUSTER_XID_AUTHORITY_BAK_REL_PATH)) + return false; + + if (stat(primary_path, &st) == 0 || errno != ENOENT) + return true; + if (stat(bak_path, &st) == 0 || errno != ENOENT) + return true; + return false; +} + +/* + * write_header -- CRC-stamp and atomically install an authority header. + */ +static void +write_header(ClusterXidAuthorityHeader *hdr) +{ + char buffer[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char tmp[MAXPGPATH]; + char baktmp[MAXPGPATH]; + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_AUTHORITY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_AUTHORITY_BAK_REL_PATH) + || !build_path(tmp, sizeof(tmp), CLUSTER_XID_AUTHORITY_TMP_REL_PATH) + || !build_path(baktmp, sizeof(baktmp), CLUSTER_XID_AUTHORITY_BAK_TMP_REL_PATH)) + ereport(PANIC, (errmsg("cluster shared_data_dir is not configured"))); + + hdr->magic = CLUSTER_XID_AUTHORITY_MAGIC; + hdr->version = CLUSTER_XID_AUTHORITY_VERSION; + hdr->reserved = 0; + INIT_CRC32C(hdr->crc); + COMP_CRC32C(hdr->crc, (char *)hdr, offsetof(ClusterXidAuthorityHeader, crc)); + FIN_CRC32C(hdr->crc); + + memset(buffer, 0, sizeof(buffer)); + memcpy(buffer, hdr, sizeof(*hdr)); + + roll_primary_to_bak(primary, bak, baktmp); + write_durable(tmp, primary, buffer); +} + +/* + * write_header_both -- CRC-stamp the header and install the SAME image as + * both primary and .bak (primary first). For FLAG transitions (unseal, + * CLUSTER_ERA stamp) the usual roll-primary-to-.bak is unsafe: it would + * preserve a pre-transition image that the fail-closed read falls back to + * when the primary is later damaged -- a stale SEALED .bak hands a joiner + * the previous pass's high-water (review F1, 8.A), and a stale + * pre-CLUSTER_ERA .bak re-opens the native-era re-entry guard. Crash + * points: both copies old (transition not yet taken: state still + * consistent), primary new + .bak old (read prefers the valid primary), + * both new. The primary-new/.bak-old window does not self-repair on its + * own: the transition entry points are re-run by every boot and re-assert + * both copies until both_copies_flags_settled holds (review r3-X1), so a + * later single-copy corruption never resurrects the pre-transition flags. + */ +static void +write_header_both(ClusterXidAuthorityHeader *hdr) +{ + char buffer[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char tmp[MAXPGPATH]; + char baktmp[MAXPGPATH]; + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_AUTHORITY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_AUTHORITY_BAK_REL_PATH) + || !build_path(tmp, sizeof(tmp), CLUSTER_XID_AUTHORITY_TMP_REL_PATH) + || !build_path(baktmp, sizeof(baktmp), CLUSTER_XID_AUTHORITY_BAK_TMP_REL_PATH)) + ereport(PANIC, (errmsg("cluster shared_data_dir is not configured"))); + + hdr->magic = CLUSTER_XID_AUTHORITY_MAGIC; + hdr->version = CLUSTER_XID_AUTHORITY_VERSION; + hdr->reserved = 0; + INIT_CRC32C(hdr->crc); + COMP_CRC32C(hdr->crc, (char *)hdr, offsetof(ClusterXidAuthorityHeader, crc)); + FIN_CRC32C(hdr->crc); + + memset(buffer, 0, sizeof(buffer)); + memcpy(buffer, hdr, sizeof(*hdr)); + + write_durable(tmp, primary, buffer); + write_durable(baktmp, bak, buffer); +} + +/* + * cluster_xid_authority_seed_if_absent -- read-then-seed: if the authority is + * already present (any trustworthy image) do nothing; otherwise create the + * unsealed initial image. Returns true when this call created it. + */ +bool +cluster_xid_authority_seed_if_absent(uint64 initial_native_hw) +{ + ClusterXidAuthorityHeader hdr; + + if (cluster_xid_authority_read(&hdr)) + return false; /* already seeded (join node / prior seed) */ + if (cluster_xid_authority_present()) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is present but corrupt at seed"), + errhint("Restore \"%s\" (or its .bak) from a backup of the shared tree; " + "do not re-seed from a stale xid high-water.", + CLUSTER_XID_AUTHORITY_REL_PATH))); + + memset(&hdr, 0, sizeof(hdr)); + hdr.flags = 0; + hdr.native_hw_full = initial_native_hw; + hdr.next_multi = 0; + write_header(&hdr); + return true; +} + +/* + * cluster_xid_authority_publish_native -- monotone raise of the native-era + * high-water; seal=true additionally stamps SEALED. Flags are never + * cleared and the high-water is never lowered (spec §3.1). An absent + * authority is created outright (crash between bootstrap-seed and the + * first publish); a present-but-corrupt one PANICs rather than re-seed. + */ +void +cluster_xid_authority_publish_native(uint64 native_hw_full, uint64 next_multi, bool seal) +{ + ClusterXidAuthorityHeader hdr; + + if (!cluster_xid_authority_read(&hdr)) { + if (cluster_xid_authority_present()) + ereport(PANIC, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is present but corrupt at publish"), + errhint("Restore \"%s\" (or its .bak) from a backup of the shared tree; " + "re-seeding from a stale high-water could re-issue consumed xids.", + CLUSTER_XID_AUTHORITY_REL_PATH))); + memset(&hdr, 0, sizeof(hdr)); + } + + if (native_hw_full > hdr.native_hw_full) + hdr.native_hw_full = native_hw_full; + if (next_multi > hdr.next_multi) + hdr.next_multi = next_multi; + if (seal) + hdr.flags |= CLUSTER_XID_AUTHORITY_FLAG_SEALED; + + write_header(&hdr); +} + +/* + * cluster_xid_authority_mark_cluster_era -- one-way CLUSTER_ERA stamp (first + * cluster.enabled=on boot; spec §3.1). Idempotent re-assert: re-run by + * every cluster boot, it rewrites BOTH copies until both carry the flag, so + * a crash between write_header_both's two renames (stamped primary, stale + * .bak) is repaired on the next boot instead of surviving the run (review + * r3-X1); once both copies are settled it is a no-write no-op. A + * missing/corrupt authority PANICs: the catalog bootstrap seeds it before + * any caller can reach this point, so absence here is real damage. + */ +void +cluster_xid_authority_mark_cluster_era(void) +{ + ClusterXidAuthorityHeader hdr; + + if (!cluster_xid_authority_read(&hdr)) + ereport(PANIC, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is missing or corrupt at cluster-era stamp"))); + + if ((hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) != 0 + && both_copies_flags_settled(CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, 0)) + return; /* transition complete in both copies */ + + hdr.flags |= CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA; + write_header_both(&hdr); /* review F1 variant: one-way flag in BOTH copies */ +} + +/* + * cluster_xid_authority_begin_native_run -- re-open the native era for a + * follow-up cluster.enabled=off seed run (spec-6.15b §3.1 multi-pass arm). + * + * A sealed authority means "the high-water and prehistory are complete as + * of a clean native shutdown". A NEW native run invalidates that + * completeness the moment it allocates its first xid, so the seal is + * cleared up front: if this run crashes, joiners fail closed (unsealed, + * 53RB5) instead of adopting the previous pass's stale high-water -- + * false-invisible for every xid the crashed pass consumed (rule 8.A). + * The clean shutdown of this run re-publishes and re-seals monotonically. + * + * Only legal while CLUSTER_ERA is unset (the caller FATALs re-entry + * first; re-checked here defensively). Formation recipes keep native-era + * boots and first cluster boots strictly ordered; if a racing first + * cluster boot stamps CLUSTER_ERA inside this read-modify-write window, + * the stamp is re-applied by the next cluster boot (mark is re-issued + * whenever unset) and this authority is left unsealed -- which only ever + * blocks adoption, never yields a wrong answer. + * + * Idempotent re-assert (review r3-X1): re-run by every native-era boot, + * it rewrites BOTH copies until neither retains SEALED, so a crash + * between write_header_both's two renames (unsealed primary, stale + * SEALED .bak) is repaired on the next boot instead of surviving the + * run; once both copies are settled it is a no-write no-op. + */ +void +cluster_xid_authority_begin_native_run(void) +{ + ClusterXidAuthorityHeader hdr; + + if (!cluster_xid_authority_read(&hdr)) + ereport(PANIC, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is missing or corrupt at native-run open"))); + + if (hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("native seed era cannot be re-entered on a formed shared catalog tree"))); + + if ((hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0 + && both_copies_flags_settled(0, CLUSTER_XID_AUTHORITY_FLAG_SEALED)) + return; /* open, and no copy retains SEALED */ + + hdr.flags &= ~CLUSTER_XID_AUTHORITY_FLAG_SEALED; + write_header_both(&hdr); /* review F1: no copy may retain SEALED */ +} diff --git a/src/backend/cluster/cluster_xid_prehistory.c b/src/backend/cluster/cluster_xid_prehistory.c new file mode 100644 index 0000000000..490804ccac --- /dev/null +++ b/src/backend/cluster/cluster_xid_prehistory.c @@ -0,0 +1,650 @@ +/*------------------------------------------------------------------------- + * + * cluster_xid_prehistory.c + * Native-era pg_xact prehistory: publish the seed node's CLOG page run + * [0, native_hw) into a CRC-guarded blob under cluster.shared_data_dir, + * and adopt it into a joiner's local pg_xact (spec-6.15b D2). + * + * The native era (pre-formation, cluster.enabled=off) is a single- + * writer window: the seed node's local pg_xact is the complete commit- + * status truth for every xid below the native high-water, including + * aborted ones. Publishing that page run with the shared tree lets a + * pre-seed-lineage joiner adopt first-hand truth instead of trusting + * hint bits (false-invisible / poison-stamp hazard; spec §0). This is + * a one-shot formation-time carry of pre-cluster history -- the same + * bytes a post-seed clone would have carried -- NOT a runtime mirror of + * cluster-era foreign commit bits (the rejected CLOG-overlay design; + * spec §1.4.1 boundary note). + * + * Both publish and adopt stream page-at-a-time (no large allocations): + * a CRC pass validates or stamps the whole image, then a second pass + * moves the bytes. Their callers run in single-writer windows (the + * seed's shutdown checkpoint; the joiner's postmaster-once bootstrap, + * strictly before StartupCLOG), so the two passes see stable bytes. + * Adopt fsyncs every touched segment and the pg_xact directory before + * returning (spec P2). + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_xid_prehistory.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-6.15b-xid-authority-native-era.md (D2, §4 U1/U4) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include + +#include "cluster/cluster_guc.h" /* cluster_shared_data_dir */ +#include "cluster/cluster_xid_authority.h" +#include "storage/fd.h" + +/* + * CLOG geometry, restated locally so the module stays standalone-linkable + * for cluster_unit: 2 status bits per xact -> 4 xacts per byte -> BLCKSZ*4 + * xacts per page (clog.c CLOG_XACTS_PER_PAGE, private to clog.c), and 32 + * pages per SLRU segment file (slru.h SLRU_PAGES_PER_SEGMENT). The unit + * truth table locks this arithmetic against drift. + */ +#define PREHISTORY_XACTS_PER_PAGE ((uint32)(BLCKSZ * 4)) +#define PREHISTORY_PAGES_PER_SEGMENT 32 + +static bool prehistory_adopted_this_boot = false; + +/* ============================================================ + * Pure layer. + * ============================================================ */ + +/* + * cluster_xid_prehistory_payload_bytes -- whole-page payload size covering + * xids [0, native_hw_full); 0 for an empty or over-cap native era (callers + * treat 0 as "nothing to carry" / refuse respectively). + */ +uint32 +cluster_xid_prehistory_payload_bytes(uint64 native_hw_full) +{ + uint64 pages; + + if (native_hw_full == 0 || native_hw_full > CLUSTER_XID_PREHISTORY_MAX_XID) + return 0; + + pages = (native_hw_full + PREHISTORY_XACTS_PER_PAGE - 1) / PREHISTORY_XACTS_PER_PAGE; + return (uint32)(pages * BLCKSZ); +} + +/* + * cluster_xid_prehistory_classify -- validate a full in-memory blob image + * (header + payload). Used by the unit truth table; the streaming adopt + * path performs the same checks incrementally. + */ +ClusterXidAuthorityValidity +cluster_xid_prehistory_classify(const char *buf, size_t len) +{ + ClusterXidPrehistoryHeader hdr; + pg_crc32c crc; + + if (buf == NULL || len < sizeof(ClusterXidPrehistoryHeader)) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + memcpy(&hdr, buf, sizeof(hdr)); + + if (hdr.magic != CLUSTER_XID_PREHISTORY_MAGIC) + return CLUSTER_XID_AUTHORITY_INVALID_MAGIC; + if (len != sizeof(hdr) + hdr.payload_len + || hdr.payload_len != cluster_xid_prehistory_payload_bytes(hdr.native_hw_full)) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, buf, offsetof(ClusterXidPrehistoryHeader, crc)); + COMP_CRC32C(crc, buf + sizeof(hdr), hdr.payload_len); + FIN_CRC32C(crc); + if (!EQ_CRC32C(crc, hdr.crc)) + return CLUSTER_XID_AUTHORITY_INVALID_CRC; + + return CLUSTER_XID_AUTHORITY_VALID; +} + +/* ============================================================ + * Streaming file plumbing. + * ============================================================ */ + +static bool +build_path(char *dst, size_t dstlen, const char *relpath) +{ + if (cluster_shared_data_dir == NULL || cluster_shared_data_dir[0] == '\0') + return false; + snprintf(dst, dstlen, "%s/%s", cluster_shared_data_dir, relpath); + return true; +} + +/* + * read_clog_page -- read local pg_xact page `pageno` into buf. Fail-closed: + * a missing or short segment is FATAL -- the caller's window (post + * CheckPointGuts / clean shutdown) guarantees every allocated page is on + * disk, so a hole would mean fabricating "aborted" truth for real xids. + */ +static void +read_clog_page(const char *local_pgdata, uint32 pageno, char *buf) +{ + char seg[MAXPGPATH]; + off_t offset; + int fd; + + snprintf(seg, sizeof(seg), "%s/pg_xact/%04X", local_pgdata, + pageno / PREHISTORY_PAGES_PER_SEGMENT); + offset = (off_t)(pageno % PREHISTORY_PAGES_PER_SEGMENT) * BLCKSZ; + + fd = OpenTransientFile(seg, O_RDONLY | PG_BINARY); + if (fd < 0) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("could not open pg_xact segment \"%s\" for prehistory publish: %m", seg), + errhint("The native-era CLOG must be complete on disk; publish runs only " + "after a CLOG flush."))); + if (lseek(fd, offset, SEEK_SET) != offset || read(fd, buf, BLCKSZ) != BLCKSZ) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("short read of pg_xact segment \"%s\" page %u for prehistory publish", seg, + pageno), + errhint("The native-era CLOG must be complete on disk; a hole would fabricate " + "commit-status truth."))); + CloseTransientFile(fd); +} + +/* + * roll_blob_to_bak -- preserve an existing primary blob into .bak (streamed + * in page-sized chunks; variable length). Missing primary is skipped. + */ +static void +roll_blob_to_bak(const char *primary, const char *bak, const char *baktmp) +{ + char chunk[BLCKSZ]; + int src; + int dst; + int r; + + src = OpenTransientFile(primary, O_RDONLY | PG_BINARY); + if (src < 0) + return; /* first write: no prior primary */ + + dst = OpenTransientFile(baktmp, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY); + if (dst < 0) { + CloseTransientFile(src); + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", baktmp))); + } + while ((r = read(src, chunk, sizeof(chunk))) > 0) { + if (write(dst, chunk, r) != r) + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", baktmp))); + } + if (r < 0) + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not read file \"%s\": %m", primary))); + CloseTransientFile(src); + if (pg_fsync(dst) != 0) + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", baktmp))); + CloseTransientFile(dst); + if (durable_rename(baktmp, bak, PANIC) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", baktmp, bak))); +} + +/* ============================================================ + * Publish (seed side). + * ============================================================ */ + +/* + * cluster_xid_prehistory_publish -- snapshot local pg_xact [0, native_hw) + * into the shared blob, torn-safe. Two passes over the source pages: one + * to compute the image CRC, one to write; the caller's single-writer + * window (seed shutdown checkpoint / catalog-seed StartupXLOG) keeps the + * bytes stable between the passes. + */ +void +cluster_xid_prehistory_publish(const char *local_pgdata, uint64 native_hw_full) +{ + ClusterXidPrehistoryHeader hdr; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char tmp[MAXPGPATH]; + char baktmp[MAXPGPATH]; + char page[BLCKSZ]; + uint32 payload_len; + uint32 pages; + uint32 p; + int fd; + + if (native_hw_full == 0 || native_hw_full > CLUSTER_XID_PREHISTORY_MAX_XID) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("native-era xid high-water " UINT64_FORMAT + " is outside the prehistory publish range", + native_hw_full), + errhint("Native-era seed loads are capped at " UINT64_FORMAT + " xids; split the seed load or move it in-protocol.", + CLUSTER_XID_PREHISTORY_MAX_XID))); + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_PREHISTORY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_PREHISTORY_BAK_REL_PATH) + || !build_path(tmp, sizeof(tmp), CLUSTER_XID_PREHISTORY_TMP_REL_PATH) + || !build_path(baktmp, sizeof(baktmp), CLUSTER_XID_PREHISTORY_BAK_TMP_REL_PATH)) + ereport(PANIC, (errmsg("cluster shared_data_dir is not configured"))); + + payload_len = cluster_xid_prehistory_payload_bytes(native_hw_full); + pages = payload_len / BLCKSZ; + + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = CLUSTER_XID_PREHISTORY_MAGIC; + hdr.version = CLUSTER_XID_PREHISTORY_VERSION; + hdr.native_hw_full = native_hw_full; + hdr.payload_len = payload_len; + hdr.reserved = 0; + + /* pass 1: CRC over header fields + source pages */ + INIT_CRC32C(hdr.crc); + COMP_CRC32C(hdr.crc, (char *)&hdr, offsetof(ClusterXidPrehistoryHeader, crc)); + for (p = 0; p < pages; p++) { + read_clog_page(local_pgdata, p, page); + COMP_CRC32C(hdr.crc, page, BLCKSZ); + } + FIN_CRC32C(hdr.crc); + + /* pass 2: stream header + pages into tmp, then install torn-safe */ + fd = OpenTransientFile(tmp, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY); + if (fd < 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", tmp))); + errno = 0; + if (write(fd, &hdr, sizeof(hdr)) != sizeof(hdr)) { + if (errno == 0) + errno = ENOSPC; + ereport(PANIC, (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", tmp))); + } + for (p = 0; p < pages; p++) { + read_clog_page(local_pgdata, p, page); + errno = 0; + if (write(fd, page, BLCKSZ) != BLCKSZ) { + if (errno == 0) + errno = ENOSPC; + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", tmp))); + } + } + if (pg_fsync(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmp))); + if (CloseTransientFile(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not close file \"%s\": %m", tmp))); + + roll_blob_to_bak(primary, bak, baktmp); + if (durable_rename(tmp, primary, PANIC) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", tmp, primary))); +} + +/* ============================================================ + * Adopt (joiner side). + * ============================================================ */ + +/* + * verify_blob -- streaming validation of one candidate blob file against + * the sealed authority's native_hw. Returns true (and leaves *out_pages + * set) only when magic/version/hw/length/CRC all hold. + */ +static bool +verify_blob(const char *path, uint64 native_hw_full, uint32 *out_pages) +{ + ClusterXidPrehistoryHeader hdr; + char page[BLCKSZ]; + pg_crc32c crc; + uint32 pages; + uint32 p; + int fd; + + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY); + if (fd < 0) + return false; + if (read(fd, &hdr, sizeof(hdr)) != sizeof(hdr)) { + CloseTransientFile(fd); + return false; + } + if (hdr.magic != CLUSTER_XID_PREHISTORY_MAGIC || hdr.version != CLUSTER_XID_PREHISTORY_VERSION + || hdr.native_hw_full != native_hw_full + || hdr.payload_len != cluster_xid_prehistory_payload_bytes(native_hw_full)) { + CloseTransientFile(fd); + return false; + } + + pages = hdr.payload_len / BLCKSZ; + INIT_CRC32C(crc); + COMP_CRC32C(crc, (char *)&hdr, offsetof(ClusterXidPrehistoryHeader, crc)); + for (p = 0; p < pages; p++) { + if (read(fd, page, BLCKSZ) != BLCKSZ) { + CloseTransientFile(fd); + return false; + } + COMP_CRC32C(crc, page, BLCKSZ); + } + FIN_CRC32C(crc); + CloseTransientFile(fd); + + if (!EQ_CRC32C(crc, hdr.crc)) + return false; + + *out_pages = pages; + return true; +} + +/* + * cluster_xid_prehistory_adopt -- decode the shared blob into local pg_xact + * page files. Validates the whole image (primary, then .bak) BEFORE the + * first local write -- a joiner must never take half a truth. Writes are + * whole-page, sequential from page 0, fsynced per touched segment, then the + * pg_xact directory is fsynced (spec P2: complete and durable strictly + * before StartupCLOG). Idempotent: re-running overwrites the same bytes. + */ +void +cluster_xid_prehistory_adopt(const char *local_pgdata, uint64 native_hw_full) +{ + ClusterXidPrehistoryHeader hdr; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char seg[MAXPGPATH]; + char dir[MAXPGPATH]; + char page[BLCKSZ]; + const char *src_path = NULL; + uint32 pages = 0; + uint32 p; + int src; + int dst = -1; + int cur_segno = -1; + int fd; + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_PREHISTORY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_PREHISTORY_BAK_REL_PATH)) + ereport(PANIC, (errmsg("cluster shared_data_dir is not configured"))); + + if (verify_blob(primary, native_hw_full, &pages)) + src_path = primary; + else if (verify_blob(bak, native_hw_full, &pages)) + src_path = bak; + else + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID prehistory is missing, corrupt, or does not match the " + "sealed authority high-water " UINT64_FORMAT, + native_hw_full), + errhint("The seed node must complete a clean native-era shutdown before " + "joiners can adopt; restore \"%s\" from the shared tree's backup " + "if it was damaged.", + CLUSTER_XID_PREHISTORY_REL_PATH))); + + /* pass 2: stream the validated pages into local pg_xact segments */ + src = OpenTransientFile(src_path, O_RDONLY | PG_BINARY); + if (src < 0 || read(src, &hdr, sizeof(hdr)) != sizeof(hdr)) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("could not re-open shared XID prehistory \"%s\": %m", src_path))); + + for (p = 0; p < pages; p++) { + int segno = (int)(p / PREHISTORY_PAGES_PER_SEGMENT); + off_t offset = (off_t)(p % PREHISTORY_PAGES_PER_SEGMENT) * BLCKSZ; + + if (read(src, page, BLCKSZ) != BLCKSZ) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("short read of shared XID prehistory \"%s\" page %u", src_path, p))); + + if (segno != cur_segno) { + if (dst >= 0) { + if (pg_fsync(dst) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not fsync pg_xact segment: %m"))); + CloseTransientFile(dst); + } + snprintf(seg, sizeof(seg), "%s/pg_xact/%04X", local_pgdata, segno); + dst = OpenTransientFile(seg, O_RDWR | O_CREAT | PG_BINARY); + if (dst < 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not open pg_xact segment \"%s\": %m", seg))); + cur_segno = segno; + } + errno = 0; + if (lseek(dst, offset, SEEK_SET) != offset || write(dst, page, BLCKSZ) != BLCKSZ) { + if (errno == 0) + errno = ENOSPC; + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not write pg_xact segment \"%s\": %m", seg))); + } + } + if (dst >= 0) { + if (pg_fsync(dst) != 0) + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not fsync pg_xact segment: %m"))); + CloseTransientFile(dst); + } + CloseTransientFile(src); + + /* P2: directory fsync makes the adopted truth durable before StartupCLOG */ + snprintf(dir, sizeof(dir), "%s/pg_xact", local_pgdata); + fd = OpenTransientFile(dir, O_RDONLY | PG_BINARY); + if (fd < 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not open pg_xact directory \"%s\": %m", dir))); + if (pg_fsync(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not fsync pg_xact directory \"%s\": %m", dir))); + CloseTransientFile(fd); + prehistory_adopted_this_boot = true; +} + +bool +cluster_xid_prehistory_was_adopted(void) +{ + return prehistory_adopted_this_boot; +} + +/* ============================================================ + * Divergent-lineage guard (review F2; spec Q6 amendment). + * ============================================================ */ + +/* + * read_local_clog_page_optional -- read local pg_xact page `pageno` into + * buf. Returns false when the segment file or the page does not exist; + * PANICs on a real read error so an I/O fault is never mistaken for a + * missing page. The caller decides what a missing page means: below + * page(oldestXid) it is a legitimately truncated frozen segment, at or + * above it is an anomaly (review r3-X2). + */ +static bool +read_local_clog_page_optional(const char *local_pgdata, uint32 pageno, char *buf) +{ + char seg[MAXPGPATH]; + off_t offset; + int fd; + int r; + + snprintf(seg, sizeof(seg), "%s/pg_xact/%04X", local_pgdata, + pageno / PREHISTORY_PAGES_PER_SEGMENT); + offset = (off_t)(pageno % PREHISTORY_PAGES_PER_SEGMENT) * BLCKSZ; + + fd = OpenTransientFile(seg, O_RDONLY | PG_BINARY); + if (fd < 0) { + if (errno == ENOENT) + return false; + ereport(PANIC, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", seg))); + } + if (lseek(fd, offset, SEEK_SET) != offset) { + CloseTransientFile(fd); + return false; + } + errno = 0; + r = read(fd, buf, BLCKSZ); + CloseTransientFile(fd); + if (r < 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not read file \"%s\": %m", seg))); + return r == BLCKSZ; +} + +/* + * clog_slot_mask -- byte mask covering the 2-bit CLOG status slots + * [lo, hi) within one byte (0 <= lo < hi <= 4). Slot n of a byte holds + * xid bits at (n * CLOG_BITS_PER_XACT), low-order first (clog.c + * TransactionIdToBIndex). + */ +static inline unsigned char +clog_slot_mask(uint32 lo, uint32 hi) +{ + unsigned int m; + + m = ((hi >= 4) ? 0xFFu : ((1u << (hi * 2)) - 1u)) & ~((1u << (lo * 2)) - 1u); + return (unsigned char)m; +} + +/* + * clog_page_range_equal -- compare blob vs local CLOG page content for the + * page-local 2-bit slots [from_xact, to_xact) only. Sub-byte boundaries + * are masked so bits outside the range never influence the verdict. + */ +static bool +clog_page_range_equal(const char *blob_page, const char *local_page, uint32 from_xact, + uint32 to_xact) +{ + uint32 first_byte = from_xact / 4; + uint32 last_byte = (to_xact - 1) / 4; + + Assert(from_xact < to_xact && to_xact <= PREHISTORY_XACTS_PER_PAGE); + + if (first_byte == last_byte) + return ((unsigned char)(blob_page[first_byte] ^ local_page[first_byte]) + & clog_slot_mask(from_xact % 4, to_xact - first_byte * 4)) + == 0; + + if (from_xact % 4 != 0) { + if (((unsigned char)(blob_page[first_byte] ^ local_page[first_byte]) + & clog_slot_mask(from_xact % 4, 4)) + != 0) + return false; + first_byte++; + } + if (to_xact % 4 != 0) { + if (((unsigned char)(blob_page[last_byte] ^ local_page[last_byte]) + & clog_slot_mask(0, to_xact % 4)) + != 0) + return false; + last_byte--; + } + return first_byte > last_byte + || memcmp(blob_page + first_byte, local_page + first_byte, last_byte - first_byte + 1) + == 0; +} + +/* + * cluster_xid_prehistory_prefix_check -- see header. Streams the sealed + * blob (primary, then .bak) and byte-compares the local pg_xact prefix + * covering xids [oldest_xid_full, min(limit_xid_full, native_hw_full)) at + * per-xact (2-bit) precision. + * + * Frozen-prefix exemption (review r3-X2): SimpleLruTruncate removes whole + * pg_xact SEGMENTS once every page they cover precedes page(oldestXid) + * (slru.c SlruMayDeleteSegment), so a joiner that froze past >=1 segment + * legitimately has no file where the blob still has pages -- the old + * break-on-first-missing-local-page treated exactly that shape as a + * tail-short clone and returned CONSISTENT without ever comparing the + * surviving pages. Neither truncation nor tuple freezing rewrites the + * CLOG bytes that survive, but bits below oldestXid are dead content + * anyway (every tuple with such an xmin/xmax is frozen, so CLOG is never + * consulted for them again) and whether they still exist is an accident + * of truncation timing; lineage evidence therefore starts at oldestXid's + * own 2-bit slot, and everything below it is skipped whether the page + * survives or not. Within the comparable range [oldestXid, limit) a + * MISSING local page is an anomaly -- pg_xact always covers + * [page(oldestXid), page(nextXid)] on a well-formed node -- and returns + * UNAVAILABLE (fail-closed), never CONSISTENT. + */ +ClusterXidPrefixVerdict +cluster_xid_prehistory_prefix_check(const char *local_pgdata, uint64 native_hw_full, + uint64 limit_xid_full, uint64 oldest_xid_full) +{ + ClusterXidPrehistoryHeader hdr; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char blob_page[BLCKSZ]; + char local_page[BLCKSZ]; + const char *src_path; + uint32 pages = 0; + uint32 start_page; + uint32 p; + uint64 limit; + uint64 oldest; + int src; + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_PREHISTORY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_PREHISTORY_BAK_REL_PATH)) + return CLUSTER_XID_PREFIX_UNAVAILABLE; + + if (verify_blob(primary, native_hw_full, &pages)) + src_path = primary; + else if (verify_blob(bak, native_hw_full, &pages)) + src_path = bak; + else + return CLUSTER_XID_PREFIX_UNAVAILABLE; + + limit = Min(limit_xid_full, native_hw_full); + if (limit == 0) + return CLUSTER_XID_PREFIX_CONSISTENT; + + oldest = Min(oldest_xid_full, limit); + if (oldest >= limit) + return CLUSTER_XID_PREFIX_CONSISTENT; /* everything comparable is frozen */ + start_page = (uint32)(oldest / PREHISTORY_XACTS_PER_PAGE); + + src = OpenTransientFile(src_path, O_RDONLY | PG_BINARY); + if (src < 0 || read(src, &hdr, sizeof(hdr)) != sizeof(hdr)) { + if (src >= 0) + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_UNAVAILABLE; + } + + for (p = 0; p < pages; p++) { + uint64 page_first_xid = (uint64)p * PREHISTORY_XACTS_PER_PAGE; + uint64 cmp_from; + uint64 cmp_to; + + if (read(src, blob_page, BLCKSZ) != BLCKSZ) { + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_UNAVAILABLE; + } + if (page_first_xid >= limit) + break; /* comparison scope ended on a page boundary */ + if (p < start_page) + continue; /* wholly frozen page: skipped, present or not */ + + cmp_from = Max(page_first_xid, oldest); + cmp_to = Min(page_first_xid + PREHISTORY_XACTS_PER_PAGE, limit); + /* oldest < limit and p >= start_page guarantee a non-empty range */ + Assert(cmp_from < cmp_to); + + if (!read_local_clog_page_optional(local_pgdata, p, local_page)) { + /* hole inside the comparable range: fail closed (r3-X2) */ + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_UNAVAILABLE; + } + + if (!clog_page_range_equal(blob_page, local_page, (uint32)(cmp_from - page_first_xid), + (uint32)(cmp_to - page_first_xid))) { + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_DIVERGED; + } + } + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_CONSISTENT; +} diff --git a/src/backend/utils/errcodes.txt b/src/backend/utils/errcodes.txt index be8af8e758..ac8fe1c784 100644 --- a/src/backend/utils/errcodes.txt +++ b/src/backend/utils/errcodes.txt @@ -875,6 +875,17 @@ Section: Class 53 - Insufficient Resources (pgrac extension) # the first own checkpoint). 53RB3 E ERRCODE_CLUSTER_RECOVERY_ANCHOR_UNAVAILABLE cluster_recovery_anchor_unavailable +# spec-6.15b: shared XID authority / native-era prehistory unavailable. +# Under cluster.shared_catalog=on a joiner must reconcile its transaction +# horizon against the sealed shared XID authority and adopt the native-era +# prehistory (pg_xact truth for pre-formation seed xids); a missing, +# unsealed or corrupt authority/prehistory fails closed rather than judge +# seed rows through a stale horizon or a truthless local CLOG. Also raised +# when a cluster.enabled=off boot tries to re-enter the native seed era on +# a formed cluster's shared tree. (53RB4 is claimed by the spec-7.1 lane's +# ERRCODE_CLUSTER_MXID_HALFSPACE_LIMIT; do not reuse.) +53RB5 E ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE cluster_xid_authority_unavailable + Section: Class 54 - Program Limit Exceeded # this is for wired-in limits, not resource exhaustion problems (class borrowed from DB2) diff --git a/src/include/cluster/cluster_catalog_migrate.h b/src/include/cluster/cluster_catalog_migrate.h index aa17cae1cb..18e3c640ec 100644 --- a/src/include/cluster/cluster_catalog_migrate.h +++ b/src/include/cluster/cluster_catalog_migrate.h @@ -61,6 +61,11 @@ typedef struct ClusterCatalogAuthorityMarker { pg_crc32c crc; /* over [0, offsetof(crc)) */ } ClusterCatalogAuthorityMarker; +typedef enum ClusterCatalogMigrateResult { + CLUSTER_CATALOG_MIGRATE_SEEDED = 0, + CLUSTER_CATALOG_MIGRATE_ADOPTED +} ClusterCatalogMigrateResult; + /* * cluster_catalog_migrate_tree -- establish the shared catalog relation tree * for this node. Postmaster-once, only meaningful when @@ -74,7 +79,8 @@ typedef struct ClusterCatalogAuthorityMarker { * Fail-closed: FATAL 53RB0 on a foreign/mismatched or unreadable authority. * system_identifier is the cluster-wide value from the shared pg_control. */ -extern void cluster_catalog_migrate_tree(const char *local_pgdata, uint64 system_identifier); +extern ClusterCatalogMigrateResult cluster_catalog_migrate_tree(const char *local_pgdata, + uint64 system_identifier); /* * cluster_catalog_vet_no_unlogged -- spec-6.14 Q12 enable-time vet: FATAL diff --git a/src/include/cluster/cluster_clean_leave.h b/src/include/cluster/cluster_clean_leave.h index e49ecfc194..6a4808f793 100644 --- a/src/include/cluster/cluster_clean_leave.h +++ b/src/include/cluster/cluster_clean_leave.h @@ -65,6 +65,8 @@ #include "port/atomics.h" #include "storage/lwlock.h" +#include "cluster/cluster_marker_async.h" + /* 128 nodes, same width as ReconfigEvent.dead_bitmap. */ #define CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES 16 /* Default drain deadline; aligns with the feature-082 30s barrier. */ @@ -149,11 +151,17 @@ typedef struct ClusterLeaveState { int32 leaving_node_id; /* -1 if none */ uint32 _pad0; uint64 leave_epoch; /* epoch this leave is bound to (CL-I3 immutable) */ - uint64 leave_baseline_dead_gen; /* CSSD dead_generation when the leave was bound; - * an unchanged value at the epoch bump means OUR - * clean-leave committed (no real death intruded) */ + uint64 leave_baseline_dead_gen; /* CSSD dead_generation when the leave was bound + * (retained for observability; the coherence gate + * now uses leave_baseline_others_dead, spec-2.29a + * ②b — the scalar counted the leaving node's own + * expected DEAD and falsely escalated) */ uint64 barrier_deadline_us; /* fail-closed deadline (drain_timeout_ms) */ uint8 ack_bitmap[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; /* survivor acks */ + /* spec-2.29a ②b: CSSD dead set (EXCLUDING the leaving node) snapshotted when + * the leave was bound; the coherence gate escalates only if a THIRD-PARTY + * death changed this set mid-drain. */ + uint8 leave_baseline_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; pg_atomic_uint64 ges_drained_count; /* shards/grants drained (observability) */ pg_atomic_uint64 gcs_flushed_count; /* dirty/X pages flushed */ pg_atomic_uint64 shards_remastered; /* shards moved off leaving node */ @@ -198,6 +206,14 @@ typedef struct ClusterLeaveState { pg_atomic_uint32 commit_point_observed; pg_atomic_uint32 committed_durable_confirmed; pg_atomic_uint32 committed_marker_durable; + /* spec-2.29a r3 (evidence latch): the committed epoch E the coordinator's + * LEAVE_COMMITTED attested (the epoch stamped into the majority-durable + * COMMITTED marker). Written by the leaving node's confirmation handler + * under `lock` BEFORE committed_durable_confirmed is set; consumed once by + * the barrier tick when the evidence latches (so the leaver records the + * exact committed epoch instead of inferring it from its possibly-stale + * local epoch view). Guarded by `lock`. */ + uint64 committed_confirmed_epoch; /* * Hardening v1.0.2 fields. @@ -335,10 +351,33 @@ extern bool cluster_clean_leave_should_abort_writable(bool in_transaction, bool /* version-coherent leave (U3 / CL-I3 / L235): the leave is still coherent only * if neither the cluster epoch nor the CSSD dead_generation moved since the - * leave was bound; any external bump (a real death intruding) => not coherent - * => the caller must ABORTED_ESCALATE. */ + * leave was bound; any external membership change (a third-party death, or a + * third-party fail-stop that already bumped the epoch) => not coherent => the + * caller must ABORTED_ESCALATE. spec-2.29a ②b: the dead set compared here + * EXCLUDES the leaving node itself (others-dead bitmap), so the leaving node's + * own expected alive→DEAD transition never falsely escalates the leave. */ extern bool cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 current_epoch, - uint64 bound_dead_gen, uint64 current_dead_gen); + const uint8 *bound_others_dead, + const uint8 *current_others_dead, int nbytes); + +/* spec-2.29a r3: the leaving node's barrier-tick own-commit latch — evidence + * over inference. Latches iff the durable COMMITTED marker for THIS leave + * attempt was confirmed (nonce-bound LEAVE_COMMITTED attestation); the two + * coherence observations are contract inputs the verdict must ignore (a + * third-party transient flap must neither refuse an evidenced latch — the + * t/331 C1/C4 false-escalation — nor latch anything without evidence — the + * r2 P2-1 refused-leave mis-latch wedge). */ +extern bool cluster_clean_leave_own_commit_latched(bool committed_marker_evidence, + bool others_dead_unchanged, + bool dead_gen_unchanged); + +/* spec-2.29a r3: identity gate for a LEAVE_COMMITTED confirmation — accept it + * as marker evidence only for THIS node's CURRENT leave attempt (self- + * addressed + currently leaving + per-attempt nonce match + committed epoch + * past the bound baseline); fail-closed on any mismatch. */ +extern bool cluster_clean_leave_committed_evidence_matches( + int32 payload_leaving_node, uint64 payload_nonce, uint64 payload_epoch, int32 self_node, + int32 current_leaving_node, uint64 current_attempt_nonce, uint64 bound_leave_epoch); /* leave-intent marker structural validation (magic/version/CRC/identity). Pure: * computes CRC32C over [magic..phase] and checks magic, version, that the @@ -408,6 +447,14 @@ extern void cluster_clean_leave_shmem_register(void); * ------------------------------------------------------------------ */ extern ClusterLeaveMarkerSubmitResult cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m); +extern bool cluster_clean_leave_submit_marker_async(ClusterMarkerAsync *a, + const ClusterLeaveIntentMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now); +extern ClusterMarkerPollResult cluster_clean_leave_poll_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern bool cluster_clean_leave_qvotec_poll_pending(void *out_slot512); extern void cluster_clean_leave_qvotec_complete(bool acked); extern void cluster_clean_leave_publish_qvotec_latch(struct Latch *latch); diff --git a/src/include/cluster/cluster_grd.h b/src/include/cluster/cluster_grd.h index 7015600fec..0c30376374 100644 --- a/src/include/cluster/cluster_grd.h +++ b/src/include/cluster/cluster_grd.h @@ -275,6 +275,8 @@ typedef struct ClusterGrdShared { pg_atomic_uint64 recovery_event_old_epoch; pg_atomic_uint64 recovery_redeclare_generation; pg_atomic_uint64 recovery_barrier_deadline; + pg_atomic_uint32 recovery_event_coordinator; + pg_atomic_uint64 recovery_done_epoch_at_accept; /* * spec-4.6 P0-1 (Fable review) — the epoch this episode is LOCKED to. @@ -292,8 +294,21 @@ typedef struct ClusterGrdShared { /* spec-4.6 P0#3 cluster gate — per-node epoch for which that node * last announced "local rebind barrier complete" (REDECLARE_DONE). - * P6 requires done_epoch[s] >= current epoch for EVERY survivor. */ + * P6 requires done_epoch[s] >= current epoch for EVERY survivor. + * + * spec-4.6a Amendment v1.2 (R2): the cross-node convergence key is the + * COMPOSITE (episode_epoch, dead_bitmap_hash). event_id is NOT globally + * consistent (it folds the per-instance cssd_dead_generation, which + * drifts with each node's own flap observation history), so keying the + * P6 gate on it wedges the cluster whenever two survivors observed a + * different flap history. The quorum-accepted dead SET is what actually + * converges; its hash rides the DONE payload instead. event_id stays a + * purely LOCAL ABA scope (P0 accept dedup). */ pg_atomic_uint64 recovery_done_epoch[CLUSTER_MAX_NODES]; + pg_atomic_uint64 recovery_done_bitmap_hash[CLUSTER_MAX_NODES]; + /* hash of THIS episode's accepted dead_bitmap (LMON writes at P0 accept; + * empty-bitmap hash for JOIN episodes, identical on every node). */ + pg_atomic_uint64 recovery_event_bitmap_hash; /* spec-4.6 D5 — 13 grd_recovery counters (dump category * 'grd_recovery'; each has a t/249 leg). Incremented along @@ -311,6 +326,10 @@ typedef struct ClusterGrdShared { pg_atomic_uint64 block_path_failclosed_count; /* D4 GCS/PCM 53R9K (L12) */ pg_atomic_uint64 unaffected_holder_survived_count; /* L13 sweep-scope guard */ pg_atomic_uint64 stale_holder_swept_count; /* P6 post-barrier sweep (L15) */ + pg_atomic_uint64 cluster_gate_timeout_count; /* WAIT_CLUSTER watchdog only */ + pg_atomic_uint64 wait_epoch_escape_count; /* proof-carrying equal-epoch advance */ + pg_atomic_uint64 + pcm_dead_cleanup_entries; /* spec-4.6a D12: dead-node PCM holder/pending-X records cleaned */ /* spec-5.1b D9 — convert state-machine observability counters. * convert_queue_full reuses the existing converts_full_count above; @@ -609,6 +628,21 @@ typedef enum ClusterGrdRecoveryState { } ClusterGrdRecoveryState; extern void cluster_grd_recovery_lmon_tick(void); +/* spec-4.6a D5/D6 — recovery state observability for pg_cluster_state. */ +extern uint32 cluster_grd_recovery_state_value(void); +extern const char *cluster_grd_recovery_state_name(uint32 state); +extern uint64 cluster_grd_recovery_last_event_id(void); +extern uint64 cluster_grd_recovery_episode_epoch_value(void); +extern uint32 cluster_grd_recovery_event_coordinator(void); +extern uint64 cluster_grd_recovery_done_epoch_for(int32 node); +extern uint64 cluster_grd_recovery_done_bitmap_hash_for(int32 node); +extern uint64 cluster_grd_recovery_event_bitmap_hash_value(void); +/* Amendment v1.2 (R2): the cross-node DONE key — hash over the dead bitmap + * ALONE (no dead_generation fold; same kernel as the event_id hash). */ +extern uint64 cluster_grd_dead_bitmap_hash(const uint8 *dead_bitmap); +extern int cluster_grd_recovery_block_redeclare_cursor(void); +extern uint64 cluster_grd_recovery_block_redeclare_epoch(void); +extern bool cluster_grd_recovery_block_redeclare_done(void); extern uint64 cluster_grd_redeclare_generation(void); /* spec-4.6 P0-1 — the epoch the current episode is locked to (0 = none). */ extern uint64 cluster_grd_redeclare_episode_epoch(void); @@ -625,8 +659,10 @@ extern bool grd_block_redeclare_scan_complete(uint64 episode_epoch); /* spec-4.6 P0#3 cluster gate — REDECLARE_DONE receiver (cluster_ges.c * inbound handler): record that `node` completed its local rebind - * barrier for `epoch`. */ -extern void cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch); + * barrier for `epoch`. Amendment v1.2 (R2): the third argument is the + * sender's dead_bitmap hash (the cross-node half of the composite + * convergence key), NOT the sender-local event_id. */ +extern void cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap_hash); /* spec-4.6 D4/D5 — recovery counter bumps for out-of-module call sites. */ extern void cluster_grd_inc_stale_request_drop(void); @@ -650,6 +686,9 @@ typedef struct ClusterGrdRecoveryCounters { uint64 block_path_failclosed; uint64 unaffected_holder_survived; uint64 stale_holder_swept; + uint64 cluster_gate_timeout; + uint64 wait_epoch_escape; + uint64 pcm_dead_cleanup_entries; /* spec-5.16 D5 — join-direction remaster counters (distinct from the * failure-driven remaster_* above; §8 Q6-A). */ uint64 join_remaster_started; @@ -660,6 +699,7 @@ typedef struct ClusterGrdRecoveryCounters { } ClusterGrdRecoveryCounters; extern void cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out); +extern uint64 cluster_grd_recovery_event_old_epoch(void); /* spec-2.29a WAIT_EPOCH baseline */ /* spec-4.6 P0#2 — pre-remaster stale-epoch sweep SCOPED to the affected * (dead-master) shards; affected_shards is a PGRAC_GRD_SHARD_COUNT-bit diff --git a/src/include/cluster/cluster_guc.h b/src/include/cluster/cluster_guc.h index e4565731eb..469ffd0619 100644 --- a/src/include/cluster/cluster_guc.h +++ b/src/include/cluster/cluster_guc.h @@ -399,6 +399,8 @@ extern int cluster_ges_reply_wait_max_entries; * 5000ms; SIGHUP). */ extern int cluster_grd_remaster_wait_ms; extern int cluster_grd_rebuild_timeout_ms; +extern int cluster_hw_remaster_retry_backoff_ms; +extern int cluster_hw_remaster_retry_max_attempts; /* spec-5.4 D8: SQ sequence lock tunables. */ extern int cluster_sequence_default_cache; @@ -469,6 +471,7 @@ extern int cluster_phase4_timeout; * range: [100, 60000] (millisecond) */ extern int cluster_lmon_main_loop_interval; +extern int cluster_lmon_slow_iteration_warn_ms; /* diff --git a/src/include/cluster/cluster_hw.h b/src/include/cluster/cluster_hw.h index 89656a384f..e021c1c445 100644 --- a/src/include/cluster/cluster_hw.h +++ b/src/include/cluster/cluster_hw.h @@ -312,12 +312,30 @@ extern uint32 cluster_hw_shard_rebuilt_generation(uint32 shard_id); extern void cluster_hw_mark_shard_rebuilt(uint32 shard_id, uint32 generation); /* - * Per-dead-origin online-remaster launch idempotency (S5d). The GRD FSM records - * the episode it last launched a rebuild worker for a dead origin under, and - * skips relaunching while the value equals the current episode. + * Per-dead-origin online-remaster launch state (S5d / spec-4.6a). The GRD FSM + * records the episode it last launched a rebuild worker for a dead origin under, + * and the worker records a terminal result. BLOCKED is retryable within the same + * episode; BLOCKED_STRUCTURAL is not. */ +typedef enum ClusterHwRemasterResult { + CLUSTER_HW_REMASTER_NONE = 0, + CLUSTER_HW_REMASTER_RUNNING, + CLUSTER_HW_REMASTER_DONE, + CLUSTER_HW_REMASTER_BLOCKED, + CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL, + CLUSTER_HW_REMASTER_NOT_APPLICABLE, +} ClusterHwRemasterResult; + extern uint64 cluster_hw_remaster_launched_episode(int node_id); extern void cluster_hw_remaster_set_launched(int node_id, uint64 episode); +extern ClusterHwRemasterResult cluster_hw_remaster_result(int node_id); +extern void cluster_hw_remaster_set_result(int node_id, ClusterHwRemasterResult result); +extern uint32 cluster_hw_remaster_attempts(int node_id); +extern void cluster_hw_remaster_set_attempts(int node_id, uint32 attempts); +extern uint64 cluster_hw_remaster_next_attempt_at(int node_id); +extern void cluster_hw_remaster_set_next_attempt_at(int node_id, uint64 ts); +extern const char *cluster_hw_remaster_result_name(ClusterHwRemasterResult result); +extern bool cluster_hw_remaster_recoverable(void); /* * cluster_hw_apply_hwm -- redo / remaster rebuild: create-if-absent and raise @@ -383,6 +401,8 @@ extern uint64 cluster_hw_failclosed_count(void); extern uint64 cluster_hw_not_ready_count(void); extern uint64 cluster_hw_remaster_done_count(void); extern uint64 cluster_hw_remaster_blocked_count(void); +extern uint64 cluster_hw_remaster_retry_count(void); +extern uint64 cluster_hw_remaster_retry_exhausted_count(void); extern void cluster_hw_bump_alloc(void); extern void cluster_hw_bump_authority_create(void); extern void cluster_hw_bump_reserve_wal(void); @@ -391,6 +411,8 @@ extern void cluster_hw_bump_failclosed(void); extern void cluster_hw_bump_not_ready(void); extern void cluster_hw_bump_remaster_done(void); extern void cluster_hw_bump_remaster_blocked(void); +extern void cluster_hw_bump_remaster_retry(void); +extern void cluster_hw_bump_remaster_retry_exhausted(void); #endif /* !FRONTEND */ diff --git a/src/include/cluster/cluster_hw_remaster.h b/src/include/cluster/cluster_hw_remaster.h index e1bf3ba529..e2fcc481f8 100644 --- a/src/include/cluster/cluster_hw_remaster.h +++ b/src/include/cluster/cluster_hw_remaster.h @@ -47,17 +47,91 @@ #ifndef CLUSTER_HW_REMASTER_H #define CLUSTER_HW_REMASTER_H -/* - * Result of an online-remaster HW authority rebuild for one dead origin. Only - * DONE marks the adopted shards rebuilt (opens the serve gate); BLOCKED and - * NOT_APPLICABLE never do, so a shard whose HWM is not provably rebuilt stays - * fail-closed (8.A). - */ -typedef enum ClusterHwRemasterResult { - CLUSTER_HW_REMASTER_DONE = 0, /* rebuilt + adoption snapshot durable + shards marked */ - CLUSTER_HW_REMASTER_BLOCKED, /* fail-closed: keep shards frozen (53RA6) */ - CLUSTER_HW_REMASTER_NOT_APPLICABLE, /* HW authority inactive / no adopted shard / bad input */ -} ClusterHwRemasterResult; +#include "cluster/cluster_hw.h" /* ClusterHwRemasterResult */ + +#define CLUSTER_HW_REMASTER_NO_DEADLINE PG_UINT64_MAX +#define CLUSTER_HW_REMASTER_BACKOFF_CAP_MS 60000 + +typedef enum ClusterHwRemasterLaunchAction { + CLUSTER_HW_REMASTER_LAUNCH_SKIP = 0, + CLUSTER_HW_REMASTER_LAUNCH_INITIAL, + CLUSTER_HW_REMASTER_LAUNCH_RETRY, + CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED, + CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL, +} ClusterHwRemasterLaunchAction; + +typedef struct ClusterHwRemasterRelaunchDecision { + ClusterHwRemasterLaunchAction action; + uint32 next_attempts; + uint64 next_attempt_at; + ClusterHwRemasterResult next_result; +} ClusterHwRemasterRelaunchDecision; + +static inline uint32 +cluster_hw_remaster_compute_backoff_ms(int base_ms, uint32 completed_retry_attempts) +{ + uint64 ms; + uint32 shift; + + if (base_ms < 1) + base_ms = 1; + ms = (uint64)base_ms; + shift = completed_retry_attempts > 0 ? completed_retry_attempts - 1 : 0; + while (shift-- > 0 && ms < CLUSTER_HW_REMASTER_BACKOFF_CAP_MS) + ms <<= 1; + if (ms > CLUSTER_HW_REMASTER_BACKOFF_CAP_MS) + ms = CLUSTER_HW_REMASTER_BACKOFF_CAP_MS; + return (uint32)ms; +} + +static inline ClusterHwRemasterRelaunchDecision +cluster_hw_remaster_relaunch_decide(uint64 launched_episode, uint64 current_episode, + ClusterHwRemasterResult result, uint32 attempts, + uint64 next_attempt_at, uint64 now, int retry_max_attempts) +{ + ClusterHwRemasterRelaunchDecision d; + + d.action = CLUSTER_HW_REMASTER_LAUNCH_SKIP; + d.next_attempts = attempts; + d.next_attempt_at = next_attempt_at; + d.next_result = result; + + if (current_episode == 0) + return d; + if (launched_episode != current_episode || result == CLUSTER_HW_REMASTER_NONE) { + d.action = CLUSTER_HW_REMASTER_LAUNCH_INITIAL; + d.next_attempts = 0; + d.next_attempt_at = 0; + d.next_result = CLUSTER_HW_REMASTER_RUNNING; + return d; + } + + if (result == CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL) { + if (next_attempt_at != CLUSTER_HW_REMASTER_NO_DEADLINE) { + d.action = CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL; + d.next_attempt_at = CLUSTER_HW_REMASTER_NO_DEADLINE; + } + return d; + } + if (result != CLUSTER_HW_REMASTER_BLOCKED) + return d; + + if (retry_max_attempts <= 0 || attempts >= (uint32)retry_max_attempts) { + if (next_attempt_at != CLUSTER_HW_REMASTER_NO_DEADLINE) { + d.action = CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED; + d.next_attempt_at = CLUSTER_HW_REMASTER_NO_DEADLINE; + } + return d; + } + if (next_attempt_at != CLUSTER_HW_REMASTER_NO_DEADLINE && now < next_attempt_at) + return d; + + d.action = CLUSTER_HW_REMASTER_LAUNCH_RETRY; + d.next_attempts = attempts + 1; + d.next_attempt_at = 0; + d.next_result = CLUSTER_HW_REMASTER_RUNNING; + return d; +} #ifndef FRONTEND @@ -81,9 +155,9 @@ extern ClusterHwRemasterResult cluster_hw_remaster_rebuild_origin(int dead_node_ /* * cluster_hw_remaster_worker_main -- dynamic-bgworker entry point (main_arg = the * dead origin node id). Captures the live reconfig episode and drives - * cluster_hw_remaster_rebuild_origin off the LMON tick. An abnormal exit simply - * leaves the shards unmarked -> the serve gate keeps them fail-closed (8.A), so - * no abnormal-exit handler is needed. + * cluster_hw_remaster_rebuild_origin off the LMON tick. A before_shmem_exit + * callback records an abnormal exit as retryable BLOCKED so the same episode can + * self-heal instead of staying pinned on the launch idempotency bit. */ extern void cluster_hw_remaster_worker_main(Datum main_arg); diff --git a/src/include/cluster/cluster_lmon.h b/src/include/cluster/cluster_lmon.h index 63efc4e045..e1099ece40 100644 --- a/src/include/cluster/cluster_lmon.h +++ b/src/include/cluster/cluster_lmon.h @@ -100,6 +100,7 @@ #include "datatype/timestamp.h" #include "storage/lwlock.h" +struct Latch; /* * ClusterLmonStatus -- HC2 SSOT for LMON lifecycle state. @@ -137,6 +138,10 @@ typedef struct ClusterLmonSharedState { TimestampTz ready_at; /* set by LMON in CLUSTER_LMON_READY */ TimestampTz last_liveness_tick_at; /* HC6: local liveness tick — NOT inter-node heartbeat */ int64 main_loop_iters; /* monotone counter; observable proof of liveness */ + uint64 last_iter_us; /* most recent main-loop iteration wall time */ + uint64 max_iter_us; /* high-water iteration wall time */ + uint64 slow_iter_count; /* iterations over cluster.lmon_slow_iteration_warn_ms */ + struct Latch *lmon_latch; /* qvotec completion wake target; LMON owns lifecycle */ bool shutdown_requested; /* postmaster sets; LMON main loop polls + exits */ } ClusterLmonSharedState; @@ -188,6 +193,10 @@ extern TimestampTz cluster_lmon_spawned_at(void); extern TimestampTz cluster_lmon_ready_at(void); extern TimestampTz cluster_lmon_last_liveness_tick_at(void); extern int64 cluster_lmon_main_loop_iters(void); +extern uint64 cluster_lmon_last_iter_us(void); +extern uint64 cluster_lmon_max_iter_us(void); +extern uint64 cluster_lmon_slow_iter_count(void); +extern void cluster_lmon_marker_complete_wakeup(void); /* * Status enum -> canonical lowercase string ("not_started", "spawning", diff --git a/src/include/cluster/cluster_marker_async.h b/src/include/cluster/cluster_marker_async.h new file mode 100644 index 0000000000..56cd836bf5 --- /dev/null +++ b/src/include/cluster/cluster_marker_async.h @@ -0,0 +1,218 @@ +/*------------------------------------------------------------------------- + * + * cluster_marker_async.h + * Small process-local async FSM for qvotec marker submit mailboxes. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/include/cluster/cluster_marker_async.h + * + * NOTES + * The mailbox remains the existing request_seq/completion_seq/result + * slot. This wrapper changes only the waiting shape: submit publishes + * a request and returns; the owning LMON tick polls completion without + * sleeping (spec-2.29a — the pre-async 2ms pg_usleep spin inside the + * LMON tick starved the CSSD heartbeat relay and caused false-DEAD + * storms during cold formation, BUG-C1). + * + * The staged has_staged_event flag is the P1-1 bump-once contract: a + * pre-bump caller (fail-stop fence / node-remove / join Phase-1) must + * not re-enter its epoch-bump path while a stage is live. + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_MARKER_ASYNC_H +#define CLUSTER_MARKER_ASYNC_H + +#include "c.h" + +#include + +#include "datatype/timestamp.h" +#include "port/atomics.h" +#include "storage/latch.h" + +typedef enum ClusterMarkerAsyncState { + CLUSTER_MARKER_ASYNC_IDLE = 0, + CLUSTER_MARKER_ASYNC_SUBMITTED +} ClusterMarkerAsyncState; + +typedef enum ClusterMarkerPollResult { + CLUSTER_MARKER_POLL_IDLE = 0, + CLUSTER_MARKER_POLL_PENDING, + CLUSTER_MARKER_POLL_ACKED, + CLUSTER_MARKER_POLL_TIMEOUT +} ClusterMarkerPollResult; + +typedef enum ClusterMarkerAsyncKind { + CLUSTER_MARKER_KIND_UNKNOWN = 0, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + CLUSTER_MARKER_KIND_JOIN_PREPARE, + CLUSTER_MARKER_KIND_JOIN_COMMITTED, + CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTING, + CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED, + CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVING, + CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK, + CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED +} ClusterMarkerAsyncKind; + +typedef struct ClusterMarkerAsync { + ClusterMarkerAsyncState state; + uint64 inflight_seq; + uint64 deadline_us; + TimestampTz submitted_at; + ClusterMarkerAsyncKind kind; + int32 target_node; + + /* + * Caller-owned staged publish/commit state. While true, the owner must not + * re-enter the pre-bump path; it either submits/polls this record, publishes + * after ACK, or releases it after failure/timeout. + */ + bool has_staged_event; + uint64 staged_expect_epoch; +} ClusterMarkerAsync; + +static inline const char * +cluster_marker_async_kind_name(ClusterMarkerAsyncKind kind) +{ + switch (kind) { + case CLUSTER_MARKER_KIND_FENCE_FAILSTOP: + return "fence_failstop"; + case CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED: + return "fence_node_removed"; + case CLUSTER_MARKER_KIND_JOIN_PREPARE: + return "join_prepare"; + case CLUSTER_MARKER_KIND_JOIN_COMMITTED: + return "join_committed"; + case CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTING: + return "clean_leave_committing"; + case CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED: + return "clean_leave_committed"; + case CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVING: + return "node_remove_removing"; + case CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK: + return "node_remove_shrunk"; + case CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED: + return "node_remove_removed"; + case CLUSTER_MARKER_KIND_UNKNOWN: + default: + return "unknown"; + } +} + +static inline void +cluster_marker_async_init(ClusterMarkerAsync *a) +{ + memset(a, 0, sizeof(*a)); + a->state = CLUSTER_MARKER_ASYNC_IDLE; + a->target_node = -1; +} + +static inline bool +cluster_marker_async_is_submitted(const ClusterMarkerAsync *a) +{ + return a != NULL && a->state == CLUSTER_MARKER_ASYNC_SUBMITTED; +} + +static inline bool +cluster_marker_async_mailbox_busy(pg_atomic_uint64 *request_seq, pg_atomic_uint64 *completion_seq) +{ + return pg_atomic_read_u64(request_seq) != pg_atomic_read_u64(completion_seq); +} + +static inline bool +cluster_marker_async_submit(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq, + pg_atomic_uint64 *completion_seq, struct Latch *qvotec_latch, + TimestampTz now, uint64 timeout_us, ClusterMarkerAsyncKind kind, + int32 target_node) +{ + uint64 seq; + + Assert(a != NULL); + Assert(request_seq != NULL); + Assert(completion_seq != NULL); + + if (a->state == CLUSTER_MARKER_ASYNC_SUBMITTED) + return true; + if (cluster_marker_async_mailbox_busy(request_seq, completion_seq)) + return false; + + pg_write_barrier(); + seq = pg_atomic_add_fetch_u64(request_seq, 1); + if (qvotec_latch != NULL) + SetLatch(qvotec_latch); + + a->state = CLUSTER_MARKER_ASYNC_SUBMITTED; + a->inflight_seq = seq; + a->deadline_us = (uint64)now + timeout_us; + a->submitted_at = now; + a->kind = kind; + a->target_node = target_node; + return true; +} + +static inline ClusterMarkerPollResult +cluster_marker_async_poll(ClusterMarkerAsync *a, pg_atomic_uint64 *completion_seq, + pg_atomic_uint32 *result_slot, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) +{ + uint64 elapsed; + + Assert(a != NULL); + Assert(completion_seq != NULL); + Assert(result_slot != NULL); + + if (out_result != NULL) + *out_result = 0; + if (out_elapsed_us != NULL) + *out_elapsed_us = 0; + if (a->state != CLUSTER_MARKER_ASYNC_SUBMITTED) + return CLUSTER_MARKER_POLL_IDLE; + + if (pg_atomic_read_u64(completion_seq) == a->inflight_seq) { + pg_read_barrier(); + if (out_result != NULL) + *out_result = pg_atomic_read_u32(result_slot); + elapsed + = ((uint64)now > (uint64)a->submitted_at) ? ((uint64)now - (uint64)a->submitted_at) : 0; + if (out_elapsed_us != NULL) + *out_elapsed_us = elapsed; + a->state = CLUSTER_MARKER_ASYNC_IDLE; + return CLUSTER_MARKER_POLL_ACKED; + } + + if ((uint64)now >= a->deadline_us) { + elapsed + = ((uint64)now > (uint64)a->submitted_at) ? ((uint64)now - (uint64)a->submitted_at) : 0; + if (out_elapsed_us != NULL) + *out_elapsed_us = elapsed; + a->state = CLUSTER_MARKER_ASYNC_IDLE; + return CLUSTER_MARKER_POLL_TIMEOUT; + } + + return CLUSTER_MARKER_POLL_PENDING; +} + +static inline void +cluster_marker_async_release_stage(ClusterMarkerAsync *a) +{ + if (a == NULL) + return; + a->state = CLUSTER_MARKER_ASYNC_IDLE; + a->inflight_seq = 0; + a->deadline_us = 0; + a->submitted_at = 0; + a->kind = CLUSTER_MARKER_KIND_UNKNOWN; + a->target_node = -1; + a->has_staged_event = false; + a->staged_expect_epoch = 0; +} + +#endif /* CLUSTER_MARKER_ASYNC_H */ diff --git a/src/include/cluster/cluster_node_remove.h b/src/include/cluster/cluster_node_remove.h index ce6bbd7b19..ef722e3f62 100644 --- a/src/include/cluster/cluster_node_remove.h +++ b/src/include/cluster/cluster_node_remove.h @@ -74,6 +74,8 @@ #include "port/atomics.h" #include "storage/lwlock.h" +#include "cluster/cluster_marker_async.h" + #include "cluster/cluster_write_fence.h" /* CLUSTER_FENCE_MARKER_BYTES (marker offset) */ /* 128 nodes, same width as ReconfigEvent.dead_bitmap. */ @@ -415,6 +417,14 @@ extern void cluster_node_remove_shmem_register(void); * ------------------------------------------------------------------ */ extern ClusterRemovalMarkerSubmitResult cluster_node_remove_submit_marker(const ClusterRemovalMarker *m); +extern bool cluster_node_remove_submit_marker_async(ClusterMarkerAsync *a, + const ClusterRemovalMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now); +extern ClusterMarkerPollResult cluster_node_remove_poll_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern bool cluster_node_remove_qvotec_poll_pending(ClusterRemovalMarker *out); extern void cluster_node_remove_qvotec_complete(bool acked); extern void cluster_node_remove_publish_qvotec_latch(struct Latch *latch); diff --git a/src/include/cluster/cluster_pcm_lock.h b/src/include/cluster/cluster_pcm_lock.h index d88d82dc8f..3fb909557f 100644 --- a/src/include/cluster/cluster_pcm_lock.h +++ b/src/include/cluster/cluster_pcm_lock.h @@ -415,6 +415,12 @@ extern void cluster_pcm_lock_clear_pending_x(BufferTag tag); extern int32 cluster_pcm_lock_query_pending_x_requester(BufferTag tag); extern uint64 cluster_pcm_lock_clear_pending_x_for_node(int32 dead_node); +/* PGRAC: spec-4.6a BUG-C2 — failure-path PCM holder cleanup. Removes a + * DEAD node from X/S/PI holder records in this master's PCM directory and + * demotes entries to N when no live holder remains. Idempotent under repeated + * dead-sweep ticks. */ +extern uint64 cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node); + /* PGRAC: spec-5.13 D5 (clean-leave PCM release) — a leaving node clears its * OWN holder records (X / S / PI) from the local PCM directory after the GCS * flush seam has persisted all dirty X blocks (CL-I5). Demotes an entry's diff --git a/src/include/cluster/cluster_reconfig.h b/src/include/cluster/cluster_reconfig.h index e34c3a1942..1aa106f690 100644 --- a/src/include/cluster/cluster_reconfig.h +++ b/src/include/cluster/cluster_reconfig.h @@ -56,7 +56,8 @@ #include "port/atomics.h" #include "storage/lwlock.h" -#include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES (spec-5.13 clean_departed_epoch) */ +#include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES (spec-5.13 clean_departed_epoch) */ +#include "cluster/cluster_marker_async.h" #include "cluster/cluster_membership.h" /* ClusterMembershipTable (spec-5.15 D2 SSOT) */ struct Latch; /* spec-5.15 D4 — join-marker qvotec mailbox latch (pointer only) */ @@ -225,6 +226,16 @@ typedef struct ClusterReconfigState { pg_atomic_uint64 apply_counter; /* total events observed */ pg_atomic_uint64 dedup_skip_counter; /* duplicate event_id skipped */ pg_atomic_uint64 procsig_broadcast_count; /* PROCSIG broadcast tally */ + /* spec-2.29a r2 t/274: a backend-context coordinator (e.g. the fail-stop + * inject test) advances the epoch then SYNCHRONOUSLY waits for the fence + * marker before publishing. During that (seconds-long) wait the epoch is + * bumped but the reconfig event is not yet published, so a concurrently + * running LMON GRD IDLE tick would re-capture the post-bump epoch as its + * WAIT_EPOCH baseline and wedge (old==cur). The LMON-process-local staged + * flags cannot see a backend's pre-bump window, so publish it in shmem: + * set before the bump, cleared after publish / on failure. The GRD IDLE + * baseline-hold guard reads this OR the local stage. */ + pg_atomic_uint32 prebump_sync_active; /* spec-5.14 D6 — touched_peers fail-stop observability (Q8 A': these * live in this existing region; the region count is unchanged). */ @@ -316,6 +327,8 @@ typedef struct ClusterReconfigState { pg_atomic_uint64 join_reject_count; pg_atomic_uint64 join_timeout_count; pg_atomic_uint64 clean_departed_cleared_count; + pg_atomic_uint64 marker_slow_ack_count; + pg_atomic_uint64 marker_timeout_count; /* * spec-5.15 D1 — per declared node, the freshest voting-slot incarnation + @@ -406,6 +419,12 @@ extern void cluster_reconfig_shmem_register(void); */ extern void cluster_reconfig_get_last_event(ReconfigEvent *out); +/* spec-2.29a: true while a pre-bump coordinator stage (fail-stop fence / + * node-remove / join Phase-1) has bumped the epoch but not yet published its + * reconfig event -- the GRD recovery IDLE tick must hold its baseline instead + * of re-capturing the post-bump epoch (else WAIT_EPOCH wedges). */ +extern bool cluster_reconfig_has_pending_prebump_stage(void); + /* ============================================================ * Coordinator path APIs (Step 2 D2 wiring). @@ -580,10 +599,24 @@ extern bool cluster_reconfig_commit_member(int32 node_id, uint64 admitted_incarn */ extern ClusterJoinMarkerSubmitResult cluster_reconfig_submit_join_marker(int32 target_node, const ClusterJoinCommitMarker *m); +extern bool cluster_reconfig_submit_join_marker_async(ClusterMarkerAsync *a, int32 target_node, + const ClusterJoinCommitMarker *m, + ClusterMarkerAsyncKind kind, TimestampTz now); +extern ClusterMarkerPollResult cluster_reconfig_poll_join_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern bool cluster_reconfig_join_qvotec_poll_pending(int32 *out_target_node, void *out_slot512); extern void cluster_reconfig_join_qvotec_complete(bool acked); extern void cluster_reconfig_publish_join_qvotec_latch(struct Latch *latch); +extern void cluster_reconfig_note_marker_slow_ack(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us); +extern void cluster_reconfig_note_marker_timeout(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us); +extern uint64 cluster_reconfig_get_marker_slow_ack_count(void); +extern uint64 cluster_reconfig_get_marker_timeout_count(void); + /* ============================================================ * spec-5.15 D5 — joiner-side write gate + admission (INV-J9 / §2.4 / §2.7). * ============================================================ @@ -756,5 +789,11 @@ extern uint64 cluster_reconfig_apply_node_removed_as_coordinator(int32 removed_n uint64 last_incarnation, bool *out_contest); +/* spec-2.29a review r1 P2: true while the staged node-removed fence marker is + * in flight -- the node-remove driver skips its FENCE_ARMING pre-work + * (REMOVING marker / stripe retire) instead of re-keying it to the live + * (already self-bumped) epoch. */ +extern bool cluster_reconfig_node_removed_fence_stage_pending(void); + #endif /* CLUSTER_RECONFIG_H */ diff --git a/src/include/cluster/cluster_write_fence.h b/src/include/cluster/cluster_write_fence.h index 7543664e72..ac5a965882 100644 --- a/src/include/cluster/cluster_write_fence.h +++ b/src/include/cluster/cluster_write_fence.h @@ -39,6 +39,8 @@ #ifndef CLUSTER_WRITE_FENCE_H #define CLUSTER_WRITE_FENCE_H +#include "cluster/cluster_marker_async.h" + /* * cluster_write_fence_decide -- the PURE cooperative write-fence judge (spec-4.12 * D3). A shared-storage write is allowed iff enforcement is on AND the local @@ -517,6 +519,14 @@ extern void cluster_write_fence_note_baseline_stale(void); */ extern ClusterFenceMarkerSubmitResult cluster_write_fence_submit_marker(const ClusterFenceMarker *m); +extern bool cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, + const ClusterFenceMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now); +extern ClusterMarkerPollResult cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern void cluster_write_fence_publish_qvotec_latch(struct Latch *latch); extern bool cluster_write_fence_qvotec_poll_pending(ClusterFenceMarker *out); extern void cluster_write_fence_qvotec_complete(bool acked); diff --git a/src/include/cluster/cluster_xid_authority.h b/src/include/cluster/cluster_xid_authority.h new file mode 100644 index 0000000000..0e73859d46 --- /dev/null +++ b/src/include/cluster/cluster_xid_authority.h @@ -0,0 +1,266 @@ +/*------------------------------------------------------------------------- + * + * cluster_xid_authority.h + * Shared XID authority + native-era prehistory (spec-6.15b D1/D2). + * + * Under cluster.shared_catalog the pre-formation ("native era", + * cluster.enabled=off single-writer) xid consumption of the seed node is + * invisible to every other node: a joiner's per-node recovery anchor was + * seeded from its pre-seed clone-era control file, so its transaction + * horizon (nextXid) stays below the seed's committed xids and MVCC + * silently judges seed rows "in the future" (false-invisible), while its + * local pg_xact carries no commit bits for them (poison-hint hazard on + * the shared catalog pages). + * + * Two small durable files under cluster.shared_data_dir close this: + * + * global/pgrac_xid_authority -- never-lowered native-era xid + * high-water (first xid NOT consumed by the native era) plus two + * one-way lifecycle flags: SEALED (a clean native-era shutdown + * published a complete high-water + prehistory; joiners may adopt) + * and CLUSTER_ERA (a cluster.enabled=on boot happened; the native + * era is closed forever and may not be re-entered). + * + * global/pgrac_xid_prehistory -- the raw local pg_xact page run + * covering [0, native_hw), CRC-guarded. A joiner whose own + * nextXid proves it is a pre-seed lineage adopts these bytes into + * its local pg_xact before StartupCLOG, giving it first-hand + * commit-status truth for every native-era xid (no hint-bit + * trust, no CLOG overlay of cluster-era foreign bits -- see the + * spec's no-clog-overlay boundary note). + * + * Torn-safe file discipline mirrors cluster_oid_lease.c: temp + fsync + + * .bak roll + durable_rename on write; primary-then-.bak fail-closed + * read; ENOENT-only-absent presence probe. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/include/cluster/cluster_xid_authority.h + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-6.15b-xid-authority-native-era.md (D1/D2) + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_XID_AUTHORITY_H +#define CLUSTER_XID_AUTHORITY_H + +#include "port/pg_crc32c.h" + +/* ============================================================ + * On-disk authority image (spec-6.15b §2). + * ============================================================ */ + +#define CLUSTER_XID_AUTHORITY_MAGIC 0x0141D617 +#define CLUSTER_XID_AUTHORITY_VERSION 1 + +/* A clean native-era shutdown published a complete hw + prehistory. + * Cleared again -- only while CLUSTER_ERA is unset -- when a follow-up + * native-era run boots, so a crash of that run leaves joiners + * fail-closed instead of adopting the previous pass's stale hw. */ +#define CLUSTER_XID_AUTHORITY_FLAG_SEALED 0x0001 +/* A cluster.enabled=on boot closed the native era forever (one-way). */ +#define CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA 0x0002 + +typedef struct ClusterXidAuthorityHeader { + uint32 magic; + uint32 version; + uint32 flags; + uint32 reserved; /* zero */ + uint64 native_hw_full; /* FullTransactionId value: first xid NOT + * consumed by the native era; never lowered */ + uint64 next_multi; /* native-era nextMulti at publish; vet-only + * (spec Q5: >FirstMultiXactId refuses seed) */ + pg_crc32c crc; /* over all preceding bytes */ +} ClusterXidAuthorityHeader; + +/* On-disk ABI locked at compile time (review F7; mirrors the recovery + * anchor's precedent). The unit truth table re-checks these at runtime. */ +StaticAssertDecl(offsetof(ClusterXidAuthorityHeader, flags) == 8, + "xid authority header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidAuthorityHeader, native_hw_full) == 16, + "xid authority header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidAuthorityHeader, next_multi) == 24, + "xid authority header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidAuthorityHeader, crc) == 32, + "xid authority header layout is on-disk ABI"); +StaticAssertDecl(sizeof(ClusterXidAuthorityHeader) == 40, + "xid authority header size is on-disk ABI"); + +/* Relative paths under cluster.shared_data_dir. */ +#define CLUSTER_XID_AUTHORITY_REL_PATH "global/pgrac_xid_authority" +#define CLUSTER_XID_AUTHORITY_BAK_REL_PATH "global/pgrac_xid_authority.bak" +#define CLUSTER_XID_AUTHORITY_TMP_REL_PATH "global/pgrac_xid_authority.tmp" +#define CLUSTER_XID_AUTHORITY_BAK_TMP_REL_PATH "global/pgrac_xid_authority.bak.tmp" + +typedef enum ClusterXidAuthorityValidity { + CLUSTER_XID_AUTHORITY_VALID = 0, + CLUSTER_XID_AUTHORITY_INVALID_SHORT, + CLUSTER_XID_AUTHORITY_INVALID_MAGIC, + CLUSTER_XID_AUTHORITY_INVALID_CRC, +} ClusterXidAuthorityValidity; + +/* ============================================================ + * On-disk prehistory image (spec-6.15b D2). + * ============================================================ */ + +#define CLUSTER_XID_PREHISTORY_MAGIC 0x0142D617 +#define CLUSTER_XID_PREHISTORY_VERSION 1 + +/* + * Refusal cap on the native era (spec §3.4): 8M xids = 256 CLOG pages = + * 2MB of payload. A seed load that consumes more must be split or moved + * in-protocol; the cap keeps the blob bounded and the adopt O(small). + */ +#define CLUSTER_XID_PREHISTORY_MAX_XID UINT64CONST(8388608) + +typedef struct ClusterXidPrehistoryHeader { + uint32 magic; + uint32 version; + uint64 native_hw_full; /* payload covers xids [0, native_hw_full) */ + uint32 payload_len; /* raw pg_xact bytes following this header */ + uint32 reserved; /* zero */ + pg_crc32c crc; /* over header fields above + payload */ +} ClusterXidPrehistoryHeader; + +StaticAssertDecl(offsetof(ClusterXidPrehistoryHeader, native_hw_full) == 8, + "xid prehistory header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidPrehistoryHeader, payload_len) == 16, + "xid prehistory header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidPrehistoryHeader, crc) == 24, + "xid prehistory header layout is on-disk ABI"); +StaticAssertDecl(sizeof(ClusterXidPrehistoryHeader) == 32, + "xid prehistory header size is on-disk ABI"); + +#define CLUSTER_XID_PREHISTORY_REL_PATH "global/pgrac_xid_prehistory" +#define CLUSTER_XID_PREHISTORY_BAK_REL_PATH "global/pgrac_xid_prehistory.bak" +#define CLUSTER_XID_PREHISTORY_TMP_REL_PATH "global/pgrac_xid_prehistory.tmp" +#define CLUSTER_XID_PREHISTORY_BAK_TMP_REL_PATH "global/pgrac_xid_prehistory.bak.tmp" + +/* ============================================================ + * Pure layer (standalone-linkable; exercised by cluster_unit). + * ============================================================ */ + +extern ClusterXidAuthorityValidity cluster_xid_authority_classify(const char *buf, size_t len); + +/* + * cluster_xid_prehistory_payload_bytes -- whole-CLOG-page payload size + * covering xids [0, native_hw_full). 0 when native_hw_full is 0 (nothing + * to carry); the per-page constant mirrors CLOG_XACTS_PER_PAGE without + * dragging clog.c internals into the unit build. + */ +extern uint32 cluster_xid_prehistory_payload_bytes(uint64 native_hw_full); + +extern ClusterXidAuthorityValidity cluster_xid_prehistory_classify(const char *buf, size_t len); + +/* ============================================================ + * Torn-safe authority file I/O (mirrors cluster_oid_lease.c). + * ============================================================ */ + +/* + * Fail-closed read: primary then .bak; false when neither validates. + * Never ereports (safe on the bootstrap early-read path). + */ +extern bool cluster_xid_authority_read(ClusterXidAuthorityHeader *out); + +/* As read(), reporting .bak fallback for a consumer-side LOG (review F8). */ +extern bool cluster_xid_authority_read_checked(ClusterXidAuthorityHeader *out, bool *used_bak); + +/* + * ENOENT-only-absent presence probe (spec-6.14 §3.6 posture): any stat() + * failure other than ENOENT reports present, so a transiently unreadable + * authority is never re-seeded over. + */ +extern bool cluster_xid_authority_present(void); + +/* + * Seed the authority (unsealed) when neither image exists. Returns true + * when this call created it. A trustworthy existing image is a no-op. + */ +extern bool cluster_xid_authority_seed_if_absent(uint64 initial_native_hw); + +/* + * Monotone raise of native_hw_full + set SEALED (never lowers, never + * clears flags). seal=false publishes a crash-safe interim high-water + * during the native era without opening adoption. PANICs on I/O error. + */ +extern void cluster_xid_authority_publish_native(uint64 native_hw_full, uint64 next_multi, + bool seal); + +/* + * One-way: stamp CLUSTER_ERA (first cluster.enabled=on boot). Idempotent + * re-assert: rewrites BOTH copies until both carry the flag (repairing a + * torn previous stamp); no-write no-op once both are stamped. + */ +extern void cluster_xid_authority_mark_cluster_era(void); + +/* + * A follow-up native-era (cluster.enabled=off) boot on a sealed authority + * re-opens the era: clears SEALED so a crash of this run never exposes the + * previous pass's stale high-water to joiners. Caller vets CLUSTER_ERA + * first (re-entry is FATAL). Idempotent re-assert: rewrites BOTH copies + * until neither retains SEALED (repairing a torn previous unseal); + * no-write no-op once both are open. + */ +extern void cluster_xid_authority_begin_native_run(void); + +/* ============================================================ + * Prehistory publish / adopt (spec-6.15b D2; P2: adopt is complete and + * fsynced strictly before StartupCLOG -- both run under the postmaster / + * shutdown-checkpoint single-writer windows, file-level only). + * ============================================================ */ + +/* + * Seed side: snapshot local pg_xact pages [0, native_hw_full) into the + * shared prehistory file (torn-safe). PANICs on I/O error; FATALs when + * the native era exceeds CLUSTER_XID_PREHISTORY_MAX_XID. + */ +extern void cluster_xid_prehistory_publish(const char *local_pgdata, uint64 native_hw_full); + +/* + * Joiner side: decode the shared prehistory into local pg_xact page files, + * pg_fsync each touched segment and the pg_xact directory before returning. + * Idempotent (same-byte overwrite). FATALs (53RB5) on a missing/corrupt + * blob or a native_hw mismatch against the sealed authority. + */ +extern void cluster_xid_prehistory_adopt(const char *local_pgdata, uint64 native_hw_full); +extern bool cluster_xid_prehistory_was_adopted(void); + +/* ============================================================ + * Divergent-lineage guard (review F2; spec Q6 amendment). + * ============================================================ */ + +typedef enum ClusterXidPrefixVerdict { + CLUSTER_XID_PREFIX_CONSISTENT = 0, /* local prefix matches the blob */ + CLUSTER_XID_PREFIX_DIVERGED, /* local pg_xact contradicts the blob */ + CLUSTER_XID_PREFIX_UNAVAILABLE, /* no trustworthy blob to compare */ +} ClusterXidPrefixVerdict; + +/* + * Compare the local pg_xact bytes covering xids [oldest_xid_full, + * limit_xid_full) with the sealed prehistory blob at 2-bit (per-xact) + * precision. Bits below oldest_xid_full are frozen truth that CLOG never + * consults again and that SimpleLruTruncate may already have removed + * (whole segments); they are exempt whether the pages survive or not. A + * local page missing INSIDE the comparable range is fail-closed + * UNAVAILABLE, never CONSISTENT: pg_xact covers [oldestXid, nextXid) on a + * well-formed node, so a hole is an anomaly, and a front-truncated + * divergent clone must not pass as a "shorter clone" (review r3-X2). + * Callers FATAL on DIVERGED/UNAVAILABLE: a joiner whose own native-era + * history contradicts the seed's is not a pre-seed lineage, and neither + * skipping (trusting local bits) nor adopting (overwriting the joiner's + * own outcomes) is sound for it. + */ +extern ClusterXidPrefixVerdict cluster_xid_prehistory_prefix_check(const char *local_pgdata, + uint64 native_hw_full, + uint64 limit_xid_full, + uint64 oldest_xid_full); + +#endif /* CLUSTER_XID_AUTHORITY_H */ diff --git a/src/test/cluster_tap/t/015_inject.pl b/src/test/cluster_tap/t/015_inject.pl index 772ef599f4..2035b40072 100644 --- a/src/test/cluster_tap/t/015_inject.pl +++ b/src/test/cluster_tap/t/015_inject.pl @@ -57,8 +57,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'pg_stat_cluster_injections returns 158 rows (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'pg_stat_cluster_injections returns 159 rows (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- @@ -86,8 +86,8 @@ 'postgres', 'SELECT string_agg(name, \',\' ORDER BY name) FROM pg_stat_cluster_injections' ), - 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-cr-construct,cluster-lms-undo-fetch,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', - '158 injection point names match the full registry (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-cr-construct,cluster-lms-undo-fetch,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', + '159 injection point names match the full registry (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/017_debug.pl b/src/test/cluster_tap/t/017_debug.pl index 1f24fab0a2..6d53bad1df 100644 --- a/src/test/cluster_tap/t/017_debug.pl +++ b/src/test/cluster_tap/t/017_debug.pl @@ -63,8 +63,8 @@ 'postgres', q{SELECT string_agg(DISTINCT category, ',' ORDER BY category) FROM pg_cluster_state}), - 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', - 'all 55 categories appear (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; L122 alphabetic verify)'); + 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', + 'all 56 categories appear (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; L122 alphabetic verify)'); # ---------- @@ -123,15 +123,15 @@ 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.fault_type'}), - '158', - 'all 158 injection points have a .fault_type entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'all 159 injection points have a .fault_type entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is( $node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.hits'}), - '158', - 'all 158 injection points have a .hits entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'all 159 injection points have a .hits entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/018_shared_fs.pl b/src/test/cluster_tap/t/018_shared_fs.pl index 676dfc9e1a..42f8817b91 100644 --- a/src/test/cluster_tap/t/018_shared_fs.pl +++ b/src/test/cluster_tap/t/018_shared_fs.pl @@ -131,8 +131,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L9 total injection registry size is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'L9 total injection registry size is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/020_shmem_registry.pl b/src/test/cluster_tap/t/020_shmem_registry.pl index cb4c48513e..eab706fd4c 100644 --- a/src/test/cluster_tap/t/020_shmem_registry.pl +++ b/src/test/cluster_tap/t/020_shmem_registry.pl @@ -280,8 +280,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L15 total injection registry size is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; full breakdown in t/015)'); + '159', + 'L15 total injection registry size is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-2.29a +1 cluster-qvotec-marker-service-hold; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/021_block_format.pl b/src/test/cluster_tap/t/021_block_format.pl index a1c589de51..57a321a3a6 100644 --- a/src/test/cluster_tap/t/021_block_format.pl +++ b/src/test/cluster_tap/t/021_block_format.pl @@ -188,8 +188,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L11 pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '159', + 'L11 pg_stat_cluster_injections is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; spec-2.29a +1 cluster-qvotec-marker-service-hold; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/022_itl_slot.pl b/src/test/cluster_tap/t/022_itl_slot.pl index 871ee4a7d0..f2cac828a8 100644 --- a/src/test/cluster_tap/t/022_itl_slot.pl +++ b/src/test/cluster_tap/t/022_itl_slot.pl @@ -204,8 +204,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L12a pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '159', + 'L12a pg_stat_cluster_injections is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; spec-2.29a +1 cluster-qvotec-marker-service-hold; full breakdown in t/015)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/023_buffer_descriptor.pl b/src/test/cluster_tap/t/023_buffer_descriptor.pl index d5aa071872..2c41642a72 100644 --- a/src/test/cluster_tap/t/023_buffer_descriptor.pl +++ b/src/test/cluster_tap/t/023_buffer_descriptor.pl @@ -157,8 +157,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L11 pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '159', + 'L11 pg_stat_cluster_injections is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; spec-2.29a +1 cluster-qvotec-marker-service-hold; full breakdown in t/015)'); # ---------- @@ -168,8 +168,8 @@ is($node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L12 pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L12 pg_cluster_state has 56 categories (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); # ---------- @@ -232,7 +232,7 @@ my $smoke_categories = $node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}); -is($smoke_categories, '55', 'L16 cluster_smoke surface integrates buffer_format + pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion + xid_stripe categories (55 categories; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); +is($smoke_categories, '56', 'L16 cluster_smoke surface integrates buffer_format + pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion + xid_stripe + reconfig categories (56 categories; spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); # ---------- diff --git a/src/test/cluster_tap/t/024_pcm_lock.pl b/src/test/cluster_tap/t/024_pcm_lock.pl index ba829df534..32024b1733 100644 --- a/src/test/cluster_tap/t/024_pcm_lock.pl +++ b/src/test/cluster_tap/t/024_pcm_lock.pl @@ -55,14 +55,15 @@ # ---------- -# L1: pg_cluster_state.pcm category has 22 keys (spec-2.30 + spec-6.14a D5 + spec-6.14 D5) +# L1: pg_cluster_state.pcm category has 23 keys (spec-2.30 + spec-6.14a D5 + spec-6.14 D5 +# + spec-4.6a D12 dead_cleanup_entries) # activates the state-machine diagnostics. # ---------- is($node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='pcm'}), - '22', - 'L1 pg_cluster_state.pcm category has 22 keys (spec-2.30 surface + spec-6.14a D5 + spec-6.14 D5)'); + '23', + 'L1 pg_cluster_state.pcm category has 23 keys (spec-2.30 surface + spec-6.14a D5 + spec-6.14 D5 + spec-4.6a D12)'); # ---------- @@ -162,8 +163,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L6a pg_stat_cluster_injections has 158 entries (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'L6a pg_stat_cluster_injections has 159 entries (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql( 'postgres', @@ -189,8 +190,8 @@ is($node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L7b pg_cluster_state has 55 distinct categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L7b pg_cluster_state has 56 distinct categories (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); # ---------- @@ -209,7 +210,7 @@ my $smoke_categories = $node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}); -is($smoke_categories, '55', 'L9 cluster_smoke surface integrates pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + undo_record + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion categories (55 categories; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); +is($smoke_categories, '56', 'L9 cluster_smoke surface integrates pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + undo_record + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion + reconfig categories (56 categories; spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); # ---------- diff --git a/src/test/cluster_tap/t/030_acceptance.pl b/src/test/cluster_tap/t/030_acceptance.pl index 6d163b4552..3672d99bce 100644 --- a/src/test/cluster_tap/t/030_acceptance.pl +++ b/src/test/cluster_tap/t/030_acceptance.pl @@ -330,7 +330,7 @@ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', 'M1 158 injection points (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', 'M1 159 injection points (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql('postgres', q{SELECT string_agg(name, ',' ORDER BY name) FROM pg_stat_cluster_injections WHERE name LIKE 'cluster-init-%'}), @@ -360,8 +360,8 @@ 'postgres', q{SELECT count(DISTINCT key) FROM pg_cluster_state WHERE category='inject' AND (key LIKE '%.fault_type' OR key LIKE '%.hits')} - ) eq '316', - 'M5 inject category has 158×2 = 316 sub-keys (.fault_type + .hits; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + ) eq '318', + 'M5 inject category has 159×2 = 318 sub-keys (.fault_type + .hits; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->get_cluster_state_value('inject', 'armed_count'), '0', 'M6 inject.armed_count starts at 0 in fresh backend'); @@ -395,8 +395,8 @@ is($node->safe_psql('postgres', q{SELECT string_agg(DISTINCT category, ',' ORDER BY category) FROM pg_cluster_state}), - 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', - 'O2 pg_cluster_state has all 55 categories (spec-6.15 adds xid_stripe; spec-6.12 adds xnode_lever; spec-6.2 adds smart_fusion; spec-6.14 adds catalog)'); + 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', + 'O2 pg_cluster_state has all 56 categories (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.12 adds xnode_lever; spec-6.2 adds smart_fusion; spec-6.14 adds catalog)'); is($node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE value IS NULL}), diff --git a/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl b/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl index f12abe0f14..9bc0e1fccf 100644 --- a/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl +++ b/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl @@ -325,8 +325,8 @@ sub wait_hint_delta # ============================================================ is($pair->node0->safe_psql('postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L12a pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L12a pg_cluster_state has 56 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; spec-2.29a adds reconfig)'); my $tt_categories = $pair->node0->safe_psql('postgres', q{ SELECT string_agg(c, ',' ORDER BY c) diff --git a/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl b/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl index 4e65595eff..56d8cd35f9 100644 --- a/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl +++ b/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl @@ -296,8 +296,8 @@ sub wait_hint_delta # ============================================================ is($pair->node0->safe_psql('postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L14a pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L14a pg_cluster_state has 56 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; spec-2.29a adds reconfig)'); is($pair->node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state diff --git a/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl b/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl index 80447a5161..8f4f1c5ce7 100644 --- a/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl +++ b/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl @@ -287,7 +287,7 @@ # ============================================================ is($pair->node0->safe_psql('postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', 'L13a pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', 'L13a pg_cluster_state has 56 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; spec-2.29a adds reconfig)'); is($pair->node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state diff --git a/src/test/cluster_tap/t/293_hw_online_remaster.pl b/src/test/cluster_tap/t/293_hw_online_remaster.pl index 9c156e1e53..2447312952 100644 --- a/src/test/cluster_tap/t/293_hw_online_remaster.pl +++ b/src/test/cluster_tap/t/293_hw_online_remaster.pl @@ -30,13 +30,18 @@ # retryable transient, NOT an HW fault -- so this leg retries and SKIPs # (never fails) when only that orthogonal recovery is outstanding. # -# Negative (corrupt the dead master's snapshot -> fail-closed): +# Negative/self-heal extensions (spec-4.6a): # L6 a fresh pair; corrupt node0's HW snapshot on shared storage, then # kill node0. The survivor's rebuild must FAIL CLOSED # (hw.remaster_blocked_count advances) -- it never trusts a corrupt # snapshot, never auto-creates at 0, never reads FileSize as the -# authority. The affected shards stay frozen (P7 gate), so no extend -# can silently re-hand a block. +# authority. +# L7 hide node0's snapshot, kill node0, observe BLOCKED, then restore +# the same file. The same reconfig episode must retry and converge +# to DONE without waiting for a later episode. +# L8 keep the snapshot corrupt with retry_max_attempts=2. The survivor +# must report retry_exhausted once and keep fail-closed; DONE must not +# advance. # # Harness: ClusterPair shared_data + wal_threads_root + 3 voting disks. # Mirrors the kill/reconfig pattern of t/249 / t/274. @@ -181,17 +186,22 @@ sub retry_sql { my ($node, $sql) = @_; my $last = ''; + my $last_retryable = 0; for my $attempt (1 .. 30) { my ($rc, $out, $err) = $node->psql('postgres', $sql, timeout => 60); - return (1, $out) if defined $rc && $rc == 0; - $last = $err // ''; + my $combined = join("\n", grep { defined $_ && $_ ne '' } ($out, $err)); + return (1, $out, 0) + if defined $rc && $rc == 0 && $combined !~ /ERROR:/; + + $last = $combined; + $last_retryable = + $last =~ /being rebuilt after reconfiguration|status unknown|not yet propagated|could not obtain X transfer|did not ship a current image/; # retry only the orthogonal post-reconfig settling transients - last - unless $last =~ /being rebuilt after reconfiguration|status unknown|not yet propagated|could not obtain X transfer|did not ship a current image/; + last unless $last_retryable; usleep(500_000); } - return (0, $last); + return (0, $last, $last_retryable); } # L5a (the robust HW proof): the rebuilt authority is SERVEABLE -- the HW serve @@ -210,22 +220,229 @@ sub retry_sql # that is NOT an HW fault. Retry, and SKIP (never fail) when only that orthogonal # recovery is outstanding; a 53RA6 from the HW gate would NOT be retried and would # fail the test. -my ($ext_ok, $ext_res) = +my ($ext_ok, $ext_res, $ext_retryable) = retry_sql($n1, 'INSERT INTO r1 SELECT g,g,g,g FROM generate_series(4001,4500) g'); SKIP: { - skip - "post-remaster extend gated by orthogonal dead-block recovery (spec-4.7/4.11), not HW: $ext_res", - 1 - unless $ext_ok; - my ($cnt_ok, $cnt) = retry_sql($n1, 'SELECT count(*) FROM r1'); - is($cnt, '4500', - 'L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite') - or diag("count result: $cnt"); + if (!$ext_ok) + { + skip + "post-remaster extend gated by orthogonal dead-block recovery (spec-4.7/4.11), not HW: $ext_res", + 1 + if $ext_retryable; + fail('L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite'); + diag("extend result: $ext_res"); + } + else + { + my ($cnt_ok, $cnt, $cnt_retryable) = retry_sql($n1, 'SELECT count(*) FROM r1'); + if (!$cnt_ok) + { + skip + "post-remaster count gated by orthogonal visibility recovery (not HW): $cnt", + 1 + if $cnt_retryable; + fail('L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite'); + diag("count result: $cnt"); + } + else + { + is($cnt, '4500', + 'L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite') + or diag("count result: $cnt"); + } + } } $pair->stop_pair; +# ====================================================================== +# SELF-HEAL: first attempt BLOCKED, same episode retries after repair. +# ====================================================================== + +my $heal = PostgreSQL::Test::ClusterPair->new_pair( + 'hw_remaster_selfheal', + quorum_voting_disks => 3, + shared_data => 1, + wal_threads_root => 1, + extra_conf => [ + 'autovacuum = off', + 'cluster.grd_max_entries = 4096', + 'cluster.ges_request_timeout_ms = 3000', + 'cluster.ges_retransmit_max_attempts = 0', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 8', + ]); +$heal->start_pair; +my $h0 = $heal->node0; +my $h1 = $heal->node1; + +ok(poll_until($h1, 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 20), + 'L7: self-heal pair in quorum'); +poll_until( + $h1, + q{SELECT (state='alive' AND heartbeat_recv_count>0)::text + FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 20); + +make_and_fill($h0, $h1, "s$_", 4000) for (1 .. 6); +$h0->safe_psql('postgres', 'CHECKPOINT'); +$h1->safe_psql('postgres', 'CHECKPOINT'); + +my $hsnap = $heal->shared_data_root . "/global/pg_hw_snapshot.0"; +my $hhold = "$hsnap.hold"; +ok(-f $hsnap, 'L7: node0 HW snapshot exists before self-heal injection'); +rename($hsnap, $hhold) or die "rename $hsnap -> $hhold: $!"; + +my $h_done_before = hw_counter($h1, 'remaster_done_count'); +my $h_blocked_before = hw_counter($h1, 'remaster_blocked_count'); +my $h_retry_before = hw_counter($h1, 'remaster_retry_count'); +$heal->kill_node9(0); +ok( poll_until( + $h1, + q{SELECT (state='dead')::text FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 30), + 'L7: self-heal pair declared node0 dead'); +ok( poll_until( + $h1, + "SELECT (value::bigint > $h_blocked_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_blocked_count'", + 'true', 40), + 'L7: first HW rebuild attempt failed closed while the snapshot was hidden'); + +rename($hhold, $hsnap) or die "rename $hhold -> $hsnap: $!"; +ok( poll_until( + $h1, + "SELECT (value::bigint > $h_retry_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_retry_count'", + 'true', 40), + 'L7: same reconfig episode launched a HW remaster retry'); +ok( poll_until( + $h1, + "SELECT (value::bigint > $h_done_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_done_count'", + 'true', 60), + 'L7: restored snapshot let the same-episode retry converge to DONE'); +is(hw_counter($h1, 'remaster_retry_exhausted_count'), 0, + 'L7: self-heal path did not exhaust the retry budget'); + +# spec-4.6a section 4 (D8, Amendment v1.2 R1): after the same-episode DONE the +# episode has converged (P7 ran: DONE counter + no exhaustion asserted above), +# but in this out-of-scope deployment (online_thread_recovery off) EVERY tag +# whose static master is the dead node stays fail-closed until that node +# returns — including brand-new blocks, which have no materialization proof +# either (the cold-block door cannot distinguish never-existed from +# not-yet-replayed; main's t/293 L5b documents the same posture). The +# survivor write is therefore the honest two-outcome contract: success (all +# touched tags survivor-mastered) or the explicit bounded 53R9L — never a +# hang and never an unbounded wedge. +{ + my ($ext_ok, $ext_res) = (0, ''); + for my $try (1 .. 10) + { + $ext_res = eval { + $h1->safe_psql('postgres', + 'CREATE TABLE IF NOT EXISTS s1_heal (a int, b int, c int, d int); ' + . 'INSERT INTO s1_heal SELECT g, g, g, g FROM generate_series(1, 4096) g'); + 1; + } ? 'ok' : ($@ // 'error'); + if ($ext_res eq 'ok') { $ext_ok = 1; last; } + sleep 1; + } + if ($ext_ok) + { + ok(1, 'L7: survivor new-table extend succeeded after same-episode DONE (P7 unfreeze)'); + } + else + { + like($ext_res, + qr/being rebuilt after reconfiguration|resource recovering|53R9L|cluster TT status unknown/i, + 'L7: survivor extend fail-closed with an explicit SQLSTATE (dead-master tag, out-of-scope posture)'); + } +} + +$heal->stop_pair; + +# ====================================================================== +# NEGATIVE: corrupt snapshot persists -> retries exhaust and stay closed. +# ====================================================================== + +my $exh = PostgreSQL::Test::ClusterPair->new_pair( + 'hw_remaster_exhausted', + quorum_voting_disks => 3, + shared_data => 1, + wal_threads_root => 1, + extra_conf => [ + 'autovacuum = off', + 'cluster.grd_max_entries = 4096', + 'cluster.ges_request_timeout_ms = 3000', + 'cluster.ges_retransmit_max_attempts = 0', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 2', + ]); +$exh->start_pair; +my $e0 = $exh->node0; +my $e1 = $exh->node1; + +ok(poll_until($e1, 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 20), + 'L8: exhausted pair in quorum'); +poll_until( + $e1, + q{SELECT (state='alive' AND heartbeat_recv_count>0)::text + FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 20); + +make_and_fill($e0, $e1, "x$_", 4000) for (1 .. 6); +$e0->safe_psql('postgres', 'CHECKPOINT'); +$e1->safe_psql('postgres', 'CHECKPOINT'); + +my $esnap = $exh->shared_data_root . "/global/pg_hw_snapshot.0"; +ok(-f $esnap, 'L8: node0 HW snapshot exists before exhaustion injection'); +open(my $efh, '>', $esnap) or die "open $esnap: $!"; +binmode $efh; +print $efh ("\xDE\xAD\xBE\xEF" x 64); +close $efh; + +my $e_done_before = hw_counter($e1, 'remaster_done_count'); +my $e_exh_before = hw_counter($e1, 'remaster_retry_exhausted_count'); +$exh->kill_node9(0); +poll_until( + $e1, + q{SELECT (state='dead')::text FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 30); +ok( poll_until( + $e1, + "SELECT (value::bigint > $e_exh_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_retry_exhausted_count'", + 'true', 70), + 'L8: persistent corrupt snapshot exhausted the same-episode retry budget'); +is(hw_counter($e1, 'remaster_done_count'), $e_done_before, + 'L8: DONE did not advance after retry exhaustion on corrupt snapshot'); + +# spec-4.6a section 4 (D8/L408): after exhaustion the WAIT_CLUSTER watchdog +# keeps sounding (counter + WARNING line) while shards stay frozen. +ok( poll_until( + $e1, + q{SELECT (value::bigint > 0)::text FROM pg_cluster_state } + . q{WHERE category='grd_recovery' AND key='cluster_gate_timeout'}, + 'true', 30), + 'L8: WAIT_CLUSTER watchdog counter advances while exhausted episode stays frozen'); +{ + my $log = PostgreSQL::Test::Utils::slurp_file($e1->logfile); + like( + $log, + qr/cluster GRD recovery WAIT_CLUSTER watchdog fired; affected shards stay frozen/, + 'L8: watchdog WARNING appears in the survivor log'); +} + +$exh->stop_pair; + # ====================================================================== # NEGATIVE: corrupt the dead master's snapshot -> rebuild fails closed. # ====================================================================== @@ -242,6 +459,9 @@ sub retry_sql 'cluster.ges_retransmit_max_attempts = 0', 'cluster.cssd_heartbeat_interval_ms = 1000', 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 2', ]); $neg->start_pair; my $m0 = $neg->node0; @@ -277,6 +497,7 @@ sub retry_sql } my $blocked_before = hw_counter($m1, 'remaster_blocked_count'); +my $neg_exh_before = hw_counter($m1, 'remaster_retry_exhausted_count'); $neg->kill_node9(0); poll_until( $m1, @@ -291,6 +512,12 @@ sub retry_sql . "WHERE category='hw' AND key='remaster_blocked_count'", 'true', 40), 'L6: HW rebuild FAILED CLOSED on the corrupt snapshot (remaster_blocked advanced; no auto-create-0)'); +ok( poll_until( + $m1, + "SELECT (value::bigint > $neg_exh_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_retry_exhausted_count'", + 'true', 70), + 'L6: corrupt snapshot eventually exhausts bounded same-episode retries'); $neg->stop_pair; diff --git a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl index a0477b88b7..a80ff120fd 100644 --- a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl +++ b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl @@ -94,6 +94,8 @@ my $shared_root = PostgreSQL::Test::Utils::tempdir(); mkdir "$shared_root/global" or die "mkdir shared global: $!"; +my $wal_root = PostgreSQL::Test::Utils::tempdir(); + my $disk_dir = PostgreSQL::Test::Utils::tempdir(); my @disks; for my $i (0 .. 2) @@ -114,7 +116,7 @@ # Step 0: node0 init -> backup -> node1 init_from_backup (one shared sysid). # ---------- my $node0 = PostgreSQL::Test::Cluster->new('sc_ddl_node0'); -$node0->init(allows_streaming => 1); +$node0->init(allows_streaming => 1, extra => [ '-X', "$wal_root/thread_1" ]); $node0->start; $node0->backup('scb'); $node0->stop; @@ -122,6 +124,24 @@ my $node1 = PostgreSQL::Test::Cluster->new('sc_ddl_node1'); $node1->init_from_backup($node0, 'scb'); +# Relocate node1's backup-copied WAL into its shared thread dir. shared_catalog +# multi-node formation is fail-fast without cluster.wal_threads_dir, and the WAL +# thread validator requires pg_wal to resolve to the configured thread_N dir. +{ + my $pgwal = $node1->data_dir . '/pg_wal'; + my $wal2 = "$wal_root/thread_2"; + mkdir $wal2 or die "mkdir $wal2: $!"; + opendir(my $dh, $pgwal) or die "opendir $pgwal: $!"; + for my $e (readdir $dh) + { + next if $e eq '.' || $e eq '..'; + rename("$pgwal/$e", "$wal2/$e") or die "rename $pgwal/$e: $!"; + } + closedir $dh; + rmdir $pgwal or die "rmdir $pgwal: $!"; + symlink($wal2, $pgwal) or die "symlink $pgwal -> $wal2: $!"; +} + # Config common to the shared-catalog feature (both nodes, both phases). my $sc_common = <append_conf('postgresql.conf', $cluster_conf); @@ -200,6 +223,24 @@ $node0->start; $node1->_update_pid(1); +# spec-6.15b D6 flipped this test to online_join+xid_striping: formation now +# passes through the join gate, whose home-block rebuild window answers +# transiently with "block master is recovering ... retry is safe" (even at +# connection time, via the pg_authid read). First-contact probes therefore +# retry; every semantic assert below stays strict. +sub psql_retry_ok +{ + my ($node, $sql, $tries) = @_; + $tries //= 120; + for (1 .. $tries) + { + my ($rc, $out) = $node->psql('postgres', $sql); + return (1, $out) if defined $rc && $rc == 0; + usleep(500_000); + } + return (0, undef); +} + # node1 must actually be up. my $n1_up = 0; for (1 .. 60) @@ -212,7 +253,7 @@ # ---------- # L1: both alive + IC-connected on the shared-sysid cluster. # ---------- -my $a0 = $node0->safe_psql('postgres', 'SELECT 1'); +my ($n0_up, $a0) = psql_retry_ok($node0, 'SELECT 1'); is($a0, '1', 'L1: node0 is up on the shared-catalog 2-node cluster'); is($n1_up, 1, 'L1: node1 is up on the shared-catalog 2-node cluster'); @@ -220,10 +261,10 @@ my $peer_ok = 0; for (1 .. 40) { - my $s = $node0->safe_psql('postgres', + my ($prc, $s) = psql_retry_ok($node0, "SELECT COALESCE(bool_or(heartbeat_recv_count > 0), false) " - . "FROM pg_cluster_ic_peers WHERE node_id = 1"); - if (defined $s && $s eq 't') { $peer_ok = 1; last; } + . "FROM pg_cluster_ic_peers WHERE node_id = 1", 1); + if ($prc && defined $s && $s eq 't') { $peer_ok = 1; last; } usleep(500_000); } is($peer_ok, 1, 'L1: node0 sees node1 heartbeats (IC connected)'); @@ -233,9 +274,10 @@ # node1 ever running the DDL. This is the Stage 2 exit debt # (spec-2.0:135) closure. # ---------- -$node0->safe_psql('postgres', +my ($ddl_ok) = psql_retry_ok($node0, 'CREATE TABLE a1_shared_cat (id int PRIMARY KEY, note text);' . "INSERT INTO a1_shared_cat VALUES (1, 'from-node0');"); +die 'first cross-node DDL never succeeded' unless $ddl_ok; # Poll node1 up to 1s (10 x 100ms). "retry is safe" per the GCS remaster # transient; a settled shared catalog resolves well within the budget. @@ -819,4 +861,23 @@ sub oid_authority_hw qr/cluster\.shared_catalog is off but the shared tree holds/, 'L7: the refusal is the designed off-flip vet FATAL (never a stale serve)'); +# ---------- +# L8 (spec-4.6a D11 level-2 fail-fast): a multi-node shared_catalog=on boot +# without cluster.wal_threads_dir is refused at startup -- that shape cannot +# rebuild a dead node's HW authority from per-thread WAL, so a node failure +# would be permanently unrecoverable (the silent-BLOCKED wedge of +# spec-4.6a section 0). Flip shared_catalog back on and blank the WAL +# threads root; the boot must fail on the D11 vet, not come up degraded. +# ---------- +$node1->append_conf('postgresql.conf', "cluster.shared_catalog = on +"); +$node1->append_conf('postgresql.conf', "cluster.wal_threads_dir = '' +"); +my $nowalret = $node1->start(fail_ok => 1); +is($nowalret, 0, 'L8: multi-node shared_catalog boot without wal_threads_dir fails'); +my $nowallog = PostgreSQL::Test::Utils::slurp_file($node1->logfile); +like($nowallog, + qr/cluster\.shared_catalog requires cluster\.wal_threads_dir in a multi-node cluster/, + 'L8: the refusal is the spec-4.6a D11 startup FATAL with the config errhint'); + done_testing(); diff --git a/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl b/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl index 301827c967..fd97003b23 100644 --- a/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl +++ b/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl @@ -44,24 +44,14 @@ # L4 neither node's log carries the pre-D9 "unsupported side effect" # fail-closed marker. # -# KNOWN-BLOCKED (two-layer, investigated 2026-07-06 after the 6.12/6.15 -# rebase). This test deterministically runs WITHOUT cluster.xid_striping -# so it brings up cleanly and reproduces Layer 1 -- turning striping on -# (which is what would make the serving legs' xids resolvable) drags in -# Layer 2, whose non-convergence makes bring-up itself flaky. +# KNOWN-BLOCKED (membership layer, investigated 2026-07-06 after the +# 6.12/6.15 rebase; D6 in spec-6.15b now requires striping for every +# shared_catalog multi-node boot). With cluster.xid_striping=on the xid +# collision layer is gone and merged recovery classifies the +# RM_CLUSTER_XID_STRIPE JOIN/RETIRE notes as GLOBAL. The remaining +# blocker is: # -# Layer 1 (xid collision) -- what striping WOULD fix: on this striped-off -# config the two threads share bare xids (pg_waldump-proven: the same xid -# on both threads writing the same catalog block), so post-merge TT -# lookups for the other thread's xids are permanently UNKNOWN and every -# cross-thread catalog read fail-closes 53R97 (honest, never false- -# visible). With cluster.xid_striping=on the collision is gone AND merged -# recovery classifies the RM_CLUSTER_XID_STRIPE JOIN/RETIRE notes as -# GLOBAL (see cluster_recovery_record_class, added this session), so both -# nodes boot and complete cold merged recovery -- verified in a striping- -# on diagnostic run. But that run then exposes: -# -# Layer 2 (the real blocker, root cause pinned 2026-07-06 via debug1): +# Layer 2 (root cause pinned 2026-07-06 via debug1): # the L2 up-check and L3 truth-matrix serving legs stay RED because of a # MEMBERSHIP COLD-FORMATION gap, not the snap-back fence (the fence # RECOVERING / "block master is recovering" errors are all downstream @@ -211,6 +201,8 @@ # ---------- my $cluster_conf = <start; $node1->_update_pid(1); +# spec-6.15b D6 flipped this test to online_join+xid_striping: formation now +# passes through the join gate, whose home-block rebuild window answers +# transiently with "block master is recovering ... retry is safe" (even at +# connection time). First-contact probes retry; semantic asserts stay strict. +sub psql_retry_ok +{ + my ($node, $sql, $tries) = @_; + $tries //= 120; + for (1 .. $tries) + { + my ($rc, $out) = $node->psql('postgres', $sql); + return (1, $out) if defined $rc && $rc == 0; + usleep(500_000); + } + return (0, undef); +} + my $n1_up = 0; for (1 .. 60) { @@ -262,16 +271,17 @@ # ---------- # L0: both alive + IC-connected. # ---------- -is($node0->safe_psql('postgres', 'SELECT 1'), '1', 'L0: node0 is up'); +my ($n0_up_rc, $n0_up_out) = psql_retry_ok($node0, 'SELECT 1'); +is($n0_up_out, '1', 'L0: node0 is up'); is($n1_up, 1, 'L0: node1 is up'); my $peer_ok = 0; for (1 .. 40) { - my $s = $node0->safe_psql('postgres', + my ($prc, $s) = psql_retry_ok($node0, "SELECT COALESCE(bool_or(heartbeat_recv_count > 0), false) " - . "FROM pg_cluster_ic_peers WHERE node_id = 1"); - if (defined $s && $s eq 't') { $peer_ok = 1; last; } + . "FROM pg_cluster_ic_peers WHERE node_id = 1", 1); + if ($prc && defined $s && $s eq 't') { $peer_ok = 1; last; } usleep(500_000); } is($peer_ok, 1, 'L0: node0 sees node1 heartbeats (IC connected)'); @@ -287,15 +297,17 @@ # CREATE t_crash_uncommitted (+ row, NO commit) -> 8.A leg # -> node1's cold merge must divert all of these. # ---------- -$node1->safe_psql('postgres', +my ($survivor_ok) = psql_retry_ok($node1, "CREATE TABLE t_survivor (id int PRIMARY KEY, note text);" . "INSERT INTO t_survivor VALUES (1, 'alive');"); +die 'node1 first DDL never succeeded' unless $survivor_ok; -$node0->safe_psql('postgres', +my ($keep_ok) = psql_retry_ok($node0, "CREATE TABLE t_keep (id int PRIMARY KEY, note text);" . "INSERT INTO t_keep VALUES (1, 'keep');" . "CREATE TABLE t_dropme (id int PRIMARY KEY, note text);" . "INSERT INTO t_dropme VALUES (1, 'doomed');"); +die 'node0 first DDL never succeeded' unless $keep_ok; # Shared-tree relfile of t_dropme, captured BEFORE the drop. my $drop_relpath = $node0->safe_psql('postgres', diff --git a/src/test/cluster_tap/t/346_shared_catalog_relmap_crash_2node.pl b/src/test/cluster_tap/t/346_shared_catalog_relmap_crash_2node.pl index 2391157fc1..26423537a2 100644 --- a/src/test/cluster_tap/t/346_shared_catalog_relmap_crash_2node.pl +++ b/src/test/cluster_tap/t/346_shared_catalog_relmap_crash_2node.pl @@ -164,6 +164,8 @@ # ---------- my $cluster_conf = <safe_psql('postgres', 'SELECT 1'), '1', 'R0: node0 is up'); +# Immediately after online_join + striping activation, a backend's bootstrap +# catalog read can transiently hit a peer whose block view is still rebuilding. +# Poll both nodes with the same tolerant shape used by t/339. +my $n0_up = 0; +for (1 .. 60) +{ + my ($rc) = $node0->psql('postgres', 'SELECT 1'); + if (defined $rc && $rc == 0) { $n0_up = 1; last; } + usleep(500_000); +} +is($n0_up, 1, 'R0: node0 is up'); is($n1_up, 1, 'R0: node1 is up'); $node0->safe_psql('postgres', diff --git a/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl b/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl new file mode 100644 index 0000000000..37f99b898f --- /dev/null +++ b/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl @@ -0,0 +1,576 @@ +#------------------------------------------------------------------------- +# +# 361_xid_authority_native_era_2node.pl +# spec-6.15b BUG-A1/A2 -- shared_catalog native-era XID authority. +# +# node1 is a cold pre-seed clone of node0: it has the same sysid and a +# pre-seed pg_control/pg_xact horizon, but it does not contain node0's +# cluster.enabled=off seed transactions. node0 then seeds shared_catalog, +# consumes native xids, cleanly shuts down, and publishes the sealed XID +# authority plus pg_xact prehistory. On first cluster boot node1 must use +# its per-node recovery anchor's pre-seed nextXid as the adoption proof, +# copy the native-era prehistory into local pg_xact before StartupCLOG, and +# max-merge nextXid with the sealed authority before ShmemVariableCache is +# seeded. +# +# Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# Portions Copyright (c) 2026, pgrac contributors +# +# Author: SqlRush +# +# IDENTIFICATION +# src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../../perl"; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::RecursiveCopy; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +if ($ENV{with_pgrac_cluster} && $ENV{with_pgrac_cluster} eq 'no') +{ + plan skip_all => 'shared catalog requires --enable-cluster'; +} + +sub make_shared_root +{ + my $root = PostgreSQL::Test::Utils::tempdir(); + mkdir "$root/global" or die "mkdir $root/global: $!"; + return $root; +} + +sub make_voting_disks +{ + my $disk_dir = PostgreSQL::Test::Utils::tempdir(); + my @disks; + for my $i (0 .. 2) + { + my $p = "$disk_dir/disk$i"; + open(my $fh, '>', $p) or die "open $p: $!"; + binmode $fh; + print $fh ("\0" x (128 * 512)); + close $fh; + push @disks, $p; + } + return join(',', @disks); +} + +sub cold_clone_data_dir +{ + my ($src, $dst) = @_; + + PostgreSQL::Test::RecursiveCopy::copypath( + $src->data_dir, + $dst->data_dir, + filterfn => sub { + my $rel = shift; + return ($rel ne 'log' && $rel ne 'postmaster.pid'); + }); + chmod(0700, $dst->data_dir) or die "chmod clone data dir: $!"; + + # The cold copy carries node0's port; append node1's port so the last + # setting wins, matching init_from_backup's post-copy rewrite. + $dst->append_conf('postgresql.conf', 'port = ' . $dst->port . "\n"); +} + +sub append_pgrac_conf +{ + my ($node, $name, $ic0, $ic1) = @_; + my $pgrac_conf = <data_dir . '/pgrac.conf', $pgrac_conf); +} + +sub start_background +{ + my ($node) = @_; + PostgreSQL::Test::Utils::system_log( + 'pg_ctl', '-W', '-D', $node->data_dir, + '-l', $node->logfile, '-o', '--cluster-name=' . $node->name, 'start'); +} + +sub wait_sql_eq +{ + my ($node, $sql, $want, $name, $attempts) = @_; + $attempts //= 80; + my $got = ''; + + for (1 .. $attempts) + { + my ($rc, $out) = $node->psql('postgres', $sql); + if (defined $rc && $rc == 0 && defined $out) + { + $got = $out; + last if $got eq $want; + } + usleep(250_000); + } + is($got, $want, $name); +} + +sub read_xid_authority +{ + my ($shared_root) = @_; + my $path = "$shared_root/global/pgrac_xid_authority"; + open(my $fh, '<:raw', $path) or die "open $path: $!"; + read($fh, my $buf, 40) or die "read $path: $!"; + close $fh; + my ($magic, $version, $flags, $reserved, $native_hw, $next_multi, $crc) = + unpack('L< L< L< L< Q< Q< L<', $buf); + return { + magic => $magic, + version => $version, + flags => $flags, + native_hw => $native_hw, + next_multi => $next_multi, + crc => $crc, + }; +} + +sub corrupt_file +{ + my ($path) = @_; + open(my $fh, '>:raw', $path) or die "open $path: $!"; + print $fh "not-a-valid-xid-authority"; + close $fh; +} + +sub append_common_shared_catalog_conf +{ + my ($node, $shared_root) = @_; + $node->append_conf('postgresql.conf', <append_conf('postgresql.conf', <new('sxid_no_stripe'); + + $n->init; + append_common_shared_catalog_conf($n, $shared_root); + $n->append_conf('postgresql.conf', <start(fail_ok => 1); + ok($start_failed, 'D6: shared_catalog multi-node refuses cluster.xid_striping=off'); + like(PostgreSQL::Test::Utils::slurp_file($n->logfile), + qr/requires cluster\.xid_striping/, + 'D6: startup log names the xid_striping requirement'); +} + +# ------------------------------------------------------------------------- +# Main green path: pre-seed clone adopts node0 native-era pg_xact truth. +# ------------------------------------------------------------------------- +my $shared_root = make_shared_root(); +my $disks_csv = make_voting_disks(); +my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); +my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); + +my $node0 = PostgreSQL::Test::Cluster->new('sxid_node0'); +$node0->init(allows_streaming => 1); +$node0->start; +$node0->safe_psql('postgres', 'CHECKPOINT'); +$node0->stop; + +my $node1 = PostgreSQL::Test::Cluster->new('sxid_node1'); +cold_clone_data_dir($node0, $node1); +# F3: real scram TCP login leg needs a host auth line AND a TCP listener on +# the joiner (test nodes default to unix sockets only). +PostgreSQL::Test::Utils::append_to_file($node1->data_dir . '/pg_hba.conf', + "host all sxid_login 127.0.0.1/32 scram-sha-256\n"); +$node1->append_conf('postgresql.conf', "listen_addresses = '127.0.0.1'\n"); + +append_common_shared_catalog_conf($node0, $shared_root); +$node0->append_conf('postgresql.conf', <start; +ok(-e "$shared_root/global/pgrac_catalog_authority", + 'G1: node0 seeded the shared catalog authority marker'); +ok(-e "$shared_root/global/pg_control", + 'G1: node0 seeded the shared pg_control authority'); +ok(-e "$shared_root/global/pgrac_xid_authority", + 'G1: node0 seeded the shared XID authority before native-era load'); + +my $abort_xid = $node0->safe_psql('postgres', q{ +BEGIN; +SELECT txid_current(); +CREATE SCHEMA sxid_abort_shadow; +ROLLBACK; +}); + +$node0->safe_psql('postgres', q{ +CREATE SCHEMA sxid; +CREATE ROLE sxid_login LOGIN PASSWORD 'sxidpass'; +}); + +my $seed_schema_xid = $node0->safe_psql('postgres', + q{SELECT xmin::text FROM pg_namespace WHERE nspname = 'sxid'}); +my $seed_role_xid = $node0->safe_psql('postgres', + q{SELECT xmin::text FROM pg_authid WHERE rolname = 'sxid_login'}); + +is($node0->safe_psql('postgres', "SELECT txid_status($seed_schema_xid)"), + 'committed', 'G1: seed schema catalog xid is committed on the seed'); +is($node0->safe_psql('postgres', "SELECT txid_status($abort_xid)"), + 'aborted', 'G1: native-era aborted xid is aborted on the seed'); + +$node0->stop; + +my $auth = read_xid_authority($shared_root); +ok(($auth->{flags} & 0x0001) != 0, 'G2: clean seed shutdown sealed the XID authority'); +cmp_ok($auth->{native_hw}, '>', $seed_schema_xid, + 'G2: native high-water is above the seed schema catalog xid'); +cmp_ok($auth->{native_hw}, '>', $seed_role_xid, + 'G2: native high-water is above the seed role xid'); +ok(-e "$shared_root/global/pgrac_xid_prehistory", + 'G2: clean seed shutdown published XID prehistory'); + +append_strict_two_node_conf($node0, $disks_csv, undef); +$node0->append_conf('postgresql.conf', "cluster.node_id = 0\n"); + +append_common_shared_catalog_conf($node1, $shared_root); +append_strict_two_node_conf($node1, $disks_csv, undef); +$node1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + +append_pgrac_conf($node0, 'sxid', $ic0, $ic1); +append_pgrac_conf($node1, 'sxid', $ic0, $ic1); + +start_background($node1); +$node0->start; +$node1->_update_pid(1); + +wait_sql_eq($node1, 'SELECT 1', '1', 'G3: node1 is up after XID prehistory adopt'); +wait_sql_eq($node0, + "SELECT COALESCE(bool_or(heartbeat_recv_count > 0), false) " + . "FROM pg_cluster_ic_peers WHERE node_id = 1", + 't', 'G3: node0 sees node1 heartbeats'); + +wait_sql_eq($node1, + q{SELECT count(*) FROM pg_namespace WHERE nspname = 'sxid'}, + '1', 'G4: node1 sees the native-era seed schema catalog row'); +wait_sql_eq($node1, + q{SELECT count(*) FROM pg_authid WHERE rolname = 'sxid_login'}, + '1', 'G4: node1 sees the seed role catalog row'); +wait_sql_eq($node1, + q{SET ROLE sxid_login; SELECT current_user}, + 'sxid_login', 'G4: node1 can resolve and SET ROLE to the seed role'); +# F3: the seed role's scram password must survive adoption -- a real TCP +# login exercises rolpassword visibility at connection time. +$node1->connect_ok( + 'host=127.0.0.1 port=' . $node1->port + . ' user=sxid_login password=sxidpass dbname=postgres', + 'G4: seed role logs in over scram TCP on node1'); + +wait_sql_eq($node1, "SELECT txid_status($seed_schema_xid)", 'committed', + 'G5: node1 txid_status sees the native-era committed schema xid'); +wait_sql_eq($node1, "SELECT txid_status($seed_role_xid)", 'committed', + 'G5: node1 txid_status sees the native-era committed role xid'); +wait_sql_eq($node1, "SELECT txid_status($abort_xid)", 'aborted', + 'G5: node1 txid_status sees the native-era aborted xid'); +is($node0->safe_psql('postgres', "SELECT txid_status($abort_xid)"), + 'aborted', 'G5: node0 still sees the aborted xid after node1 reads'); +# F4: the poison-stamp proof must re-read the shared HEAP rows on node0 -- +# a wrong HEAP_XMIN_INVALID hint written by node1 lives in the shared +# catalog block, not in node0's CLOG. +is($node0->safe_psql('postgres', + q{SELECT count(*) FROM pg_namespace WHERE nspname = 'sxid'}), + '1', 'G5: node0 still sees the seed schema row after node1 scans (no poison stamp)'); +is($node0->safe_psql('postgres', + q{SELECT count(*) FROM pg_authid WHERE rolname = 'sxid_login'}), + '1', 'G5: node0 still sees the seed role row after node1 scans (no poison stamp)'); + +my $state_hw = $node1->safe_psql('postgres', q{ +SELECT value FROM pg_cluster_state + WHERE category = 'catalog' AND key = 'xid_authority_native_hw'}); +my $state_sealed = $node1->safe_psql('postgres', q{ +SELECT value FROM pg_cluster_state + WHERE category = 'catalog' AND key = 'xid_authority_sealed'}); +my $state_adopted = $node1->safe_psql('postgres', q{ +SELECT value FROM pg_cluster_state + WHERE category = 'catalog' AND key = 'xid_prehistory_adopted'}); + +is($state_hw, "$auth->{native_hw}", 'G6: pg_cluster_state exposes authority native high-water'); +is($state_sealed, 't', 'G6: pg_cluster_state exposes sealed XID authority'); +is($state_adopted, 't', 'G6: pg_cluster_state exposes node1 prehistory adoption'); +like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), + qr/adopted XID prehistory through native high-water/, + 'G6: node1 log records XID prehistory adoption'); +like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), + qr/merged nextXid with XID authority native high-water/, + 'G6: node1 log records StartupXLOG nextXid max-merge'); + +$node1->stop; +$node0->stop; + +# ------------------------------------------------------------------------- +# Negative lifecycle legs on the formed shared tree. +# ------------------------------------------------------------------------- +$node0->append_conf('postgresql.conf', <start(fail_ok => 1); +ok($reentry_failed, 'N1: cluster.enabled=off cannot re-enter native seed era after formation'); +like(PostgreSQL::Test::Utils::slurp_file($node0->logfile), + qr/native seed era cannot be re-entered/, + 'N1: startup log names native-era re-entry refusal'); + +corrupt_file("$shared_root/global/pgrac_xid_authority"); +corrupt_file("$shared_root/global/pgrac_xid_authority.bak"); +# F9: pin the 53RB5 SQLSTATE contract on one leg (startup FATALs surface in +# the server log only, so %e in log_line_prefix carries the SQLSTATE; the +# other negative legs deliberately match message text). +# %e must precede %q: the refusal FATAL comes from the postmaster (a +# non-session process), and %q stops expansion there. +$node1->append_conf('postgresql.conf', + "log_line_prefix = '%m [%p] %e %q%a '\n"); +my $corrupt_failed = !$node1->start(fail_ok => 1); +ok($corrupt_failed, 'N2: corrupt shared XID authority fails closed'); +like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), + qr/shared XID authority is unavailable|present but corrupt|cluster_xid_authority_unavailable/, + 'N2: startup log names corrupt/unavailable XID authority'); +like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), + qr/53RB5/, 'N2: the refusal carries SQLSTATE 53RB5'); + +# ------------------------------------------------------------------------- +# Unsealed authority negative: seed startup wrote the authority, but the seed +# crashed before clean shutdown could publish prehistory and seal it. +# ------------------------------------------------------------------------- +{ + my $u_shared = make_shared_root(); + my $u_disks = make_voting_disks(); + my $u_ic0 = PostgreSQL::Test::Cluster::get_free_port(); + my $u_ic1 = PostgreSQL::Test::Cluster::get_free_port(); + my $u0 = PostgreSQL::Test::Cluster->new('sxid_unsealed_node0'); + my $u1 = PostgreSQL::Test::Cluster->new('sxid_unsealed_node1'); + + $u0->init(allows_streaming => 1); + $u0->start; + $u0->safe_psql('postgres', 'CHECKPOINT'); + $u0->stop; + cold_clone_data_dir($u0, $u1); + + append_common_shared_catalog_conf($u0, $u_shared); + $u0->append_conf('postgresql.conf', <start; + ok(-e "$u_shared/global/pgrac_xid_authority", + 'N3: unsealed fixture created the authority during seed startup'); + $u0->stop('immediate'); + + append_common_shared_catalog_conf($u1, $u_shared); + append_strict_two_node_conf($u1, $u_disks, undef); + $u1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + append_pgrac_conf($u1, 'sxid_unsealed', $u_ic0, $u_ic1); + + my $unsealed_failed = !$u1->start(fail_ok => 1); + ok($unsealed_failed, 'N3: joiner refuses an unsealed XID authority'); + like(PostgreSQL::Test::Utils::slurp_file($u1->logfile), + qr/shared XID authority is not sealed|shared XID authority is unavailable/, + 'N3: startup log names the unsealed authority'); +} + +# ------------------------------------------------------------------------- +# Multi-pass seed negative: a SECOND native-era run unseals the authority at +# boot, so crashing that run leaves joiners fail-closed instead of silently +# adopting the first pass's stale high-water (spec-6.15b §3.1 multi-pass arm). +# ------------------------------------------------------------------------- +{ + my $m_shared = make_shared_root(); + my $m_disks = make_voting_disks(); + my $m_ic0 = PostgreSQL::Test::Cluster::get_free_port(); + my $m_ic1 = PostgreSQL::Test::Cluster::get_free_port(); + my $m0 = PostgreSQL::Test::Cluster->new('sxid_multipass_node0'); + my $m1 = PostgreSQL::Test::Cluster->new('sxid_multipass_node1'); + + $m0->init(allows_streaming => 1); + $m0->start; + $m0->safe_psql('postgres', 'CHECKPOINT'); + $m0->stop; + cold_clone_data_dir($m0, $m1); + + append_common_shared_catalog_conf($m0, $m_shared); + $m0->append_conf('postgresql.conf', < sealed + $m0->start; + $m0->safe_psql('postgres', 'CREATE SCHEMA sxid_pass1;'); + $m0->stop; + my $m_auth = read_xid_authority($m_shared); + ok(($m_auth->{flags} & 0x0001) != 0, + 'N4: first native pass sealed the XID authority'); + + # pass 2: booting a follow-up native run re-opens (unseals) the era + $m0->start; + $m_auth = read_xid_authority($m_shared); + ok(($m_auth->{flags} & 0x0001) == 0, + 'N4: follow-up native pass unsealed the XID authority at boot'); + like(PostgreSQL::Test::Utils::slurp_file($m0->logfile), + qr/re-opened native seed era/, + 'N4: seed log names the re-opened native era'); + + # crash pass 2: the stale pass-1 high-water must NOT become adoptable + $m0->stop('immediate'); + append_common_shared_catalog_conf($m1, $m_shared); + append_strict_two_node_conf($m1, $m_disks, undef); + $m1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + append_pgrac_conf($m1, 'sxid_multipass', $m_ic0, $m_ic1); + + my $multipass_failed = !$m1->start(fail_ok => 1); + ok($multipass_failed, + 'N4: joiner refuses adoption after a crashed follow-up native pass'); + like(PostgreSQL::Test::Utils::slurp_file($m1->logfile), + qr/shared XID authority is not sealed|shared XID authority is unavailable/, + 'N4: startup log fails closed on the unsealed authority'); +} + +# ------------------------------------------------------------------------- +# Divergent-lineage negative (review F2 / spec Q6 amendment): a same-sysid +# clone that ran STANDALONE after cloning holds its own outcomes in the +# native xid range; neither skipping nor adopting is sound -> FATAL 53RB5. +# ------------------------------------------------------------------------- +{ + my $d_shared = make_shared_root(); + my $d_disks = make_voting_disks(); + my $d_ic0 = PostgreSQL::Test::Cluster::get_free_port(); + my $d_ic1 = PostgreSQL::Test::Cluster::get_free_port(); + my $d0 = PostgreSQL::Test::Cluster->new('sxid_diverge_node0'); + my $d1 = PostgreSQL::Test::Cluster->new('sxid_diverge_node1'); + + $d0->init(allows_streaming => 1); + $d0->start; + $d0->safe_psql('postgres', 'CHECKPOINT'); + $d0->stop; + cold_clone_data_dir($d0, $d1); + + # d1 diverges: a plain standalone run committing 30 transactions writes + # d1's own outcomes into the xid range the seed is about to consume. + $d1->start; + $d1->safe_psql('postgres', + join('', map { "BEGIN; CREATE TABLE d_t$_ (i int); COMMIT;\n" } 1 .. 30)); + $d1->stop; + + # d0 seeds: one aborted xid guarantees a byte-level contradiction with + # d1's straight-committed overlap. + append_common_shared_catalog_conf($d0, $d_shared); + $d0->append_conf('postgresql.conf', <start; + $d0->safe_psql('postgres', q{ +BEGIN; +CREATE SCHEMA d_abort_shadow; +ROLLBACK; +}); + $d0->safe_psql('postgres', 'CREATE SCHEMA d_seed;'); + $d0->stop; + + append_common_shared_catalog_conf($d1, $d_shared); + append_strict_two_node_conf($d1, $d_disks, undef); + $d1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + append_pgrac_conf($d1, 'sxid_diverge', $d_ic0, $d_ic1); + + my $diverge_failed = !$d1->start(fail_ok => 1); + ok($diverge_failed, 'N5: divergent-lineage joiner is refused'); + like(PostgreSQL::Test::Utils::slurp_file($d1->logfile), + qr/divergent native-era lineage/, + 'N5: startup log names the divergent lineage'); +} + +# ------------------------------------------------------------------------- +# L9 dormant leg: with cluster.shared_catalog=off nothing of the XID +# authority machinery may touch the shared tree. +# ------------------------------------------------------------------------- +{ + my $o_shared = make_shared_root(); + my $o0 = PostgreSQL::Test::Cluster->new('sxid_off_node0'); + + $o0->init(allows_streaming => 1); + $o0->append_conf('postgresql.conf', <start; + $o0->safe_psql('postgres', 'CREATE SCHEMA off_mode_probe;'); + $o0->stop; + ok(!-e "$o_shared/global/pgrac_xid_authority", + 'L9: shared_catalog=off never creates the XID authority'); + ok(!-e "$o_shared/global/pgrac_xid_prehistory", + 'L9: shared_catalog=off never creates the XID prehistory'); +} + +done_testing(); diff --git a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl new file mode 100644 index 0000000000..75747a223c --- /dev/null +++ b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl @@ -0,0 +1,522 @@ +#------------------------------------------------------------------------- +# +# 362_shared_catalog_4node_kill_selfheal.pl +# spec-4.6a D9 -- 4-node shared_catalog fail-stop self-heal. +# +# Formation uses ClusterQuad(shared_catalog + wal_threads_root + shared_data) +# so a killed node's HW authority is recoverable from its per-thread WAL. +# Amendment v1.2 (R1): in this out-of-scope deployment (4 nodes, online +# thread recovery off) the dead master's PRE-EXISTING writes stay +# fail-closed until the failed node restarts — its unproven blocks (bounded +# 53R9L) and equally its transaction verdicts (TT status unknown for its +# xids, the visibility door). Reads and DDL are asserted on the honest +# two-outcome contract (success or explicit SQLSTATE, never a hang); +# new-object DDL/writes prove P7 unfreeze. L4h (r3-P1-1) is the serve-gate +# LOCK: a fresh backend's wide catalog scan on a cold survivor MUST fail +# 53R9L (no success arm) while the dead node is down. +# After the kill legs start there is no SKIP path: BLOCKED_STRUCTURAL means +# the test harness is misconfigured, and lack of convergence is a real FAIL. +# +# Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# Portions Copyright (c) 2026, pgrac contributors +# +# Author: SqlRush +# +# IDENTIFICATION +# src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl +# Portions Copyright (c) 2026, pgrac contributors +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../../perl"; + +use PostgreSQL::Test::ClusterQuad; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +if ($ENV{with_pgrac_cluster} && $ENV{with_pgrac_cluster} eq 'no') +{ + plan skip_all => 'shared_catalog 4-node kill self-heal requires --enable-cluster'; +} + +sub poll_until +{ + my ($node, $query, $want, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + my $last = ''; + while (time() < $deadline) + { + $last = eval { $node->safe_psql('postgres', $query) }; + return 1 if defined $last && $last eq $want; + usleep(300_000); + } + diag("poll_until timed out: '$query' last='" . ($last // '(undef)') + . "' want='$want'"); + return 0; +} + +sub retry_sql +{ + my ($node, $sql, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + my $last = ''; + while (time() < $deadline) + { + my ($rc, $out, $err) = $node->psql('postgres', $sql, timeout => 60); + return (1, $out // '') if defined $rc && $rc == 0; + $last = $err // ''; + usleep(500_000); + } + return (0, $last); +} + +sub state_counter +{ + my ($node, $cat, $key) = @_; + my $v = eval { + $node->safe_psql('postgres', + "SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'") + }; + return defined $v && $v ne '' ? $v + 0 : 0; +} + +sub sum_hw +{ + my ($quad, $key, @idx) = @_; + my $sum = 0; + for my $i (@idx) + { + $sum += state_counter($quad->node($i), 'hw', $key); + } + return $sum; +} + +# Amendment v1.2 (R1): after the kill, a NEW backend's parse may cold-read a +# dead-master catalog page and honestly fail closed (53R9L) until the failed +# node returns. The convergence observability therefore rides PRE-WARMED +# long-lived psql sessions (catcache hot; pg_cluster_* reads touch shmem, not +# dead-master blocks), while fresh-connection legs assert the two-outcome +# contract. bg_query returns (1, value) or (0, error-ish) without dying. +sub bg_query +{ + my ($bg, $sql) = @_; + my $out = eval { $bg->query($sql) }; + return (0, $@ // 'query failed') if $@; + return (1, $out // ''); +} + +sub bg_counter +{ + my ($bg, $cat, $key) = @_; + my ($ok, $v) = bg_query($bg, + "SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'"); + return ($ok && defined $v && $v ne '') ? $v + 0 : 0; +} + +sub bg_sum +{ + my ($bgs, $cat, $key, @idx) = @_; + my $sum = 0; + $sum += bg_counter($bgs->{$_}, $cat, $key) for @idx; + return $sum; +} + +sub bg_poll_until +{ + my ($bg, $sql, $want, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + my $last = ''; + while (time() < $deadline) + { + my ($ok, $out) = bg_query($bg, $sql); + $last = $out; + return 1 if $ok && defined $out && $out eq $want; + usleep(500_000); + } + diag("bg_poll_until timed out: '$sql' last='$last' want='$want'"); + return 0; +} + +sub logs_contain +{ + my ($quad, $pattern, @idx) = @_; + for my $i (@idx) + { + my $log = eval { PostgreSQL::Test::Utils::slurp_file($quad->node($i)->logfile) } // ''; + return 1 if $log =~ /$pattern/; + } + return 0; +} + +sub log_until +{ + my ($quad, $idx, $pattern, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + while (time() < $deadline) + { + my $log = eval { PostgreSQL::Test::Utils::slurp_file($quad->node($idx)->logfile) } // ''; + return 1 if $log =~ /$pattern/; + usleep(300_000); + } + diag("log_until timed out on node$idx waiting for /$pattern/"); + return 0; +} + +sub startup_env_blocker +{ + my ($quad) = @_; + my $log = ''; + if ($quad) + { + for my $i (0 .. 3) + { + $log .= eval { PostgreSQL::Test::Utils::slurp_file($quad->node($i)->logfile) } // ''; + } + } + return $log =~ /could not create shared memory segment|No space left on device|No space left|SHMMNI|semget/i; +} + +my $quad = eval { + PostgreSQL::Test::ClusterQuad->new_quad( + 'sc4_selfheal', + shared_catalog => 1, + quorum_voting_disks => 3, + extra_conf => [ + 'autovacuum = off', + 'cluster.grd_max_entries = 4096', + 'cluster.lms_enabled = on', + 'cluster.ges_request_timeout_ms = 3000', + 'cluster.ges_retransmit_max_attempts = 0', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 8', + ]); +}; +if (!$quad) +{ + my $err = $@ || 'unknown constructor failure'; + plan skip_all => "L1 shared_catalog ClusterQuad formation hit local substrate blocker: $err" + if $err =~ /could not create shared memory|No space left|SHMMNI|semget/i; + BAIL_OUT("ClusterQuad(shared_catalog) constructor failed: $err"); +} + +my $started = 1; +for my $i (1 .. 3) +{ + PostgreSQL::Test::Utils::system_log( + 'pg_ctl', '-W', '-D', $quad->node($i)->data_dir, + '-l', $quad->node($i)->logfile, + '-o', "--cluster-name=sc4_selfheal_node$i", 'start'); +} +unless ($quad->node(0)->start(fail_ok => 1)) +{ + $started = 0; +} +for my $i (1 .. 3) +{ + $quad->node($i)->_update_pid(-1); +} +if (!$started) +{ + if (startup_env_blocker($quad)) + { + eval { $quad->stop_quad; }; + plan skip_all => 'L1 shared_catalog ClusterQuad formation hit local shared-memory/resource limit'; + } + my $fatal = logs_contain($quad, qr/cluster\.shared_catalog requires cluster\.wal_threads_dir|structurally blocked|BLOCKED_STRUCTURAL/i, 0 .. 3); + eval { $quad->stop_quad; }; + ok(!$fatal, 'L1: shared_catalog quad must not hit wal_threads_dir/BLOCKED_STRUCTURAL configuration failure'); + BAIL_OUT('ClusterQuad(shared_catalog) failed to start for a non-environment reason'); +} + +# L1: all four members formed with quorum and per-thread WAL configured. +for my $i (0 .. 3) +{ + ok(poll_until($quad->node($i), 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 60), + "L1: node$i reached quorum"); + is(state_counter($quad->node($i), 'hw', 'remaster_recoverable'), 1, + "L1: node$i reports HW remaster recoverable"); +} +for my $to (1 .. 3) +{ + ok($quad->wait_for_peer_state(0, $to, 'connected', 60), + "L1: node0 sees node$to connected"); +} + +my $n0 = $quad->node0; +my $dead = 3; +my @survivors = (0, 1, 2); + +# Seed shared-catalog DDL and force node3 to own real HW state before death. +for my $t (1 .. 2) +{ + my ($ddl_ok, $ddl_res) = retry_sql($n0, + "CREATE TABLE sc4_hw_$t (id int, pad int)", 60); + ok($ddl_ok, "L1: coordinator created shared_catalog table sc4_hw_$t") + or diag($ddl_res); + my ($ok, $res) = retry_sql($quad->node($dead), + "INSERT INTO sc4_hw_$t SELECT g, g FROM generate_series(1,5000) g", 60); + ok($ok, "L1: node$dead extended shared_catalog table sc4_hw_$t before kill") + or diag($res); +} +for my $i (0 .. 3) +{ + retry_sql($quad->node($i), 'CHECKPOINT', 30); +} + +# Amendment v1.2 (R1): pre-warmed observability sessions (see helper note). +my %bg; +for my $i (@survivors) +{ + $bg{$i} = $quad->node($i)->background_psql('postgres', on_error_stop => 0); + bg_query($bg{$i}, "SELECT value FROM pg_cluster_state WHERE category='hw' AND key='remaster_done_count'"); + bg_query($bg{$i}, "SELECT count(*) FROM pg_cluster_grd_shards WHERE master_node_id=$dead"); + bg_query($bg{$i}, 'SELECT txid_current()'); +} + +my $done_before = bg_sum(\%bg, 'hw', 'remaster_done_count', @survivors); +my $blocked_before = bg_sum(\%bg, 'hw', 'remaster_blocked_count', @survivors); +my $retry_before = bg_sum(\%bg, 'hw', 'remaster_retry_count', @survivors); +my $cleanup_before = bg_sum(\%bg, 'pcm', 'dead_cleanup_entries', @survivors); +my %grd_done_before; +$grd_done_before{$_} = bg_counter($bg{$_}, 'grd_recovery', 'remaster_done') + for @survivors; + +# L2: kill node3 and require every survivor to see the real DEAD edge. +# Use the CSSD log instead of a SQL view: during fail-closed GRD recovery, +# ordinary catalog reads may legitimately return 53R9I until the barrier opens. +$quad->kill_node9($dead); +for my $i (@survivors) +{ + ok(log_until($quad, $i, qr/cssd: peer $dead transitioned .* DEAD/, 45), + "L2: survivor node$i declared node$dead dead"); +} + +# L3: GRD leaves no shard mastered by the dead node, and HW remaster converges. +# The map read is hard-asserted on the warm COORDINATOR session only: the view +# scan on a cold survivor deterministically cold-reads a dead-master catalog +# page out of the small test buffer pool and honestly fails closed (53R9L) +# until node3 returns — the map itself is a deterministic recompute barriered +# by P6, and each cold survivor is instead hard-asserted on its own HW +# remaster worker DONE log line below. +ok(bg_poll_until($bg{0}, + "SELECT (count(*)=0)::text FROM pg_cluster_grd_shards WHERE master_node_id=$dead", + 'true', 60), + 'L3: coordinator remastered all GRD shards off node3 (warm session)'); +for my $i (@survivors) +{ + ok(log_until($quad, $i, qr/HW remaster worker: dead node $dead -> done/, 90), + "L3: survivor node$i HW remaster worker reported done"); +} +my $done_converged = 0; +my $deadline = time() + 90; +while (time() < $deadline) +{ + if (bg_sum(\%bg, 'hw', 'remaster_done_count', @survivors) > $done_before) + { + $done_converged = 1; + last; + } + usleep(500_000); +} +ok($done_converged, 'L3: HW remaster DONE advanced on at least one survivor after kill'); +ok(!logs_contain($quad, qr/blocked_structural|structurally blocked|BLOCKED_STRUCTURAL/i, @survivors), + 'L3: no survivor reported BLOCKED_STRUCTURAL after kill'); + +# L6: if a transient BLOCKED occurred, a same-episode retry must also have run. +my $blocked_after = bg_sum(\%bg, 'hw', 'remaster_blocked_count', @survivors); +if ($blocked_after > $blocked_before) +{ + ok(bg_sum(\%bg, 'hw', 'remaster_retry_count', @survivors) > $retry_before, + 'L6: transient BLOCKED was followed by same-episode HW remaster retry'); +} +else +{ + pass('L6: no transient HW BLOCKED occurred on the clean 4-node path'); +} + +# L4h (r3-P1-1 hard leg): with the serve gate in place, an out-of-scope +# formation (4 nodes / online_thread_recovery off) NEVER publishes the dead +# node's materialization proof, so EVERY node3-mastered page stays RECOVERING +# until node3 returns — and node3 is never restarted in this test. A wide +# catalog scan from a FRESH backend on a cold survivor therefore MUST fail +# closed with 53R9L: pg_class + pg_attribute span dozens of pages hashed ~1/4 +# to node3, and a fresh backend's full scan cold-reads pages no prior session +# pulled into the 16MB test pool. Deliberately NO success arm — a NORMAL +# serve of a dead-master page here is the exact 8.A regression this leg locks +# (reverting the serve-gate fix turns this leg red while the two-outcome legs +# stay green). Gate on the survivor's OWN episode reaching IDLE first so the +# failure surface is deterministically the post-episode materialization proof +# (53R9L), never the in-episode shard freeze (53R9I). The scan touches no +# node3-written rows (node3 ran only heap INSERTs, no DDL), so the TT +# visibility door cannot fire first: 53R9L is the ONLY legal outcome. +{ + my $cold = 2; + + # Compare in Perl, not SQL: the warm session's catcache covers exactly the + # bg_counter query shape it ran pre-kill; a cast/operator added in SQL + # would itself cold-read a dead-master catalog page and trip 53R9L. + my $idle = 0; + my $idle_deadline = time() + 90; + while (time() < $idle_deadline) + { + if (bg_counter($bg{$cold}, 'grd_recovery', 'remaster_done') + > $grd_done_before{$cold}) + { + $idle = 1; + last; + } + usleep(500_000); + } + ok($idle, "L4h: survivor node$cold GRD recovery episode reached IDLE (warm session)"); + + my $t0 = time(); + my ($rc, $out, $err) = $quad->node($cold)->psql('postgres', + 'SELECT count(*) FROM pg_class c JOIN pg_attribute a ON a.attrelid = c.oid', + timeout => 60); + my $elapsed = time() - $t0; + isnt(defined $rc ? $rc : 0, 0, + "L4h: fresh wide catalog scan on survivor node$cold MUST fail while node$dead is down"); + like($err // '', + qr/block-level cache protocol state is being rebuilt after reconfiguration/, + 'L4h: the failure is the explicit 53R9L dead-master fail-closed gate'); + cmp_ok($elapsed, '<', 60, 'L4h: fail-closed scan returned bounded (no hang)'); + diag("L4h: scan failed closed as required: " . ($err // '(no stderr)')) + if defined $err && $err ne ''; +} + +# L4a (Amendment v1.2 R1): reading the dead master's PRE-EXISTING blocks in an +# out-of-scope deployment (online_thread_recovery off / 4 nodes) must be a +# BOUNDED, EXPLICIT fail-closed error — never a silent stale read, never a +# hang. sc4_hw_* spans ~20+ pages, so some page's static master is node3 with +# overwhelming probability; the redo-coverage proof cannot be published without +# online thread recovery, so the scan hits 53R9L (resource recovering). If by +# hash luck no scanned page is node3-mastered the scan succeeds — both are the +# honest two-outcome contract; only a hang/timeout or an unexpected error fails. +for my $t (1 .. 2) +{ + my $t0 = time(); + my ($rc, $out, $err) = $quad->node(1)->psql('postgres', + "SELECT count(*) FROM sc4_hw_$t", timeout => 60); + my $elapsed = time() - $t0; + if (defined $rc && $rc == 0) + { + ok(1, "L4a: dead-master table sc4_hw_$t read succeeded (no node$dead-mastered page scanned)"); + diag("L4a: sc4_hw_$t count=$out"); + } + else + { + like($err // '', + qr/being rebuilt after reconfiguration|resource recovering|53R9L|cluster TT status unknown/i, + "L4a: dead-master table sc4_hw_$t read fail-closed with an explicit SQLSTATE"); + } + cmp_ok($elapsed, '<', 60, "L4a: sc4_hw_$t read returned bounded (no hang)"); +} + +# L4: survivor SQL recovers after fail-stop — hard-asserted on the pre-warmed +# sessions; a FRESH connection is the honest two-outcome contract (v1.2 R1): +# success, or an explicit fail-closed SQLSTATE on a dead-master catalog page — +# never a hang. +for my $i (@survivors) +{ + my $got = 0; + my $dl = time() + 60; + while (time() < $dl) + { + my ($ok, $out) = bg_query($bg{$i}, 'SELECT txid_current()'); + if ($ok && defined $out && $out =~ /^\d+$/) { $got = 1; last; } + usleep(500_000); + } + ok($got, "L4: survivor node$i accepts txid_current after reconfig (warm session)"); + + my $t0 = time(); + my ($rc, $out, $err) = $quad->node($i)->psql('postgres', 'SELECT txid_current()', + timeout => 60); + if (defined $rc && $rc == 0) + { + ok(1, "L4: fresh connection txid_current succeeded on node$i"); + } + else + { + like($err // '', + qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown|write-fenced/i, + "L4: fresh connection on node$i fail-closed with an explicit SQLSTATE"); + } + cmp_ok(time() - $t0, '<', 60, "L4: fresh connection probe on node$i returned bounded"); +} +my ($ddl_rc, $ddl_out, $ddl_err) = $n0->psql('postgres', + 'CREATE TABLE sc4_after_kill (id int)', timeout => 60); +my $ddl_ok = defined $ddl_rc && $ddl_rc == 0; +if ($ddl_ok) +{ + ok(1, 'L4: coordinator survivor DDL succeeded after node kill'); +} +else +{ + # write-fenced (53R51, spec-4.12): a transient lease/epoch fence on the + # writer right after the reconfig — an explicit, retryable fail-closed + # rejection, squarely inside the honest two-outcome contract. + like($ddl_err // '', + qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown|write-fenced/i, + 'L4: coordinator DDL fail-closed with an explicit SQLSTATE (dead-master catalog block)'); + diag("L4: DDL fail-closed honestly: $ddl_err"); +} +for my $i (1, 2) +{ + if ($ddl_ok) + { + ok(poll_until($quad->node($i), + "SELECT (count(*)=1)::text FROM pg_class WHERE relname='sc4_after_kill'", + 'true', 60), + "L4: survivor node$i sees post-kill shared-catalog DDL"); + } + else + { + ok(1, "L4: survivor node$i visibility leg not applicable (DDL fail-closed honestly)"); + } +} + +# L4 (spec-4.6a D12, Amendment v1.2 R3): node3 held real PCM/HW state at kill +# time (it extended sc4_hw_1/2 with 5000 rows each), so the dead-node residue +# cleanup MUST account a positive delta across the survivors — a >= 0 +# assertion would stay green with D12 entirely disabled. +{ + my $cleanup_total = 0; + for my $i (@survivors) + { + my ($ok, $n) = bg_query($bg{$i}, + "SELECT count(*) FROM pg_cluster_state WHERE category='pcm' AND key='dead_cleanup_entries'"); + ok($ok && defined $n && $n eq '1', + "L4: survivor node$i exposes the pcm/dead_cleanup_entries key"); + $cleanup_total += bg_counter($bg{$i}, 'pcm', 'dead_cleanup_entries'); + } + cmp_ok($cleanup_total - $cleanup_before, '>', 0, + 'L4: dead-node PCM residue cleanup accounted a positive delta after kill'); + diag("L4: pcm/dead_cleanup_entries delta across survivors = " + . ($cleanup_total - $cleanup_before)); +} + +# L5: watchdog counters are allowed to be zero, but the KEYS must exist +# (state_counter reads a missing key as 0, so presence is the real assertion). +for my $i (@survivors) +{ + my ($ok1, $n1) = bg_query($bg{$i}, + "SELECT count(*) FROM pg_cluster_state WHERE category='grd_recovery' AND key='cluster_gate_timeout'"); + my ($ok2, $n2) = bg_query($bg{$i}, + "SELECT count(*) FROM pg_cluster_state WHERE category='grd_recovery' AND key='wait_epoch_escape'"); + ok($ok1 && $ok2 && defined $n1 && $n1 eq '1' && defined $n2 && $n2 eq '1', + "L5: survivor node$i exposes WAIT_CLUSTER/WAIT_EPOCH watchdog counter keys"); +} + +$bg{$_}->quit for @survivors; + +$quad->stop_quad; +done_testing(); diff --git a/src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl b/src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl new file mode 100644 index 0000000000..005c0d4a4b --- /dev/null +++ b/src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl @@ -0,0 +1,191 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 363_cluster_2_29a_marker_async_liveness.pl +# BUG-C1 / spec-2.29a — qvotec marker backlog must not park LMON. +# +# The hold injection freezes qvotec marker completion until released. Before +# BUG-C1, LMON synchronously waited inside marker submit, stopped sending HELLOs, +# and peers could false-DEAD it during a long marker write. This TAP holds the +# marker service for >=90s while an online join wants a PREPARE marker and checks: +# L1 hold injection is registered. +# L2 a true leave/restart creates a join marker backlog. +# L3 during a 95s hold, LMON keeps iterating and both live nodes keep each +# other CSSD-alive (no false-DEAD). +# L4 marker timeout/backlog is observable and does not publish admission. +# L5 releasing the hold lets the join converge. +# L6 an actual stopped node still becomes DEAD (no true-DEAD regression). +# +# Portions Copyright (c) 2026, pgrac contributors +# +# Author: SqlRush +# +# IDENTIFICATION +# src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../../perl"; + +use PostgreSQL::Test::ClusterPair; +use Test::More; +use Time::HiRes qw(usleep); + +sub poll_until +{ + my ($node, $query, $expected, $timeout_s, $label) = @_; + $expected //= 't'; + $timeout_s //= 30; + my $deadline = time + $timeout_s; + my $last = ''; + while (time < $deadline) + { + $last = eval { $node->safe_psql('postgres', $query) } // ''; + return 1 if defined $last && $last eq $expected; + usleep(200_000); + } + diag("poll_until timeout ($label): last='$last' expected='$expected'"); + return 0; +} + +sub state_int +{ + my ($node, $category, $key) = @_; + my $v = $node->safe_psql('postgres', + "SELECT COALESCE((SELECT value::bigint FROM pg_cluster_state " + . "WHERE category = '$category' AND key = '$key'), 0)"); + return $v + 0; +} + +sub set_injection +{ + my ($node, $value) = @_; + $node->safe_psql('postgres', "ALTER SYSTEM SET cluster.injection_points = '$value'"); + $node->safe_psql('postgres', 'SELECT pg_reload_conf()'); + usleep(700_000); + return; +} + +my @conf = ( + 'autovacuum = off', + 'jit = off', + 'cluster.online_join = on', + 'cluster.join_convergence_timeout_ms = 180000', + 'cluster.quorum_poll_interval_ms = 500', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 8', + 'cluster.ges_request_timeout_ms = 30000', + 'cluster.lmon_slow_iteration_warn_ms = 0', +); + +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'marker_async_liveness', + quorum_voting_disks => 3, + shared_data => 1, + extra_conf => [@conf]); +$pair->start_pair; +usleep(3_000_000); + +my $node0 = $pair->node0; +my $node1 = $pair->node1; + +is($node0->safe_psql('postgres', + q{SELECT count(*) FROM pg_stat_cluster_injections + WHERE name = 'cluster-qvotec-marker-service-hold'}), + '1', + 'L1 hold injection point is registered'); + +ok($pair->wait_for_peer_state(0, 1, 'connected', 30), 'L1 node0 sees node1 connected'); +ok($pair->wait_for_peer_state(1, 0, 'connected', 30), 'L1 node1 sees node0 connected'); +ok(poll_until($node0, + q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 30, 'node1 initially member'), + 'L1 node1 initially MEMBER'); + +my $epoch0 = $node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0; + +$node1->stop; +ok(poll_until($node0, + q{SELECT state IN ('suspected','dead') FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 60, 'node1 true dead'), + 'L2 node0 detects the stopped node1 as dead'); +ok(poll_until($node0, + q{SELECT state <> 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 60, 'node1 demoted from member'), + 'L2 node1 is demoted from MEMBER before rejoin'); + +set_injection($node0, 'cluster-qvotec-marker-service-hold'); +my $lmon_iters_before = state_int($node0, 'lmon', 'lmon_slow_iter_count'); +my $timeouts_before = state_int($node0, 'reconfig', 'marker_timeout_count'); +my $epoch_hold_start = + $node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0; + +$node1->start; +ok(poll_until($node0, + q{SELECT state = 'alive' FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 60, 'node1 CSSD alive after restart'), + 'L2 node1 is live again while marker service is held'); + +sleep 95; + +my $lmon_iters_after = state_int($node0, 'lmon', 'lmon_slow_iter_count'); +my $timeouts_after = state_int($node0, 'reconfig', 'marker_timeout_count'); + +cmp_ok($lmon_iters_after, '>', $lmon_iters_before + 20, + 'L3 node0 LMON keeps iterating during >=90s qvotec marker hold'); +ok(poll_until($node1, + q{SELECT state = 'alive' FROM pg_cluster_cssd_peers WHERE node_id = 0}, + 't', 5, 'node1 still sees node0 alive after hold'), + 'L3 node1 still sees node0 CSSD-alive after the hold (no false-DEAD)'); +ok(poll_until($node0, + q{SELECT state = 'alive' FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 5, 'node0 still sees node1 alive after hold'), + 'L3 node0 still sees node1 CSSD-alive after the hold'); + +cmp_ok($timeouts_after, '>', $timeouts_before, + 'L4 marker timeout/backlog counter advances while qvotec marker service is held'); +isnt($node0->safe_psql('postgres', + q{SELECT state FROM pg_cluster_membership WHERE node_id = 1}), + 'member', + 'L4 held/timeout marker does not publish node1 admission'); + +# P1-1 bump-once (spec-2.29a review r1): the whole >=95s hold must cost at most +# ONE join Phase-1 epoch bump (staged record; PREPARE timeouts drain best-effort +# and never revert the joiner, so compute_join_bitmap cannot re-detect + re-bump +# per deadline period). Pre-fix abort+revert behavior re-bumped ~once per 3.5s +# deadline (~27 bumps in this window); a <=2 bound separates cleanly without +# being flaky against one incidental episode. +my $epoch_hold_end = + $node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0; +cmp_ok($epoch_hold_end - $epoch_hold_start, '<=', 2, + 'L4 epoch advance across the hold window is bounded (no per-deadline bump storm)'); + +set_injection($node0, ''); +ok(poll_until($node0, + q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 90, 'node1 joins after hold release'), + 'L5 node0 admits node1 after hold release'); +ok(poll_until($node1, + q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 90, 'node1 self member after hold release'), + 'L5 node1 self-state reaches MEMBER after hold release'); +cmp_ok($node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0, + '>', $epoch0, + 'L5 membership epoch advances after rejoin'); + +$node1->stop; +ok(poll_until($node0, + q{SELECT state = 'dead' FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 60, 'node1 true dead after final stop'), + 'L6 true DEAD detection still works after async marker liveness fix'); +ok(poll_until($node0, + q{SELECT state = 'dead' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 60, 'membership dead after final stop'), + 'L6 membership fail-stop path still demotes a truly stopped peer'); + +$pair->stop_pair; +done_testing(); diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 11c235416b..ae08788210 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -41,7 +41,7 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_buffer_desc test_cluster_pcm_lock test_cluster_bufmgr_pcm_hook test_cluster_gcs_dispatch test_cluster_gcs_block test_cluster_gcs_block_retransmit test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_sinval test_cluster_sinval_ack test_cluster_stage2_acceptance test_cluster_tt_status test_cluster_tt_status_hint test_cluster_visibility_fork test_cluster_visibility_decide_scn test_cluster_snapshot_source test_cluster_itl_touch test_cluster_itl_wal test_cluster_uba \ test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision \ test_cluster_xlog test_cluster_tt_slot test_cluster_undo_segment \ - test_cluster_epoch test_cluster_fence test_cluster_reconfig \ + test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_marker_async \ test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory \ test_cluster_tt_slot_allocator test_cluster_itl_reader_real_triple \ test_cluster_itl_cleanout test_cluster_visibility_inject test_cluster_itl_cleanout_perf \ @@ -63,8 +63,10 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_sequence \ test_cluster_shared_catalog \ test_cluster_oid_lease \ + test_cluster_xid_authority \ test_cluster_relmap_authority \ test_cluster_hw \ + test_cluster_hw_remaster_retry \ test_cluster_dl \ test_cluster_extend_gate \ test_cluster_ir \ @@ -178,7 +180,7 @@ test_cluster_backup: test_cluster_backup.c unit_test.h $(CLUSTER_VERSION_O) \ # separate rules because they also link additional cluster_*.o # objects (the test files stub the PG backend symbols those # objects reference). -SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_recovery_anchor test_cluster_relmap_authority,$(TESTS)) +SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_xid_authority test_cluster_recovery_anchor test_cluster_relmap_authority,$(TESTS)) # spec-2.4 D16: test_cluster_epoch links cluster_epoch.o standalone. # cluster_epoch.c references ShmemInitStruct + cluster_shmem_register_region @@ -443,6 +445,21 @@ test_cluster_oid_lease: test_cluster_oid_lease.c unit_test.h \ $(CLUSTER_OID_LEASE_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ +# spec-6.15b D1/D2: test_cluster_xid_authority links cluster_xid_authority.o +# (the shared XID authority + native-era prehistory: torn-safe file I/O, +# seal/flag monotonicity, prehistory publish/adopt byte round trip). fd.c +# openers, pg_fsync, durable_rename and the ereport machinery are stubbed +# locally; CRC32C comes from libpgport. The startup wiring (bootstrap adopt, +# StartupXLOG horizon merge, shutdown-checkpoint publish) is TAP territory. +CLUSTER_XID_AUTHORITY_O = $(top_builddir)/src/backend/cluster/cluster_xid_authority.o \ + $(top_builddir)/src/backend/cluster/cluster_xid_prehistory.o +test_cluster_xid_authority: test_cluster_xid_authority.c unit_test.h \ + $(CLUSTER_XID_AUTHORITY_O) + $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CLUSTER_XID_AUTHORITY_O) \ + $(top_builddir)/src/port/libpgport_srv.a -o $@ + + # spec-6.14 D5: test_cluster_relmap_authority links cluster_relmap_authority.o # (the two-phase pending/committed authority substrate). fd.c + ereport # stubbed locally; CRC32C from libpgport. The write-path activation (relmapper diff --git a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c index 540cff5b7d..90151fc3cb 100644 --- a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c +++ b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c @@ -181,6 +181,17 @@ ExceptionalCondition(const char *conditionName pg_attribute_unused(), abort(); } +/* spec-4.6a Amendment v1.2 (R5): linker-only exception-stack stubs (the + * linked cluster_pcm_lock.o now wraps the S->X upgrade in PG_TRY). */ +sigjmp_buf *PG_exception_stack = NULL; +ErrorContextCallback *error_context_stack = NULL; + +void +pg_re_throw(void) +{ + abort(); +} + static BufferTag make_tag(uint32 blockno) { diff --git a/src/test/cluster_unit/test_cluster_clean_leave.c b/src/test/cluster_unit/test_cluster_clean_leave.c index b703798b38..04363fd736 100644 --- a/src/test/cluster_unit/test_cluster_clean_leave.c +++ b/src/test/cluster_unit/test_cluster_clean_leave.c @@ -159,14 +159,104 @@ UT_TEST(test_phase_fsm) UT_TEST(test_version_coherent) { - /* nothing moved -> coherent */ - UT_ASSERT(cluster_clean_leave_version_coherent(7, 7, 3, 3)); - /* epoch bumped by an external death -> incoherent */ - UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, 3, 3)); - /* dead_generation moved -> incoherent */ - UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, 3, 4)); - /* both moved -> incoherent */ - UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, 3, 4)); + /* spec-2.29a ②b: coherence = epoch unchanged AND the others-dead bitmap + * (dead set EXCLUDING the leaving node) unchanged. The reflexive-case + * matrix from the spec §②b 8.A argument, exercised on the pure predicate. */ + uint8 od_none[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES] = { 0 }; + uint8 od_third[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES] = { 0 }; + const int n = CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES; + + od_third[0] = 0x04; /* a THIRD-PARTY node (e.g. node 2) became DEAD */ + + /* (i) nothing moved (leaving node's own DEAD is already excluded upstream, + * so its transition does not appear here) -> coherent */ + UT_ASSERT(cluster_clean_leave_version_coherent(7, 7, od_none, od_none, n)); + /* (iii) epoch bumped by an external fail-stop -> incoherent */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, od_none, od_none, n)); + /* (ii) a third-party death entered the others-dead set, epoch not yet + * bumped (the P1-b window) -> incoherent */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, od_none, od_third, n)); + /* (iv) both moved -> incoherent */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, od_none, od_third, n)); + /* fail-closed on a missing view */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, NULL, od_none, n)); + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, od_none, NULL, n)); +} + + +/* ============================================================ + * U3b — leaver barrier-tick own-commit latch (spec-2.29a r3, evidence + * over inference) + * ============================================================ */ + +UT_TEST(test_own_commit_latched) +{ + /* The latch verdict is EVIDENCE-only: the first argument is "the durable + * COMMITTED marker for THIS leave attempt was confirmed" (the coordinator's + * nonce-bound LEAVE_COMMITTED attestation). The two coherence observations + * (others-dead bitmap / scalar dead_generation) stay in the signature as + * contract inputs the verdict must IGNORE — a third-party transient + * false-DEAD flap on the leaver's local CSSD view advances the (monotone) + * dead_generation and can transiently disturb the bitmap, and neither may + * refuse a latch that marker evidence backs (the t/331 C1/C4 false- + * escalation), nor may any combination latch without evidence (the r2 P2-1 + * refused-leave mis-latch hang). */ + + /* (a) evidence present, no flap noise -> latch */ + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, true, true)); + + /* (d) PINNING LEG (t/331 C1/C4 regression): a third-party flap advanced the + * scalar dead_generation (it never rebounds) while the bitmap rebounded to + * its bound value — WITH marker evidence the leave still latches; the flap + * must not false-escalate a committed leave. */ + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, true, false)); + + /* (d) flap currently visible in the others-dead bitmap too -> still latch */ + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, false, true)); + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, false, false)); + + /* (c) no evidence -> never latch, whatever the coherence observations say + * (a refused leave never produces a COMMITTED marker, so the barrier + * deadline escalation stays armed and bounds the wait — the r2 P2-1 + * mis-latch wedge stays closed). */ + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, true, true)); + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, true, false)); + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, false, true)); + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, false, false)); +} + + +/* ============================================================ + * U3c — LEAVE_COMMITTED evidence identity gate (spec-2.29a r3) + * ============================================================ */ + +UT_TEST(test_committed_evidence_matches) +{ + /* args: payload_leaving_node, payload_nonce, payload_epoch, + * self_node, current_leaving_node, current_attempt_nonce, + * bound_leave_epoch */ + + /* (a) confirmation for THIS node's CURRENT attempt, committed epoch past + * the bound baseline -> evidence accepted */ + UT_ASSERT(cluster_clean_leave_committed_evidence_matches(1, 42, 8, 1, 1, 42, 7)); + + /* (b) stale identity: a LEAVE_COMMITTED (and through it a COMMITTED + * marker) from a PREVIOUS leave attempt of the same node — nonce differs + * -> fail-closed, no latch */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 41, 8, 1, 1, 42, 7)); + + /* (b) misrouted: confirmation addressed to a different leaving node */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(2, 42, 8, 1, 1, 42, 7)); + + /* (b) this node is not currently leaving (idle: leaving_node_id == -1, + * or tracking someone else's leave) */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 8, 1, -1, 42, 7)); + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 8, 1, 2, 42, 7)); + + /* (b) committed epoch not past the bound baseline (attested epoch must be + * the guarded-CAS successor of the baseline this leave bound) */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 7, 1, 1, 42, 7)); + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 0, 1, 1, 42, 7)); } @@ -382,10 +472,12 @@ UT_TEST(test_ic_payload_validation) int main(void) { - UT_PLAN(8); + UT_PLAN(10); UT_RUN(test_struct_layout); UT_RUN(test_phase_fsm); UT_RUN(test_version_coherent); + UT_RUN(test_own_commit_latched); + UT_RUN(test_committed_evidence_matches); UT_RUN(test_writable_only_gate); UT_RUN(test_marker_validation); UT_RUN(test_should_invalidate); diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index be82b60482..2dedadc7c2 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -45,6 +45,7 @@ #include "cluster/cluster_catalog_stats.h" /* spec-6.14 D10b catalog counter stubs */ #include "cluster/cluster_debug.h" +#include "cluster/cluster_grd.h" /* ClusterGrdRecoveryCounters */ #include "cluster/cluster_hang.h" /* spec-5.11: ClusterHangDumpData for dump_hang stubs */ #include "cluster/cluster_hang_resolve.h" /* spec-5.12: ClusterHangResolveCounters for dump stubs */ #include "cluster/cluster_reconfig.h" /* spec-5.14 D6 touched getter stubs */ @@ -53,6 +54,7 @@ #include "cluster/cluster_xnode_lever.h" /* spec-6.12 lever counter stub */ #include "cluster/cluster_hw_lease.h" /* spec-6.12d lease counter stub */ #include "cluster/cluster_relmap_authority.h" /* spec-6.14 D5 header-read stub */ +#include "cluster/cluster_xid_authority.h" /* spec-6.15b XID authority dump stubs */ #undef printf #undef fprintf @@ -342,6 +344,21 @@ cluster_relmap_authority_read_header(bool shared_map, Oid dbid, ClusterRelmapAut return false; } +/* spec-6.15b D7 stubs: dump_catalog reads the XID authority observation + * keys; this standalone debug unit does not link the file-I/O authority + * objects. */ +bool +cluster_xid_authority_read(ClusterXidAuthorityHeader *out) +{ + return false; +} + +bool +cluster_xid_prehistory_was_adopted(void) +{ + return false; +} + /* spec-4.12 D7 + spec-4.12b D6 stubs: dump_write_fence (cluster_debug.o) reads 8 * counters now, and cluster_startup_phase.o (linked here) references the rejoin * self-fence gate. cluster_write_fence.o is not linked -- provide stubs returning @@ -564,6 +581,9 @@ uint64 cluster_hw_failclosed_count(void); uint64 cluster_hw_not_ready_count(void); uint64 cluster_hw_remaster_done_count(void); uint64 cluster_hw_remaster_blocked_count(void); +uint64 cluster_hw_remaster_retry_count(void); +uint64 cluster_hw_remaster_retry_exhausted_count(void); +bool cluster_hw_remaster_recoverable(void); uint64 cluster_hw_alloc_count(void) { @@ -604,6 +624,21 @@ cluster_hw_remaster_blocked_count(void) { return 0; } +uint64 +cluster_hw_remaster_retry_count(void) +{ + return 0; +} +uint64 +cluster_hw_remaster_retry_exhausted_count(void) +{ + return 0; +} +bool +cluster_hw_remaster_recoverable(void) +{ + return true; +} /* spec-5.7 D4 dump_dl stubs (cluster_dl.c not linked in this binary). */ uint64 cluster_dl_lease_count(void); @@ -2638,6 +2673,21 @@ cluster_lmon_main_loop_iters(void) { return 0; } +uint64 +cluster_lmon_last_iter_us(void) +{ + return 0; +} +uint64 +cluster_lmon_max_iter_us(void) +{ + return 0; +} +uint64 +cluster_lmon_slow_iter_count(void) +{ + return 0; +} int cluster_lmon_status(void) { @@ -3288,15 +3338,77 @@ cluster_grd_deadlock_chunk_oo_buffer_overflow_count(void) return 0; } -/* spec-4.6 D5 stub: dump_grd_recovery consumes the bulk counter - * snapshot. Zero-fill; layout = 13 × uint64 (must track the - * ClusterGrdRecoveryCounters struct in cluster_grd.h). */ -struct ClusterGrdRecoveryCounters; -void cluster_grd_recovery_counters_snapshot(struct ClusterGrdRecoveryCounters *out); +uint32 +cluster_grd_recovery_state_value(void) +{ + return (uint32)GRD_RECOVERY_IDLE; +} + +const char * +cluster_grd_recovery_state_name(uint32 state pg_attribute_unused()) +{ + return "idle"; +} + +uint64 +cluster_grd_recovery_last_event_id(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_event_old_epoch(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_episode_epoch_value(void) +{ + return 0; +} + +uint32 +cluster_grd_recovery_event_coordinator(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_done_epoch_for(int32 node pg_attribute_unused()) +{ + return 0; +} + +uint64 +cluster_grd_recovery_done_bitmap_hash_for(int32 node pg_attribute_unused()) +{ + return 0; +} + +int +cluster_grd_recovery_block_redeclare_cursor(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_block_redeclare_epoch(void) +{ + return 0; +} + +bool +cluster_grd_recovery_block_redeclare_done(void) +{ + return true; +} + +/* spec-4.6 D5 stub: dump_grd_recovery consumes the bulk counter snapshot. */ void -cluster_grd_recovery_counters_snapshot(struct ClusterGrdRecoveryCounters *out) +cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out) { - memset(out, 0, 13 * sizeof(uint64)); + memset(out, 0, sizeof(*out)); } uint32 @@ -3377,6 +3489,16 @@ cluster_reconfig_get_clean_departed_cleared_count(void) { return 0; } +uint64 +cluster_reconfig_get_marker_slow_ack_count(void) +{ + return 0; +} +uint64 +cluster_reconfig_get_marker_timeout_count(void) +{ + return 0; +} void cluster_touched_peers_self_hex(char *buf, Size buflen) { diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 444accf8f0..8729fbd744 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -70,6 +70,7 @@ #include "cluster/cluster_gcs.h" #include "cluster/cluster_gcs_block.h" #include "cluster/cluster_ic_envelope.h" +#include "cluster/cluster_thread_recovery.h" #include "common/hashfn.h" #include "storage/buf_internals.h" #include "storage/lwlock.h" diff --git a/src/test/cluster_unit/test_cluster_ges.c b/src/test/cluster_unit/test_cluster_ges.c index 1df0a6b7fc..dd2a47185a 100644 --- a/src/test/cluster_unit/test_cluster_ges.c +++ b/src/test/cluster_unit/test_cluster_ges.c @@ -296,7 +296,8 @@ cluster_cancel_token_consume(void) * completion. Standalone fixture has no recovery shmem; no-op. */ void cluster_grd_recovery_mark_peer_done(int32 node pg_attribute_unused(), - uint64 epoch pg_attribute_unused()) + uint64 epoch pg_attribute_unused(), + uint64 event_id pg_attribute_unused()) {} ClusterGrdEntryResult diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index 2a0b568524..b53a5a568c 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -55,6 +55,7 @@ #include "cluster/cluster_ges_mode.h" /* spec-5.1b — frozen matrix + convert classification */ #include "access/transam.h" /* spec-5.8 D1c — InvalidTransactionId */ #include "cluster/cluster_grd.h" +#include "cluster/cluster_hw.h" /* spec-4.6a HW remaster watchdog stubs */ #include "cluster/cluster_lmd.h" /* spec-5.8 D1b — WFG vertex + submit/cancel edge */ #include "cluster/cluster_reconfig.h" /* spec-4.6 D1 — ReconfigEvent stub type */ #include "cluster/cluster_thread_recovery.h" /* spec-4.11 D3 (L238) — gate_unfreeze proto */ @@ -85,6 +86,15 @@ bool IsUnderPostmaster = false; +/* spec-4.6a D12 stub: cluster_grd_cleanup_on_node_dead chains into the PCM + * dead-holder cleanup; the GRD fixture has no PCM shmem — no-op (the cleanup + * logic itself is unit-tested in test_cluster_pcm_lock.c). */ +uint64 +cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node pg_attribute_unused()) +{ + return 0; +} + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), @@ -316,6 +326,13 @@ cluster_epoch_get_current(void) return ut_mock_epoch; } +void +cluster_epoch_adopt_admitted(uint64 admitted_epoch) +{ + if (admitted_epoch > ut_mock_epoch) + ut_mock_epoch = admitted_epoch; +} + /* * spec-4.7 D2 (L238) — cluster_grd.o's reconfig tick now references the block * re-declare scan/send (grd_block_redeclare_step / _cb). This test exercises @@ -377,10 +394,30 @@ cluster_bufmgr_redeclare_scan_chunk(int start_buf, int max_scan, return end; } +/* spec-4.6a r2-P1-1: controllable last-event mock so the recovery FSM's P0 + * accept + WAIT_EPOCH event-scoped witness can be unit-driven. Default + * zeroed = pre-existing inert behavior. */ +static ReconfigEvent ut_mock_last_event; void cluster_reconfig_get_last_event(ReconfigEvent *out) { - memset(out, 0, sizeof(*out)); + *out = ut_mock_last_event; +} + +uint64 +cluster_reconfig_get_observed_epoch(int32 node_id pg_attribute_unused()) +{ + return 0; +} + +/* spec-2.29a nightly-regression fix: controllable pre-bump-stage flag so the + * GRD IDLE baseline-hold can be unit-driven (default false = pre-fix + * re-capture behavior; existing tests are unaffected). */ +static bool ut_mock_pending_prebump = false; +bool +cluster_reconfig_has_pending_prebump_stage(void) +{ + return ut_mock_pending_prebump; } /* spec-4.11 D3 stub: cluster_grd.c's WAIT_CLUSTER->IDLE transition consults the @@ -419,6 +456,24 @@ cluster_hw_remaster_gate_unfreeze(void) return false; } +ClusterHwRemasterResult +cluster_hw_remaster_result(int node_id pg_attribute_unused()) +{ + return CLUSTER_HW_REMASTER_NONE; +} + +uint32 +cluster_hw_remaster_attempts(int node_id pg_attribute_unused()) +{ + return 0; +} + +const char * +cluster_hw_remaster_result_name(ClusterHwRemasterResult result pg_attribute_unused()) +{ + return "none"; +} + struct PGPROC * BackendIdGetProc(int backendID pg_attribute_unused()) { @@ -432,10 +487,12 @@ SendProcSignal(int pid pg_attribute_unused(), int reason pg_attribute_unused(), return 0; } +/* spec-4.6a r2-P1-1: controllable clock (default 0 = pre-existing value). */ +static int64 ut_mock_now = 0; int64 GetCurrentTimestamp(void) { - return 0; + return ut_mock_now; } void @@ -3701,6 +3758,30 @@ ut_jr_setup_3node(void) cluster_grd_master_map_init(); } +/* spec-4.6a Amendment v1.2 (R2): REDECLARE_DONE accounting converges on the + * composite key (episode_epoch, dead_bitmap_hash), so a test that feeds DONEs + * must first drive a P0 accept to stamp the episode's bitmap hash. Accept an + * empty-dead-bitmap FAIL event (hash identical to a JOIN episode's, no fence + * side effects; the FSM parks in WAIT_EPOCH awaiting a strict bump, which the + * fence assertions never touch). Returns the stamped hash for the callers' + * mark_peer_done payloads. */ +static uint64 +ut_jr_stamp_episode_bitmap_hash(uint64 event_id) +{ + bool saved_enabled = cluster_enabled; + + cluster_enabled = true; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); /* idle tick: baseline capture */ + ut_mock_last_event.event_id = event_id; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + cluster_grd_recovery_lmon_tick(); /* P0 accept stamps the bitmap hash */ + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_enabled = saved_enabled; + return cluster_grd_recovery_event_bitmap_hash_value(); +} + /* U1 — recompute moves the joiner's home shards back from the survivor. */ UT_TEST(test_jr_u1_recompute_moves_joiner_home) { @@ -3890,9 +3971,12 @@ UT_TEST(test_jr_u13_view_rebuilt_all_members_barrier) int n1[1] = { 1 }; BufferTag tag; + uint64 done_hash; + memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(901); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; /* the rejoiner's home block */ @@ -3903,15 +3987,15 @@ UT_TEST(test_jr_u13_view_rebuilt_all_members_barrier) /* HARDENING v1.1 — the joiner (node 1) announcing its OWN trivial barrier * must NOT lift the fence: survivors 0 and 2 have not re-declared yet. */ - cluster_grd_recovery_mark_peer_done(1, 10); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* One survivor done is still not enough. */ - cluster_grd_recovery_mark_peer_done(0, 10); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* All members done → view rebuilt → fence lifts. */ - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } @@ -3922,10 +4006,13 @@ UT_TEST(test_jr_u14_fence_arm_monotonic_and_scope) int n1[1] = { 1 }; BufferTag tag; + uint64 done_hash; + memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(902); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); @@ -3934,15 +4021,15 @@ UT_TEST(test_jr_u14_fence_arm_monotonic_and_scope) ut_mock_epoch = 5; cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; - cluster_grd_recovery_mark_peer_done(0, 5); - cluster_grd_recovery_mark_peer_done(1, 5); - cluster_grd_recovery_mark_peer_done(2, 5); + cluster_grd_recovery_mark_peer_done(0, 5, done_hash); + cluster_grd_recovery_mark_peer_done(1, 5, done_hash); + cluster_grd_recovery_mark_peer_done(2, 5, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* 5 < fence epoch 10 */ /* Done at the real fence epoch lifts it. */ - cluster_grd_recovery_mark_peer_done(0, 10); - cluster_grd_recovery_mark_peer_done(1, 10); - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } @@ -3955,10 +4042,12 @@ UT_TEST(test_jr_u15_master_side_gate_decision) int n1[1] = { 1 }; BufferTag tag; bool deny; + uint64 done_hash; memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(903); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; @@ -3967,9 +4056,9 @@ UT_TEST(test_jr_u15_master_side_gate_decision) deny = cluster_grd_join_remaster_active_for_shard(tag) && !cluster_grd_block_view_rebuilt(tag); UT_ASSERT(deny); - cluster_grd_recovery_mark_peer_done(0, 10); - cluster_grd_recovery_mark_peer_done(1, 10); - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); deny = cluster_grd_join_remaster_active_for_shard(tag) && !cluster_grd_block_view_rebuilt(tag); UT_ASSERT(!deny); } @@ -4012,17 +4101,20 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) int n2[1] = { 2 }; BufferTag tag; + uint64 done_hash; + memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); /* declared {0,1,2}, all members */ /* --- Episode 1: node 1 rejoins and completes (all survivors re-declare). --- */ ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(904); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; /* a node-1-home block */ - cluster_grd_recovery_mark_peer_done(0, 10); - cluster_grd_recovery_mark_peer_done(1, 10); - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); /* episode 1 converged */ /* --- Episode 2: node 2 rejoins. node 1 is now a steady survivor whose held @@ -4035,15 +4127,253 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) /* Only the OTHER survivor (node 0) has re-declared for episode 2; node 1 has * NOT. node 1's stale episode-1 fence bit must NOT exclude it from the * barrier — it must still gate the fence lift. */ - cluster_grd_recovery_mark_peer_done(0, 20); + cluster_grd_recovery_mark_peer_done(0, 20, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* must still wait for node 1 */ /* Once node 1 re-declares for episode 2 the barrier converges and lifts. */ - cluster_grd_recovery_mark_peer_done(1, 20); + cluster_grd_recovery_mark_peer_done(1, 20, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } +/* ============================================================ + * spec-4.6a Amendment v1.2 (R2) — composite-key REDECLARE_DONE proof. + * + * The cross-node convergence key is (episode_epoch, dead_bitmap_hash). + * A DONE naming a DIFFERENT dead set (the old-event ABA shape) must be + * dropped at the accounting layer and never satisfy the WAIT_EPOCH + * coordinator witness. A DONE naming the SAME dead set must be + * accounted even though the sender's local event_id differs (the + * dead_generation-drift shape that wedged the event_id key: R2), and + * then satisfies the witness — the only equal-epoch escape + * (proof-carrying, never timeout-only). + * ============================================================ */ + +UT_TEST(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance) +{ + ClusterGrdRecoveryCounters before; + ClusterGrdRecoveryCounters after; + uint8 old_dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint64 episode_hash; + uint64 old_event_hash; + + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_now = 0; + ut_mock_epoch = 10; + + /* Idle tick captures the pre-reconfig baseline. ut_mock_epoch already + * holds the post-bump value, reproducing the mis-captured-baseline wedge + * shape (cur == old) from spec-4.6a section 0. */ + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); + + /* Stage the fail-stop event: dead node 2, coordinator 0. The local + * event_id (77) folds this node's own dead_generation; a peer that saw a + * different flap history computes a DIFFERENT id for the same episode. */ + ut_mock_last_event.event_id = 77; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + ut_mock_last_event.dead_bitmap[0] = 0x04; + cluster_grd_recovery_lmon_tick(); /* P0 accept -> WAIT_EPOCH, no proof yet */ + UT_ASSERT_EQ(cluster_grd_recovery_last_event_id(), 77); + episode_hash = cluster_grd_recovery_event_bitmap_hash_value(); + UT_ASSERT_EQ(episode_hash, cluster_grd_dead_bitmap_hash(ut_mock_last_event.dead_bitmap)); + + /* Old-event ABA shape: a late DONE naming a DIFFERENT dead set (node 3 + * instead of node 2 — e.g. the previous episode before a coordinator + * re-election) is dropped: nothing is accounted. */ + memset(old_dead, 0, sizeof(old_dead)); + old_dead[0] = 0x08; + old_event_hash = cluster_grd_dead_bitmap_hash(old_dead); + cluster_grd_recovery_mark_peer_done(0, 10, old_event_hash); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 0); + + /* Generation-drift shape (the R2 wedge): the peer's local event_id + * differs, but its DONE names the SAME quorum dead set -> the composite + * key matches and the DONE is accounted. */ + cluster_grd_recovery_mark_peer_done(0, 10, episode_hash); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 10); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), episode_hash); + + /* Witness advance: deadline expired + cur==old + coordinator DONE naming + * THIS dead set at exactly cur, accounted after the accept snapshot. The + * FSM takes the composite-key equal-epoch escape exactly once. */ + cluster_grd_recovery_counters_snapshot(&before); + ut_mock_now = (int64)60 * 1000000; + cluster_grd_recovery_lmon_tick(); + cluster_grd_recovery_counters_snapshot(&after); + UT_ASSERT_EQ(after.wait_epoch_escape, before.wait_epoch_escape + 1); + + cluster_enabled = false; + ut_mock_now = 0; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); +} + +/* r3-P2-2 — survivor-behind multi-death detection skew. The coordinator + * folded {1,2} into ONE epoch bump; this survivor accepted only {1} first and + * stamped H({1}) at that epoch. Its later local detection of node 2 publishes + * a new event with the GROWN set at the SAME epoch (no further bump), which + * pre-fix was never re-consumed past WAIT_EPOCH: DONE hashes could never match + * and P6 (no timeout) wedged permanently. The mid-episode guard must abort to + * IDLE, re-consume, re-stamp the full set's hash, re-run the remaster against + * the fuller dead set, and let full-set DONEs account. */ +UT_TEST(test_recovery_same_epoch_dead_set_growth_restamps) +{ + ClusterGrdRecoveryCounters c0; + ClusterGrdRecoveryCounters c1; + uint8 grown[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint64 h_small; + uint64 h_grown; + int i; + + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_now = 0; + ut_mock_epoch = 10; + + /* Idle tick captures the pre-reconfig baseline (old = 10). */ + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); + + /* Event 1: this survivor's CSSD has seen only node 1 dead so far. */ + ut_mock_last_event.event_id = 601; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + ut_mock_last_event.dead_bitmap[0] = 0x02; /* {1} */ + cluster_grd_recovery_lmon_tick(); /* P0 accept -> parks in WAIT_EPOCH (cur==old) */ + h_small = cluster_grd_recovery_event_bitmap_hash_value(); + + /* The coordinator's single folded bump arrives via piggyback: this node + * sails past WAIT_EPOCH with the SMALL set stamped. */ + ut_mock_epoch = 11; + cluster_grd_recovery_lmon_tick(); /* P1-P5 -> WAIT_BARRIER, episode epoch 11 */ + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); + UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_small); + + /* The wedge shape: a full-set DONE from the coordinator is dropped while + * this node's stamp is behind. */ + memset(grown, 0, sizeof(grown)); + grown[0] = 0x06; /* {1,2} */ + h_grown = cluster_grd_dead_bitmap_hash(grown); + UT_ASSERT_NE(h_grown, h_small); + cluster_grd_recovery_mark_peer_done(0, 11, h_grown); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 0); + + /* Local CSSD catches up: NEW event_id, SAME epoch, GROWN dead set. */ + cluster_grd_recovery_counters_snapshot(&c0); + ut_mock_last_event.event_id = 602; + ut_mock_last_event.dead_bitmap[0] = 0x06; + cluster_grd_recovery_lmon_tick(); /* WAIT_BARRIER guard -> abort to IDLE */ + cluster_grd_recovery_counters_snapshot(&c1); + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_IDLE); + UT_ASSERT_EQ(c1.remaster_failed, c0.remaster_failed + 1); + + /* Re-consume: re-stamp to the FULL set's hash, re-run the remaster against + * the fuller set (nothing may stay mastered by 1 OR 2), land back in + * WAIT_BARRIER under the same epoch. */ + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); + UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_grown); + for (i = 0; i < PGRAC_GRD_SHARD_COUNT; i++) + UT_ASSERT_EQ(cluster_grd_shard_master(i), (int32)0); + + /* The coordinator's per-tick full-set DONE re-announce now accounts. */ + cluster_grd_recovery_mark_peer_done(0, 11, h_grown); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 11); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), h_grown); + + cluster_enabled = false; + ut_mock_now = 0; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); +} + +/* r3-P2-2 no-regression leg — dead_generation drift re-fires the SAME dead + * set under the same epoch (new event_id, identical bitmap). The guard must + * ABSORB it: no abort-to-IDLE churn, stamp unchanged, and the event_id is + * consumed so the post-episode IDLE dedup does not re-run the episode. */ +UT_TEST(test_recovery_same_epoch_same_set_drift_absorbed_no_churn) +{ + ClusterGrdRecoveryCounters c0; + ClusterGrdRecoveryCounters c1; + uint64 h_set; + + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_now = 0; + ut_mock_epoch = 10; + + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); /* idle baseline (old = 10) */ + + ut_mock_last_event.event_id = 611; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + ut_mock_last_event.dead_bitmap[0] = 0x04; /* {2} */ + cluster_grd_recovery_lmon_tick(); /* P0 accept -> WAIT_EPOCH */ + ut_mock_epoch = 11; + cluster_grd_recovery_lmon_tick(); /* P1-P5 -> WAIT_BARRIER */ + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); + h_set = cluster_grd_recovery_event_bitmap_hash_value(); + + /* Same-set re-fire (drift): absorbed, no episode churn. The tick may + * legitimately advance the episode (WAIT_BARRIER -> WAIT_CLUSTER with the + * trivial unit barrier) — the assertion is that it never falls back to + * IDLE and the stamp/counters do not move. */ + cluster_grd_recovery_counters_snapshot(&c0); + ut_mock_last_event.event_id = 612; + cluster_grd_recovery_lmon_tick(); + cluster_grd_recovery_counters_snapshot(&c1); + UT_ASSERT_EQ(c1.remaster_failed, c0.remaster_failed); /* no abort */ + UT_ASSERT_NE(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_IDLE); + UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_set); + UT_ASSERT_EQ(cluster_grd_recovery_last_event_id(), 612); + + cluster_enabled = false; + ut_mock_now = 0; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); +} + + +/* ============================================================ + * spec-2.29a nightly-regression fix — GRD IDLE baseline hold while a + * pre-bump coordinator stage is in flight. + * + * The async fence/join/node-remove marker staging bumps the epoch + * several ticks before publishing. While that window is open the GRD + * IDLE tick must NOT re-capture the post-bump epoch as its WAIT_EPOCH + * baseline, or the P0 accept that follows reads old == cur and wedges. + * ============================================================ */ + +UT_TEST(test_recovery_idle_holds_baseline_during_prebump_stage) +{ + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_pending_prebump = false; + + /* Idle tick with epoch 5 captures baseline 5 (no staging; get_last_event + * returns event_id 0 so the tick stays IDLE and re-captures). */ + ut_mock_epoch = 5; + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_event_old_epoch(), 5); + + /* A pre-bump stage goes live and the coordinator bumps the epoch to 6 + * before publishing. The IDLE tick must HOLD baseline 5. */ + ut_mock_pending_prebump = true; + ut_mock_epoch = 6; + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_event_old_epoch(), 5); + + /* Marker ACKs, stage clears; a subsequent idle tick resumes tracking. */ + ut_mock_pending_prebump = false; + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_event_old_epoch(), 6); + + cluster_enabled = false; +} + int /* cppcheck-suppress constParameter * Reason: main() keeps the standard test harness signature used by the @@ -4054,8 +4384,10 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) * spec-6.3a:+6 lifecycle; 5.8 D1b:+4 (U2a-d); D1c:+2 (U3a-b); * D1e:+2 (U4a-b); 5.9 Hardening:+1 (convert ABA); * spec-5.16:+12 (join-remaster U1-U5/U10-U16); - * +1 (U17 cross-episode fence Hardening). */ - UT_PLAN(80); + * +1 (U17 cross-episode fence Hardening); + * spec-4.6a r3-P2-2:+2 (same-epoch dead-set growth re-stamp + no-churn); + * spec-2.29a:+1 (idle baseline hold during pre-bump stage). */ + UT_PLAN(84); UT_RUN(test_grd_clusterresid_size_16); UT_RUN(test_grd_resid_encode_decode_roundtrip); @@ -4158,6 +4490,12 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_jr_u15_master_side_gate_decision); UT_RUN(test_jr_u16_pre_member_guard); UT_RUN(test_jr_u17_stale_recipient_not_excluded_next_episode); + UT_RUN(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance); + + /* spec-4.6a r3-P2-2 — same-epoch multi-death detection-skew closure. */ + UT_RUN(test_recovery_same_epoch_dead_set_growth_restamps); + UT_RUN(test_recovery_same_epoch_same_set_drift_absorbed_no_churn); + UT_RUN(test_recovery_idle_holds_baseline_during_prebump_stage); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/cluster_unit/test_cluster_grd_starvation.c b/src/test/cluster_unit/test_cluster_grd_starvation.c index 44b24f4099..e2bb7352b8 100644 --- a/src/test/cluster_unit/test_cluster_grd_starvation.c +++ b/src/test/cluster_unit/test_cluster_grd_starvation.c @@ -61,6 +61,7 @@ #include "cluster/cluster_ges_mode.h" /* spec-5.1b — frozen matrix + convert classification */ #include "access/transam.h" /* spec-5.8 D1c — InvalidTransactionId */ #include "cluster/cluster_grd.h" +#include "cluster/cluster_hw.h" /* spec-4.6a HW remaster watchdog stubs */ #include "cluster/cluster_lmd.h" /* spec-5.8 D1b — WFG vertex + submit/cancel edge */ #include "cluster/cluster_reconfig.h" /* spec-4.6 D1 — ReconfigEvent stub type */ #include "cluster/cluster_thread_recovery.h" /* spec-4.11 D3 (L238) — gate_unfreeze proto */ @@ -91,6 +92,14 @@ bool IsUnderPostmaster = false; +/* spec-4.6a D12 stub: cluster_grd_cleanup_on_node_dead chains into the PCM + * dead-holder cleanup; this fixture has no PCM shmem — no-op. */ +uint64 +cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node pg_attribute_unused()) +{ + return 0; +} + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), @@ -298,6 +307,10 @@ cluster_epoch_get_current(void) return 0; } +void +cluster_epoch_adopt_admitted(uint64 admitted_epoch pg_attribute_unused()) +{} + /* * spec-4.7 D2 (L238) — cluster_grd.o's reconfig tick now references the block * re-declare scan/send (grd_block_redeclare_step / _cb). This test exercises @@ -355,6 +368,19 @@ cluster_reconfig_get_last_event(ReconfigEvent *out) memset(out, 0, sizeof(*out)); } +uint64 +cluster_reconfig_get_observed_epoch(int32 node_id pg_attribute_unused()) +{ + return 0; +} + +/* spec-2.29a: no pre-bump staging in the starvation fixture (no-op false). */ +bool +cluster_reconfig_has_pending_prebump_stage(void) +{ + return false; +} + /* spec-4.11 D3 stub: cluster_grd.c's WAIT_CLUSTER->IDLE transition consults the * thread-recovery unfreeze gate before P7. These tests drive the GES/GRD * remaster FSM, not online thread recovery, so the gate is out of scope -> no-op @@ -391,6 +417,24 @@ cluster_hw_remaster_gate_unfreeze(void) return false; } +ClusterHwRemasterResult +cluster_hw_remaster_result(int node_id pg_attribute_unused()) +{ + return CLUSTER_HW_REMASTER_NONE; +} + +uint32 +cluster_hw_remaster_attempts(int node_id pg_attribute_unused()) +{ + return 0; +} + +const char * +cluster_hw_remaster_result_name(ClusterHwRemasterResult result pg_attribute_unused()) +{ + return "none"; +} + struct PGPROC * BackendIdGetProc(int backendID pg_attribute_unused()) { diff --git a/src/test/cluster_unit/test_cluster_hw_remaster_retry.c b/src/test/cluster_unit/test_cluster_hw_remaster_retry.c new file mode 100644 index 0000000000..456c5979f8 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_hw_remaster_retry.c @@ -0,0 +1,156 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_hw_remaster_retry.c + * Unit tests for spec-4.6a same-episode HW remaster retry decisions. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_hw_remaster_retry.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_hw_remaster.h" + +#undef printf +#undef fprintf +#undef snprintf + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +void +ExceptionalCondition(const char *conditionName pg_attribute_unused(), + const char *fileName pg_attribute_unused(), + int lineNumber pg_attribute_unused()) +{ + abort(); +} + +static ClusterHwRemasterRelaunchDecision +decide(uint64 launched, uint64 episode, ClusterHwRemasterResult result, uint32 attempts, + uint64 next_attempt_at, uint64 now, int max_attempts) +{ + return cluster_hw_remaster_relaunch_decide(launched, episode, result, attempts, next_attempt_at, + now, max_attempts); +} + +UT_TEST(test_new_episode_initial_launch_resets_state) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(10, 11, CLUSTER_HW_REMASTER_BLOCKED, 7, 1234, 2000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_INITIAL); + UT_ASSERT_EQ(d.next_attempts, 0); + UT_ASSERT_EQ(d.next_attempt_at, 0); + UT_ASSERT_EQ(d.next_result, CLUSTER_HW_REMASTER_RUNNING); +} + +UT_TEST(test_running_done_and_not_applicable_do_not_relaunch) +{ + UT_ASSERT_EQ(decide(11, 11, CLUSTER_HW_REMASTER_RUNNING, 0, 0, 2000, 16).action, + CLUSTER_HW_REMASTER_LAUNCH_SKIP); + UT_ASSERT_EQ(decide(11, 11, CLUSTER_HW_REMASTER_DONE, 0, 0, 2000, 16).action, + CLUSTER_HW_REMASTER_LAUNCH_SKIP); + UT_ASSERT_EQ(decide(11, 11, CLUSTER_HW_REMASTER_NOT_APPLICABLE, 0, 0, 2000, 16).action, + CLUSTER_HW_REMASTER_LAUNCH_SKIP); +} + +UT_TEST(test_structural_blocked_warns_once_and_never_retries) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL, 0, 0, 2000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL); + UT_ASSERT_EQ(d.next_attempt_at, CLUSTER_HW_REMASTER_NO_DEADLINE); + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL, 0, CLUSTER_HW_REMASTER_NO_DEADLINE, + 3000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_SKIP); +} + +UT_TEST(test_blocked_waits_until_backoff_deadline) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 0, 5000, 4999, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_SKIP); + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 0, 5000, 5000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_RETRY); + UT_ASSERT_EQ(d.next_attempts, 1); + UT_ASSERT_EQ(d.next_result, CLUSTER_HW_REMASTER_RUNNING); +} + +UT_TEST(test_cap_exhausts_once_and_sighup_raise_recovers) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 2, 0, 5000, 2); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED); + UT_ASSERT_EQ(d.next_attempt_at, CLUSTER_HW_REMASTER_NO_DEADLINE); + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 2, CLUSTER_HW_REMASTER_NO_DEADLINE, 6000, 2); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_SKIP); + + /* Operator raised cluster.hw_remaster_retry_max_attempts via SIGHUP. */ + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 2, CLUSTER_HW_REMASTER_NO_DEADLINE, 7000, 3); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_RETRY); + UT_ASSERT_EQ(d.next_attempts, 3); +} + +UT_TEST(test_zero_max_attempts_disables_retry) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 0, 0, 5000, 0); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED); +} + +/* spec-4.6a section 2.1 truth-table row 8: latched == episode but the result + * slot reads NONE (registration failed and was reverted) -> take the + * first-launch row again; no attempt is charged. */ +UT_TEST(test_none_result_registration_failed_falls_back_to_initial) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_NONE, 3, 999, 2000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_INITIAL); + UT_ASSERT_EQ(d.next_attempts, 0); + UT_ASSERT_EQ(d.next_attempt_at, 0); + UT_ASSERT_EQ(d.next_result, CLUSTER_HW_REMASTER_RUNNING); +} + +UT_TEST(test_backoff_exponential_cap) +{ + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 0), 1000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 1), 1000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 2), 2000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 7), 60000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(100, 20), 60000); +} + + +int +main(void) +{ + UT_PLAN(8); + UT_RUN(test_new_episode_initial_launch_resets_state); + UT_RUN(test_running_done_and_not_applicable_do_not_relaunch); + UT_RUN(test_structural_blocked_warns_once_and_never_retries); + UT_RUN(test_blocked_waits_until_backoff_deadline); + UT_RUN(test_cap_exhausts_once_and_sighup_raise_recovers); + UT_RUN(test_zero_max_attempts_disables_retry); + UT_RUN(test_none_result_registration_failed_falls_back_to_initial); + UT_RUN(test_backoff_exponential_cap); + UT_DONE(); + + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/cluster_unit/test_cluster_lmon.c b/src/test/cluster_unit/test_cluster_lmon.c index d5b86a9bfd..15bc7894b1 100644 --- a/src/test/cluster_unit/test_cluster_lmon.c +++ b/src/test/cluster_unit/test_cluster_lmon.c @@ -178,6 +178,7 @@ cluster_postmaster_start_lmon(void) * MyLatch + cluster_inject framework. Stubs cover them all -- * runtime LmonMain is not exercised. */ int cluster_lmon_main_loop_interval = 1000; +int cluster_lmon_slow_iteration_warn_ms = 1000; #include "cluster/cluster_inject.h" int cluster_injection_armed_count = 0; @@ -590,6 +591,9 @@ pg_usleep(long microsec pg_attribute_unused()) /* Sprint B: Latch / WaitLatch / ResetLatch stubs (LmonMain runtime * is not invoked at unit-test level). */ struct Latch *MyLatch = NULL; +void +SetLatch(struct Latch *latch pg_attribute_unused()) +{} int WaitLatch(struct Latch *latch pg_attribute_unused(), int wakeEvents pg_attribute_unused(), long timeout pg_attribute_unused(), uint32 wait_event_info pg_attribute_unused()) @@ -600,6 +604,11 @@ void ResetLatch(struct Latch *latch pg_attribute_unused()) {} +#include "storage/ipc.h" +void +on_shmem_exit(pg_on_exit_callback function pg_attribute_unused(), Datum arg pg_attribute_unused()) +{} + /* cluster_lmon.c references MyBackendType (set by LmonMain). */ #include "miscadmin.h" BackendType MyBackendType = B_INVALID; @@ -804,9 +813,21 @@ UT_TEST(test_lmon_public_symbols_linkable) UT_ASSERT_NOT_NULL((void *)cluster_lmon_shmem_size); UT_ASSERT_NOT_NULL((void *)cluster_lmon_shmem_init); UT_ASSERT_NOT_NULL((void *)cluster_lmon_shmem_register); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_last_iter_us); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_max_iter_us); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_slow_iter_count); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_marker_complete_wakeup); UT_ASSERT_NOT_NULL((void *)LmonMain); } +UT_TEST(test_lmon_iteration_counters_null_safe) +{ + UT_ASSERT_EQ((unsigned long long)cluster_lmon_last_iter_us(), 0ULL); + UT_ASSERT_EQ((unsigned long long)cluster_lmon_max_iter_us(), 0ULL); + UT_ASSERT_EQ((unsigned long long)cluster_lmon_slow_iter_count(), 0ULL); + cluster_lmon_marker_complete_wakeup(); +} + /* ============================================================ * Test runner @@ -815,12 +836,13 @@ UT_TEST(test_lmon_public_symbols_linkable) int main(void) { - UT_PLAN(5); + UT_PLAN(6); UT_RUN(test_lmon_status_enum_values_frozen); UT_RUN(test_lmon_shared_state_size_under_4kb); UT_RUN(test_lmon_status_to_string_lookup); UT_RUN(test_lmon_status_unknown_returns_unknown); UT_RUN(test_lmon_public_symbols_linkable); + UT_RUN(test_lmon_iteration_counters_null_safe); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_lock_acquire.c b/src/test/cluster_unit/test_cluster_lock_acquire.c index 5636b8fff8..a5ca5a719c 100644 --- a/src/test/cluster_unit/test_cluster_lock_acquire.c +++ b/src/test/cluster_unit/test_cluster_lock_acquire.c @@ -365,6 +365,47 @@ cluster_grd_lookup_master(const ClusterResId *resid pg_attribute_unused()) return -1; } +/* spec-4.6a: the S4-reject diagnostic references the CSSD peer-state view; + * fixture has no CSSD shmem — stub DEAD-free defaults. */ +#include "cluster/cluster_cssd.h" +ClusterCssdPeerState +cluster_cssd_get_peer_state(int32 peer_id pg_attribute_unused()) +{ + return CLUSTER_CSSD_PEER_ALIVE; +} +const char * +cluster_cssd_peer_state_to_string(ClusterCssdPeerState s pg_attribute_unused()) +{ + return "alive"; +} + +/* spec-4.6a: the same diagnostic is the first ereport in this .o — swallow + * elog machinery (mirror test_cluster_grd.c pattern). */ +bool +errstart(int e pg_attribute_unused(), const char *d pg_attribute_unused()) +{ + return false; +} +bool +errstart_cold(int e pg_attribute_unused(), const char *d pg_attribute_unused()) +{ + return false; +} +void +errfinish(const char *f pg_attribute_unused(), int l pg_attribute_unused(), + const char *fn pg_attribute_unused()) +{} +int +errmsg_internal(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errcode(int s pg_attribute_unused()) +{ + return 0; +} + /* spec-4.6 P0 regression harness — test-controlled S3/S4 outcomes. * stub_reserve_result default NOT_READY keeps the legacy tests on the * pre-reservation-fail path; the S4 default-deny test flips it to OK diff --git a/src/test/cluster_unit/test_cluster_marker_async.c b/src/test/cluster_unit/test_cluster_marker_async.c new file mode 100644 index 0000000000..b90fa463a2 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_marker_async.c @@ -0,0 +1,246 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_marker_async.c + * Unit tests for the process-local qvotec marker async FSM. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_marker_async.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_marker_async.h" + +#undef printf +#undef fprintf +#undef snprintf +#undef sprintf +#undef vsnprintf +#undef vfprintf +#undef vprintf +#undef vsprintf +#undef strerror +#undef strerror_r + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +static int ut_set_latch_count = 0; + +void +ExceptionalCondition(const char *conditionName pg_attribute_unused(), + const char *fileName pg_attribute_unused(), + int lineNumber pg_attribute_unused()) +{ + abort(); +} + +void +SetLatch(Latch *latch pg_attribute_unused()) +{ + ut_set_latch_count++; +} + +static void +ut_reset(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq, pg_atomic_uint64 *completion_seq, + pg_atomic_uint32 *result_slot) +{ + cluster_marker_async_init(a); + pg_atomic_init_u64(request_seq, 0); + pg_atomic_init_u64(completion_seq, 0); + pg_atomic_init_u32(result_slot, 0); + ut_set_latch_count = 0; +} + +UT_TEST(test_init_defaults_idle) +{ + ClusterMarkerAsync a; + + cluster_marker_async_init(&a); + + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); + UT_ASSERT_EQ(a.inflight_seq, 0); + UT_ASSERT_EQ(a.target_node, -1); + UT_ASSERT(!a.has_staged_event); +} + +UT_TEST(test_submit_publishes_one_request) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + Latch latch; + bool ok; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + + ok = cluster_marker_async_submit(&a, &request_seq, &completion_seq, &latch, 1000, 5000, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, 7); + + UT_ASSERT(ok); + UT_ASSERT_EQ(pg_atomic_read_u64(&request_seq), 1); + UT_ASSERT_EQ(pg_atomic_read_u64(&completion_seq), 0); + UT_ASSERT_EQ(ut_set_latch_count, 1); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_SUBMITTED); + UT_ASSERT_EQ(a.inflight_seq, 1); + UT_ASSERT_EQ(a.deadline_us, 6000); + UT_ASSERT_EQ(a.kind, CLUSTER_MARKER_KIND_FENCE_FAILSTOP); + UT_ASSERT_EQ(a.target_node, 7); +} + +UT_TEST(test_poll_pending_before_completion) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + ClusterMarkerPollResult pr; + uint32 result = 99; + uint64 elapsed = 99; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_JOIN_PREPARE, 2)); + + pr = cluster_marker_async_poll(&a, &completion_seq, &result_slot, 1500, &result, &elapsed); + + UT_ASSERT_EQ(pr, CLUSTER_MARKER_POLL_PENDING); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_SUBMITTED); + UT_ASSERT_EQ(result, 0); + UT_ASSERT_EQ(elapsed, 0); +} + +UT_TEST(test_poll_ack_returns_result_and_elapsed) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + ClusterMarkerPollResult pr; + uint32 result = 0; + uint64 elapsed = 0; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_JOIN_COMMITTED, 3)); + pg_atomic_write_u32(&result_slot, 42); + pg_atomic_write_u64(&completion_seq, 1); + + pr = cluster_marker_async_poll(&a, &completion_seq, &result_slot, 2500, &result, &elapsed); + + UT_ASSERT_EQ(pr, CLUSTER_MARKER_POLL_ACKED); + UT_ASSERT_EQ(result, 42); + UT_ASSERT_EQ(elapsed, 1500); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); +} + +UT_TEST(test_poll_timeout_releases_submit_state) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + ClusterMarkerPollResult pr; + uint64 elapsed = 0; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 1000, + CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED, 4)); + + pr = cluster_marker_async_poll(&a, &completion_seq, &result_slot, 2000, NULL, &elapsed); + + UT_ASSERT_EQ(pr, CLUSTER_MARKER_POLL_TIMEOUT); + UT_ASSERT_EQ(elapsed, 1000); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); +} + +UT_TEST(test_mailbox_busy_does_not_publish) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + Latch latch; + bool ok; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + pg_atomic_write_u64(&request_seq, 2); + pg_atomic_write_u64(&completion_seq, 1); + + ok = cluster_marker_async_submit(&a, &request_seq, &completion_seq, &latch, 1000, 5000, + CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK, 5); + + UT_ASSERT(!ok); + UT_ASSERT_EQ(pg_atomic_read_u64(&request_seq), 2); + UT_ASSERT_EQ(ut_set_latch_count, 0); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); +} + +UT_TEST(test_reentrant_submit_does_not_bump_again) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + bool ok; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, 6)); + + ok = cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1200, 5000, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, 99); + + UT_ASSERT(ok); + UT_ASSERT_EQ(pg_atomic_read_u64(&request_seq), 1); + UT_ASSERT_EQ(a.inflight_seq, 1); + UT_ASSERT_EQ(a.kind, CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED); + UT_ASSERT_EQ(a.target_node, 6); +} + +UT_TEST(test_release_stage_clears_staged_record) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + a.has_staged_event = true; + a.staged_expect_epoch = 123; + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED, 8)); + + cluster_marker_async_release_stage(&a); + + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); + UT_ASSERT_EQ(a.inflight_seq, 0); + UT_ASSERT_EQ(a.target_node, -1); + UT_ASSERT(!a.has_staged_event); + UT_ASSERT_EQ(a.staged_expect_epoch, 0); +} + +int +main(void) +{ + UT_PLAN(8); + UT_RUN(test_init_defaults_idle); + UT_RUN(test_submit_publishes_one_request); + UT_RUN(test_poll_pending_before_completion); + UT_RUN(test_poll_ack_returns_result_and_elapsed); + UT_RUN(test_poll_timeout_releases_submit_state); + UT_RUN(test_mailbox_busy_does_not_publish); + UT_RUN(test_reentrant_submit_does_not_bump_again); + UT_RUN(test_release_stage_clears_staged_record); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/cluster_unit/test_cluster_pcm_lock.c b/src/test/cluster_unit/test_cluster_pcm_lock.c index c0ff3b6710..08ba568886 100644 --- a/src/test/cluster_unit/test_cluster_pcm_lock.c +++ b/src/test/cluster_unit/test_cluster_pcm_lock.c @@ -100,6 +100,9 @@ static Size fake_pcm_keysize = 0; static Size fake_pcm_entrysize = 0; static LWLock *fake_lwlock_held = NULL; static LWLockMode fake_lwlock_mode = LW_EXCLUSIVE; +static LWLock *fake_lwlock_stack[16]; +static LWLockMode fake_lwlock_mode_stack[16]; +static int fake_lwlock_depth = 0; static uint32 fake_init_wait_event_seen = 0; static uint32 fake_lwlock_wait_event_seen = 0; @@ -121,6 +124,7 @@ static struct { static sigjmp_buf ut_ereport_jump; static bool ut_ereport_jump_armed = false; static int ut_ereport_fired_count = 0; +static bool fake_local_x_upgrade_result = false; void ExceptionalCondition(const char *conditionName pg_attribute_unused(), @@ -130,6 +134,19 @@ ExceptionalCondition(const char *conditionName pg_attribute_unused(), abort(); } +/* spec-4.6a Amendment v1.2 (R5): the S->X upgrade is now wrapped in + * PG_TRY/PG_CATCH, which references the exception stack + re-throw. The + * unit's errfinish mock longjmps to its own buffer (never through + * PG_exception_stack), so these exist for the linker only. */ +sigjmp_buf *PG_exception_stack = NULL; +ErrorContextCallback *error_context_stack = NULL; + +void +pg_re_throw(void) +{ + abort(); +} + static BufferTag make_tag(uint32 blockno) { @@ -155,6 +172,10 @@ reset_fake_pcm_runtime(int max_entries) fake_pcm_keysize = 0; fake_pcm_entrysize = 0; fake_lwlock_held = NULL; + fake_lwlock_mode = LW_EXCLUSIVE; + fake_lwlock_depth = 0; + memset(fake_lwlock_stack, 0, sizeof(fake_lwlock_stack)); + memset(fake_lwlock_mode_stack, 0, sizeof(fake_lwlock_mode_stack)); fake_init_wait_event_seen = 0; fake_lwlock_wait_event_seen = 0; ut_wait_event_info_storage = 0; @@ -167,6 +188,7 @@ reset_fake_pcm_runtime(int max_entries) fake_cv_broadcast_count = 0; fake_cv_sleep_wait_event = 0; fake_cv_wake_release.armed = false; + fake_local_x_upgrade_result = false; cluster_node_id = 0; NBuffers = max_entries; cluster_pcm_grd_max_entries = max_entries; @@ -191,9 +213,20 @@ bool cluster_read_scache = false; bool cluster_gcs_block_local_x_upgrade(BufferTag tag); bool -cluster_gcs_block_local_x_upgrade(BufferTag tag pg_attribute_unused()) +cluster_gcs_block_local_x_upgrade(BufferTag tag) { - return false; + uint32 holders_bm; + int n; + + if (!fake_local_x_upgrade_result) + return false; + holders_bm = cluster_pcm_lock_query_s_holders_bitmap(tag); + if (cluster_node_id >= 0 && cluster_node_id < 32) + holders_bm &= ~((uint32)1u << (uint32)cluster_node_id); + for (n = 0; n < 32; n++) + if ((holders_bm & ((uint32)1u << n)) != 0) + (void)cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_S_TO_N_INVALIDATE, n); + return cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_S_TO_X_UPGRADE, cluster_node_id); } /* spec-4.7a D4 — stub CSSD peer liveness for the other-live-holder gate. @@ -348,6 +381,10 @@ LWLockInitialize(LWLock *lock pg_attribute_unused(), int tranche_id pg_attribute bool LWLockAcquire(LWLock *lock, LWLockMode mode) { + Assert(fake_lwlock_depth < (int)lengthof(fake_lwlock_stack)); + fake_lwlock_stack[fake_lwlock_depth] = lock; + fake_lwlock_mode_stack[fake_lwlock_depth] = mode; + fake_lwlock_depth++; fake_lwlock_held = lock; fake_lwlock_mode = mode; fake_lwlock_wait_event_seen = ut_wait_event_info_storage; @@ -357,14 +394,28 @@ LWLockAcquire(LWLock *lock, LWLockMode mode) void LWLockRelease(LWLock *lock) { - Assert(fake_lwlock_held == lock); - fake_lwlock_held = NULL; + Assert(fake_lwlock_depth > 0); + Assert(fake_lwlock_stack[fake_lwlock_depth - 1] == lock); + fake_lwlock_depth--; + fake_lwlock_stack[fake_lwlock_depth] = NULL; + if (fake_lwlock_depth > 0) { + fake_lwlock_held = fake_lwlock_stack[fake_lwlock_depth - 1]; + fake_lwlock_mode = fake_lwlock_mode_stack[fake_lwlock_depth - 1]; + } else { + fake_lwlock_held = NULL; + fake_lwlock_mode = LW_EXCLUSIVE; + } } bool LWLockHeldByMeInMode(LWLock *lock, LWLockMode mode) { - return fake_lwlock_held == lock && fake_lwlock_mode == mode; + int i; + + for (i = fake_lwlock_depth - 1; i >= 0; i--) + if (fake_lwlock_stack[i] == lock && fake_lwlock_mode_stack[i] == mode) + return true; + return false; } /* ---------- @@ -1280,6 +1331,79 @@ UT_TEST(test_pcm_d3_not_double_x) } } +UT_TEST(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades) +{ + BufferTag tag = make_tag(96); + BufferDesc buf; + bool save = cluster_gcs_block_local_cache; + + reset_fake_pcm_runtime(4); + fake_cssd_dead_node = -1; + cluster_gcs_block_local_cache = true; + fake_local_x_upgrade_result = true; + + cluster_node_id = 2; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(tag), (int)PCM_LOCK_MODE_S); + + memset(&buf, 0, sizeof(buf)); + buf.tag = tag; + buf.pcm_state = (uint8)PCM_STATE_N; + cluster_node_id = 0; + UT_ASSERT(cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_X)); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(tag), (int)PCM_LOCK_MODE_X); + UT_ASSERT(!cluster_pcm_master_other_live_holder_exists(tag, 0)); + UT_ASSERT(cluster_pcm_master_requester_is_holder(tag, 0, PCM_TRANS_N_TO_X)); + + cluster_gcs_block_local_cache = save; +} + +UT_TEST(test_pcm_dead_node_cleanup_drops_holder_records) +{ + BufferTag stag = make_tag(94); + BufferTag xtag = make_tag(95); + uint32 s_bitmap; + + reset_fake_pcm_runtime(4); + fake_cssd_dead_node = -1; + + /* Dead node 2 was the first S holder, so master_holder points at it. */ + cluster_node_id = 2; + cluster_pcm_lock_acquire(stag, PCM_LOCK_MODE_S); + cluster_node_id = 1; + cluster_pcm_lock_acquire(stag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_master_holder_node_by_tag(stag), 2); + UT_ASSERT(!cluster_pcm_lock_clean_leave_verify_no_leftover(2)); + + cluster_node_id = 2; + cluster_pcm_lock_acquire(xtag, PCM_LOCK_MODE_X); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(xtag), (int)PCM_LOCK_MODE_X); + + UT_ASSERT_EQ((uint64)cluster_pcm_lock_cleanup_on_node_dead(2), (uint64)2); + + s_bitmap = cluster_pcm_lock_query_s_holders_bitmap(stag); + UT_ASSERT_EQ((int)(s_bitmap & (1u << 2)), 0); + UT_ASSERT((s_bitmap & (1u << 1)) != 0); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(stag), (int)PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_master_holder_node_by_tag(stag), 1); + + UT_ASSERT_EQ((int)cluster_pcm_lock_query(xtag), (int)PCM_LOCK_MODE_N); + UT_ASSERT(cluster_pcm_lock_clean_leave_verify_no_leftover(2)); + UT_ASSERT((fake_cv_broadcast_count) >= (1)); + + /* Idempotent: a repeated dead-sweep pass has nothing left to clean. */ + UT_ASSERT_EQ((uint64)cluster_pcm_lock_cleanup_on_node_dead(2), (uint64)0); + + /* spec-4.6a D12 (r2-P1-3) pending-X form: a dead requester's parked X + * intent is cleared by the companion HC124 sweep the same dead-sweep hook + * drives, and the sweep is idempotent too. */ + cluster_pcm_lock_set_pending_x(stag, 2, 1234); + UT_ASSERT_EQ(cluster_pcm_lock_query_pending_x_requester(stag), 2); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_clear_pending_x_for_node(2), (uint64)1); + UT_ASSERT_EQ(cluster_pcm_lock_query_pending_x_requester(stag), -1); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_clear_pending_x_for_node(2), (uint64)0); +} + /* spec-5.2a D2 (U1): clean-page X-transfer arm is one-shot. arm(true) sets * the backend-local flag; consume() reads-and-clears it (the acquire path * calls consume() once so the eligibility can never leak into a SUBSEQUENT @@ -1313,7 +1437,7 @@ UT_TEST(test_clean_page_xfer_arm_is_one_shot) int main(void) { - UT_PLAN(39); + UT_PLAN(41); UT_RUN(test_pcm_lock_mode_constant_aliases_match_pcm_state); UT_RUN(test_pcm_lock_transition_count_is_9); UT_RUN(test_pcm_lock_transition_enum_values_are_1_to_9); @@ -1352,6 +1476,8 @@ main(void) UT_RUN(test_pcm_d1_recovering_gate_fail_closed); UT_RUN(test_pcm_d2_rebuild_from_redeclare); UT_RUN(test_pcm_d3_not_double_x); + UT_RUN(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades); + UT_RUN(test_pcm_dead_node_cleanup_drops_holder_records); UT_RUN(test_clean_page_xfer_arm_is_one_shot); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/cluster_unit/test_cluster_pi_shadow.c b/src/test/cluster_unit/test_cluster_pi_shadow.c index b9e82a4c02..7d075852dd 100644 --- a/src/test/cluster_unit/test_cluster_pi_shadow.c +++ b/src/test/cluster_unit/test_cluster_pi_shadow.c @@ -207,6 +207,9 @@ UT_TEST(test_gate_raw_compare_traps) } int +/* cppcheck-suppress constParameter + * Reason: main() keeps the standard test harness signature used by the + * other cluster_unit binaries; argv is intentionally unused. */ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) { /* stand-in for cluster_pi_shadow_shmem_init over malloc-backed memory */ diff --git a/src/test/cluster_unit/test_cluster_qvotec.c b/src/test/cluster_unit/test_cluster_qvotec.c index 62b2099932..3400d68962 100644 --- a/src/test/cluster_unit/test_cluster_qvotec.c +++ b/src/test/cluster_unit/test_cluster_qvotec.c @@ -212,6 +212,14 @@ void cluster_elog_init(void) {} +#include "cluster/cluster_inject.h" +bool +cluster_cr_injection_armed(const char *name pg_attribute_unused(), + uint64 *out_param pg_attribute_unused()) +{ + return false; +} + /* Step 3 D7 stubs: signal/ps_display/procsignal symbols not linked * here (cluster_qvotec.c references them for ClusterQvotecMain; * unit test never invokes Main, just address-takes for T-6). */ diff --git a/src/test/cluster_unit/test_cluster_reconfig.c b/src/test/cluster_unit/test_cluster_reconfig.c index 573406761c..6d6fcbfd0f 100644 --- a/src/test/cluster_unit/test_cluster_reconfig.c +++ b/src/test/cluster_unit/test_cluster_reconfig.c @@ -61,6 +61,14 @@ UT_DEFINE_GLOBALS(); +/* spec-2.29a: cluster_reconfig.c gates the async marker stage on + * MyBackendType == B_LMON. The fixture exercises the pre-2.29a bounded-wait + * semantics (write_fence submit stub returns FAILED synchronously), so run as + * a non-LMON backend; the async FSM itself is covered by + * test_cluster_marker_async.c. */ +#include "miscadmin.h" +BackendType MyBackendType = B_INVALID; + /* ============================================================ * Stubs — link cluster_reconfig.o + cluster_epoch.o standalone. @@ -357,6 +365,48 @@ cluster_write_fence_submit_marker(const ClusterFenceMarker *m pg_attribute_unuse { return CLUSTER_FENCE_MARKER_SUBMIT_FAILED; } +/* spec-2.29a review r1 P1-c: controllable async-marker stubs so the tick-level + * P1-1 invariants (bump-once while PENDING, node-remove zero-false-contest, + * publish deferred to ACK) can be hard-asserted. Defaults preserve the + * pre-existing inert behavior (submit rejected, poll idle). */ +static bool ut_fence_async_submit_ok = false; +static ClusterMarkerPollResult ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_IDLE; +static uint32 ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; +static int ut_fence_async_submit_calls = 0; +static int ut_fence_async_poll_calls = 0; + +bool +cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, + const ClusterFenceMarker *m pg_attribute_unused(), + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now pg_attribute_unused()) +{ + ut_fence_async_submit_calls++; + if (!ut_fence_async_submit_ok) + return false; + /* mirror the real wrapper's FSM effect so is_submitted() sees SUBMITTED */ + a->state = CLUSTER_MARKER_ASYNC_SUBMITTED; + a->kind = kind; + a->target_node = target_node; + return true; +} +ClusterMarkerPollResult +cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now pg_attribute_unused(), + uint32 *out_result, uint64 *out_elapsed_us) +{ + ut_fence_async_poll_calls++; + if (out_result != NULL) + *out_result = ut_fence_async_poll_result; + if (out_elapsed_us != NULL) + *out_elapsed_us = 0; + if (ut_fence_async_poll_pr == CLUSTER_MARKER_POLL_ACKED + || ut_fence_async_poll_pr == CLUSTER_MARKER_POLL_TIMEOUT) + a->state = CLUSTER_MARKER_ASYNC_IDLE; + return ut_fence_async_poll_pr; +} +void +cluster_lmon_marker_complete_wakeup(void) +{} /* spec-3.1 D7 stub: cluster_reconfig_apply_epoch_bump_as_coordinator * calls cluster_tt_status_flush_all. Fixture has no TT overlay shmem; @@ -962,6 +1012,142 @@ UT_TEST(test_reconfig_lmon_tick_survivor_does_not_advance_epoch) } +/* ============================================================ + * T-reconfig-2.29a — P1-1 staged-record hard assertions (spec-2.29a + * review r1 P1-c). + * + * (a) fail-stop fence stage: while the marker is PENDING across ticks + * the epoch is bumped EXACTLY ONCE and nothing publishes; the + * staged event publishes only on ACKED+ACK. + * (b) node-remove stage: a tick re-entry while the fence marker is + * PENDING returns 0 with *out_contest == false (no false contest + * against our own already-advanced baseline) and no re-bump. + * ============================================================ */ + +UT_TEST(test_reconfig_failstop_fence_stage_bump_once_while_pending) +{ + uint64 epoch_after_bump; + uint64 apply_before; + + ut_reset_mocks(); + reconfig_init_done = false; + epoch_init_done = false; + cluster_reconfig_shmem_init(); + cluster_epoch_shmem_init(); + + cluster_node_id = 0; + ut_in_quorum_value = true; + ut_declared_set[0] = true; + ut_declared_set[1] = true; + ut_peer_state[1] = CLUSTER_CSSD_PEER_DEAD; + ut_dead_generation = 1; + + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_ON; + MyBackendType = B_LMON; + ut_fence_async_submit_ok = true; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_PENDING; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + + apply_before = cluster_reconfig_get_apply_counter(); + + /* tick #1: stage-entry — bump once, submit, DO NOT publish. */ + cluster_reconfig_lmon_tick(); + epoch_after_bump = cluster_epoch_get_current(); + UT_ASSERT(epoch_after_bump > 0); + UT_ASSERT_EQ((unsigned long long)cluster_reconfig_get_apply_counter(), + (unsigned long long)apply_before); + + /* ticks #2/#3: marker PENDING — the bump path must NOT re-enter + * (P1-1 invariant 0: zero re-bump, zero publish, zero side-effects). */ + cluster_reconfig_lmon_tick(); + cluster_reconfig_lmon_tick(); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + UT_ASSERT_EQ((unsigned long long)cluster_reconfig_get_apply_counter(), + (unsigned long long)apply_before); + + /* tick #4: ACKED + ACK — the STAGED event publishes; still no re-bump. */ + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_ACKED; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_ACK; + cluster_reconfig_lmon_tick(); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + UT_ASSERT_EQ((unsigned long long)cluster_reconfig_get_apply_counter(), + (unsigned long long)(apply_before + 1)); + + /* restore fixture defaults for the neighboring tests */ + MyBackendType = B_INVALID; + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_OFF; + ut_fence_async_submit_ok = false; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_IDLE; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; +} + + +UT_TEST(test_reconfig_node_remove_stage_no_false_contest) +{ + uint64 baseline; + uint64 epoch_after_bump; + uint64 ret; + bool contest; + + ut_reset_mocks(); + reconfig_init_done = false; + epoch_init_done = false; + cluster_reconfig_shmem_init(); + cluster_epoch_shmem_init(); + + cluster_node_id = 0; + ut_in_quorum_value = true; + ut_declared_set[0] = true; + ut_declared_set[1] = true; + + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_ON; + MyBackendType = B_LMON; + ut_fence_async_submit_ok = true; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_PENDING; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + + baseline = cluster_epoch_get_current(); + + /* call #1: guarded CAS advances baseline→baseline+1, stages the fence + * marker, returns 0 (not yet committed), contest MUST be false. */ + contest = true; + ret = cluster_reconfig_apply_node_removed_as_coordinator(1, baseline, 42, 7, &contest); + UT_ASSERT_EQ((unsigned long long)ret, 0ULL); + UT_ASSERT(!contest); + epoch_after_bump = cluster_epoch_get_current(); + UT_ASSERT_EQ((unsigned long long)epoch_after_bump, (unsigned long long)(baseline + 1)); + + /* call #2 (tick re-entry with the SAME stale baseline while PENDING): + * without the staged record this would re-run the baseline CAS against + * our own advanced epoch and report a FALSE contest. P1-1: it must + * poll the stage instead — 0, contest==false, no second bump. */ + contest = true; + ret = cluster_reconfig_apply_node_removed_as_coordinator(1, baseline, 42, 7, &contest); + UT_ASSERT_EQ((unsigned long long)ret, 0ULL); + UT_ASSERT(!contest); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + + /* call #3: ACKED + ACK — removal publishes at the STAGED epoch. */ + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_ACKED; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_ACK; + contest = true; + ret = cluster_reconfig_apply_node_removed_as_coordinator(1, baseline, 42, 7, &contest); + UT_ASSERT_EQ((unsigned long long)ret, (unsigned long long)epoch_after_bump); + UT_ASSERT(!contest); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + + MyBackendType = B_INVALID; + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_OFF; + ut_fence_async_submit_ok = false; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_IDLE; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; +} + + /* ============================================================ * T-reconfig-8 — ProcessInterrupts I6 guard (D4). * @@ -1575,6 +1761,8 @@ main(void) /* T-reconfig-3 — lmon_tick dedup. */ UT_RUN(test_reconfig_lmon_tick_dedups_same_event_id); UT_RUN(test_reconfig_lmon_tick_refires_on_dead_gen_bump); + UT_RUN(test_reconfig_failstop_fence_stage_bump_once_while_pending); + UT_RUN(test_reconfig_node_remove_stage_no_false_contest); /* T-reconfig-4 / 4b — Q2 A'' + I2 + L20 + F11 + empty-dead-set gates. */ UT_RUN(test_reconfig_lmon_tick_skips_when_not_in_quorum); diff --git a/src/test/cluster_unit/test_cluster_xid_authority.c b/src/test/cluster_unit/test_cluster_xid_authority.c new file mode 100644 index 0000000000..19b98855fa --- /dev/null +++ b/src/test/cluster_unit/test_cluster_xid_authority.c @@ -0,0 +1,890 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_xid_authority.c + * Runtime unit tests for the shared XID authority + native-era + * prehistory module (spec-6.15b D1/D2): on-disk layout lock, the pure + * classification and payload-sizing logic, the torn-safe read/write + * round trip with .bak fallback, seal/flag monotonicity, and a real + * prehistory publish -> adopt byte round trip between two fake PGDATA + * pg_xact trees. + * + * Like test_cluster_oid_lease.c this exercises the module for REAL + * against temp directories: fd.c openers map onto open(2)/close(2), + * pg_fsync onto fsync(2), durable_rename onto rename(2). The errstart + * stub aborts on elevel >= ERROR, so every FATAL/PANIC leg (corrupt + * authority at adopt, cap refusal, era re-entry) is TAP territory + * (t/361), not unit territory; reaching an abort here is a real bug. + * + * Covers (spec §4): + * U1 classify truth table (short / magic / CRC / valid), both files + * U2 publish monotonicity (native_hw never lowered) + * U3 flags one-way (SEALED / CLUSTER_ERA never cleared) + * U4 prehistory payload sizing + publish/adopt byte round trip + * U5 header layout locked by offset assertions + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_xid_authority.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-6.15b-xid-authority-native-era.md (D1/D2, §4 U1-U5) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include +#include +#include +#include + +#include "cluster/cluster_xid_authority.h" +#include "port/pg_crc32c.h" +#include "storage/fd.h" +#include "utils/elog.h" + +#undef printf +#undef fprintf +#undef snprintf +#undef sprintf +#undef vsnprintf +#undef vfprintf +#undef vprintf +#undef vsprintf +#undef strerror +#undef strerror_r + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +/* Global read by cluster_xid_authority.o's file I/O. */ +char *cluster_shared_data_dir = NULL; + +/* ---- Assert + ereport machinery (aborts on ERROR; the read paths never + * ereport, the write paths only PANIC on real I/O failure). ---- */ +void +ExceptionalCondition(const char *conditionName, const char *fileName, int lineNumber) +{ + printf("# Assert failed: %s at %s:%d\n", conditionName, fileName, lineNumber); + abort(); +} +bool +errstart(int elevel, const char *domain pg_attribute_unused()) +{ + if (elevel >= ERROR) { + printf("# unexpected ereport(elevel=%d) -- aborting\n", elevel); + abort(); + } + return false; +} +bool +errstart_cold(int elevel, const char *domain) +{ + return errstart(elevel, domain); +} +void +errfinish(const char *filename pg_attribute_unused(), int lineno pg_attribute_unused(), + const char *funcname pg_attribute_unused()) +{} +int +errcode(int sqlerrcode pg_attribute_unused()) +{ + return 0; +} +int +errcode_for_file_access(void) +{ + return 0; +} +int +errmsg(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errmsg_internal(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errdetail(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errhint(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} + +/* ---- functional fd.c stubs: map onto the kernel ---- */ +int +OpenTransientFile(const char *fileName, int fileFlags) +{ + return open(fileName, fileFlags, 0600); +} +int +CloseTransientFile(int fd) +{ + return close(fd); +} +int +pg_fsync(int fd) +{ + return fsync(fd); +} +int +durable_rename(const char *oldfile, const char *newfile, int elevel pg_attribute_unused()) +{ + return (rename(oldfile, newfile) != 0) ? -1 : 0; +} + +/* ---- temp dir scaffolding ---- */ +static char test_root[MAXPGPATH]; + +static void +setup_shared_dir(void) +{ + char globaldir[MAXPGPATH]; + + snprintf(test_root, sizeof(test_root), "/tmp/pgrac_xid_auth_ut_%d", (int)getpid()); + mkdir(test_root, 0700); + snprintf(globaldir, sizeof(globaldir), "%s/global", test_root); + mkdir(globaldir, 0700); + cluster_shared_data_dir = test_root; +} + +static void +unlink_files(void) +{ + const char *rels[] = { CLUSTER_XID_AUTHORITY_REL_PATH, CLUSTER_XID_AUTHORITY_BAK_REL_PATH, + CLUSTER_XID_PREHISTORY_REL_PATH, CLUSTER_XID_PREHISTORY_BAK_REL_PATH }; + char p[MAXPGPATH]; + int i; + + for (i = 0; i < (int)lengthof(rels); i++) { + snprintf(p, sizeof(p), "%s/%s", test_root, rels[i]); + unlink(p); + } +} + +/* Build a fake PGDATA with a pg_xact/0000 of n_pages, filled with byte. */ +static void +make_fake_pgdata(char *pgdata, size_t len, const char *tag, int n_pages, unsigned char byte) +{ + char dir[MAXPGPATH]; + char seg[MAXPGPATH]; + char page[BLCKSZ]; + int fd; + int i; + + snprintf(pgdata, len, "%s/pgdata_%s", test_root, tag); + mkdir(pgdata, 0700); + snprintf(dir, sizeof(dir), "%s/pg_xact", pgdata); + mkdir(dir, 0700); + snprintf(seg, sizeof(seg), "%s/0000", dir); + fd = open(seg, O_RDWR | O_CREAT | O_TRUNC, 0600); + memset(page, byte, sizeof(page)); + for (i = 0; i < n_pages; i++) + (void)!write(fd, page, sizeof(page)); + close(fd); +} + +/* Build a fake PGDATA whose pg_xact spans SLRU segments: n_pages whole CLOG + * pages, 32 per segment file (0000, 0001, ...), page p filled with byte + * (p & 0xFF) so cross-segment ordering is byte-verifiable. */ +static void +make_fake_pgdata_multiseg(char *pgdata, size_t len, const char *tag, int n_pages) +{ + char dir[MAXPGPATH]; + char seg[MAXPGPATH]; + char page[BLCKSZ]; + int fd = -1; + int cur_seg = -1; + int p; + + snprintf(pgdata, len, "%s/pgdata_%s", test_root, tag); + mkdir(pgdata, 0700); + snprintf(dir, sizeof(dir), "%s/pg_xact", pgdata); + mkdir(dir, 0700); + for (p = 0; p < n_pages; p++) { + int segno = p / 32; + + if (segno != cur_seg) { + if (fd >= 0) + close(fd); + snprintf(seg, sizeof(seg), "%s/%04X", dir, segno); + fd = open(seg, O_RDWR | O_CREAT | O_TRUNC, 0600); + cur_seg = segno; + } + memset(page, p & 0xFF, sizeof(page)); + (void)!write(fd, page, sizeof(page)); + } + if (fd >= 0) + close(fd); +} + +/* ============================================================ + * U5: on-disk layout locked + * ============================================================ */ + +UT_TEST(test_layout_offsets_locked) +{ + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, magic), 0); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, version), 4); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, flags), 8); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, reserved), 12); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, native_hw_full), 16); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, next_multi), 24); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, crc), 32); + + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, magic), 0); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, version), 4); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, native_hw_full), 8); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, payload_len), 16); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, reserved), 20); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, crc), 24); +} + +/* ============================================================ + * U1: classify truth table + * ============================================================ */ + +UT_TEST(test_classify_short_magic_crc_valid) +{ + ClusterXidAuthorityHeader hdr; + char buf[sizeof(ClusterXidAuthorityHeader)]; + + memset(buf, 0, sizeof(buf)); + UT_ASSERT_EQ(cluster_xid_authority_classify(NULL, 0), CLUSTER_XID_AUTHORITY_INVALID_SHORT); + UT_ASSERT_EQ(cluster_xid_authority_classify(buf, 3), CLUSTER_XID_AUTHORITY_INVALID_SHORT); + + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = 0xDEADBEEF; + memcpy(buf, &hdr, sizeof(hdr)); + UT_ASSERT_EQ(cluster_xid_authority_classify(buf, sizeof(buf)), + CLUSTER_XID_AUTHORITY_INVALID_MAGIC); + + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = CLUSTER_XID_AUTHORITY_MAGIC; + hdr.version = CLUSTER_XID_AUTHORITY_VERSION; + hdr.native_hw_full = 816; + INIT_CRC32C(hdr.crc); + COMP_CRC32C(hdr.crc, (char *)&hdr, offsetof(ClusterXidAuthorityHeader, crc)); + FIN_CRC32C(hdr.crc); + memcpy(buf, &hdr, sizeof(hdr)); + UT_ASSERT_EQ(cluster_xid_authority_classify(buf, sizeof(buf)), CLUSTER_XID_AUTHORITY_VALID); + + buf[16] ^= 0x01; /* flip a native_hw bit -> CRC mismatch */ + UT_ASSERT_EQ(cluster_xid_authority_classify(buf, sizeof(buf)), + CLUSTER_XID_AUTHORITY_INVALID_CRC); +} + +/* ============================================================ + * U4: prehistory payload sizing + * ============================================================ */ + +UT_TEST(test_payload_bytes_boundaries) +{ + const uint32 xids_per_page = BLCKSZ * 4; /* CLOG: 2 bits/xact */ + + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(0), 0); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(1), (uint32)BLCKSZ); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(816), (uint32)BLCKSZ); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(xids_per_page), (uint32)BLCKSZ); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(xids_per_page + 1), (uint32)(2 * BLCKSZ)); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(CLUSTER_XID_PREHISTORY_MAX_XID), + (uint32)(CLUSTER_XID_PREHISTORY_MAX_XID / 4)); + /* over the refusal cap: 0 = "no valid payload" (publish FATALs) */ + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(CLUSTER_XID_PREHISTORY_MAX_XID + 1), 0); +} + +/* ============================================================ + * torn-safe authority round trips + * ============================================================ */ + +UT_TEST(test_seed_if_absent_creates_unsealed) +{ + ClusterXidAuthorityHeader got; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_present(), false); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + UT_ASSERT_EQ(cluster_xid_authority_present(), true); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 791); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, 0); + + /* second seed is a no-op and keeps the existing value */ + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(123456), false); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 791); +} + +UT_TEST(test_publish_monotone_and_seal) +{ + ClusterXidAuthorityHeader got; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + + /* interim (unsealed) publish raises the high-water */ + cluster_xid_authority_publish_native(810, 1, false); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 810); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + + /* sealing publish raises further and stamps SEALED */ + cluster_xid_authority_publish_native(816, 1, true); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 816); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, CLUSTER_XID_AUTHORITY_FLAG_SEALED); + + /* a lower publish never lowers, and never clears SEALED */ + cluster_xid_authority_publish_native(500, 1, false); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 816); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, CLUSTER_XID_AUTHORITY_FLAG_SEALED); +} + +UT_TEST(test_mark_cluster_era_one_way) +{ + ClusterXidAuthorityHeader got; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + + cluster_xid_authority_mark_cluster_era(); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.flags, + CLUSTER_XID_AUTHORITY_FLAG_SEALED | CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); + + /* later publishes keep both flags (never cleared) */ + cluster_xid_authority_publish_native(900, 1, false); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 900); + UT_ASSERT_EQ(got.flags, + CLUSTER_XID_AUTHORITY_FLAG_SEALED | CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); +} + +UT_TEST(test_primary_corrupt_falls_back_to_bak) +{ + ClusterXidAuthorityHeader got; + char p[MAXPGPATH]; + int fd; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + /* second write rolls the 791 image into .bak */ + cluster_xid_authority_publish_native(816, 1, true); + + /* corrupt the primary in place */ + snprintf(p, sizeof(p), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_REL_PATH); + fd = open(p, O_RDWR, 0600); + (void)!write(fd, "garbage!", 8); + close(fd); + + /* falls back to the rolled 791 image */ + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 791); +} + +UT_TEST(test_present_distinguishes_corrupt_from_absent) +{ + ClusterXidAuthorityHeader got; + char p[MAXPGPATH]; + int fd; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_present(), false); + + /* a lone corrupt primary: present=true, read=false (fail-closed) */ + snprintf(p, sizeof(p), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_REL_PATH); + fd = open(p, O_RDWR | O_CREAT | O_TRUNC, 0600); + (void)!write(fd, "garbage!", 8); + close(fd); + UT_ASSERT_EQ(cluster_xid_authority_present(), true); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), false); +} + +/* ============================================================ + * U4: prehistory publish -> adopt byte round trip + * ============================================================ */ + +UT_TEST(test_prehistory_publish_adopt_round_trip) +{ + char pgdata_seed[MAXPGPATH]; + char pgdata_join[MAXPGPATH]; + char seg[MAXPGPATH]; + char seed_page[BLCKSZ]; + char join_page[BLCKSZ]; + int fd; + + unlink_files(); + + /* seed: one clog page of 0xAA; joiner: same-size page of zeros */ + make_fake_pgdata(pgdata_seed, sizeof(pgdata_seed), "seed", 1, 0xAA); + make_fake_pgdata(pgdata_join, sizeof(pgdata_join), "join", 1, 0x00); + + cluster_xid_prehistory_publish(pgdata_seed, 816); + UT_ASSERT_EQ(cluster_xid_prehistory_was_adopted(), false); + cluster_xid_prehistory_adopt(pgdata_join, 816); + UT_ASSERT_EQ(cluster_xid_prehistory_was_adopted(), true); + + snprintf(seg, sizeof(seg), "%s/pg_xact/0000", pgdata_seed); + fd = open(seg, O_RDONLY, 0); + UT_ASSERT_EQ(read(fd, seed_page, BLCKSZ), BLCKSZ); + close(fd); + snprintf(seg, sizeof(seg), "%s/pg_xact/0000", pgdata_join); + fd = open(seg, O_RDONLY, 0); + UT_ASSERT_EQ(read(fd, join_page, BLCKSZ), BLCKSZ); + close(fd); + UT_ASSERT_EQ(memcmp(seed_page, join_page, BLCKSZ), 0); + + /* adopt is idempotent: run again, bytes unchanged */ + cluster_xid_prehistory_adopt(pgdata_join, 816); + fd = open(seg, O_RDONLY, 0); + UT_ASSERT_EQ(read(fd, join_page, BLCKSZ), BLCKSZ); + close(fd); + UT_ASSERT_EQ(memcmp(seed_page, join_page, BLCKSZ), 0); +} + +UT_TEST(test_prehistory_multi_segment_round_trip) +{ + const uint64 xids_per_page = (uint64)BLCKSZ * 4; + const int n_pages = 33; /* segment 0000 full (32 pages) + 0001 (1 page) */ + const uint64 hw = (uint64)n_pages * xids_per_page; + char pgdata_seed[MAXPGPATH]; + char pgdata_join[MAXPGPATH]; + char seg[MAXPGPATH]; + char want[BLCKSZ]; + char got[BLCKSZ]; + int p; + + unlink_files(); + make_fake_pgdata_multiseg(pgdata_seed, sizeof(pgdata_seed), "seedms", n_pages); + /* short pre-seed clone: only one zeroed page in segment 0000 */ + make_fake_pgdata_multiseg(pgdata_join, sizeof(pgdata_join), "joinms", 1); + + cluster_xid_prehistory_publish(pgdata_seed, hw); + cluster_xid_prehistory_adopt(pgdata_join, hw); + + for (p = 0; p < n_pages; p++) { + int fd; + + snprintf(seg, sizeof(seg), "%s/pg_xact/%04X", pgdata_join, p / 32); + fd = open(seg, O_RDONLY, 0); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(lseek(fd, (off_t)(p % 32) * BLCKSZ, SEEK_SET), (off_t)(p % 32) * BLCKSZ); + UT_ASSERT_EQ(read(fd, got, BLCKSZ), BLCKSZ); + close(fd); + memset(want, p & 0xFF, sizeof(want)); + UT_ASSERT_EQ(memcmp(got, want, BLCKSZ), 0); + } +} + +UT_TEST(test_unseal_survives_primary_corruption_via_bak) +{ + ClusterXidAuthorityHeader got; + char path[MAXPGPATH]; + int fd; + + /* F1: after begin_native_run, NO on-disk copy may still say SEALED -- + * a corrupt primary falling back to a stale SEALED .bak would hand a + * joiner the previous pass's high-water (8.A false-invisible). */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + cluster_xid_authority_begin_native_run(); + + snprintf(path, sizeof(path), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_REL_PATH); + fd = open(path, O_RDWR, 0600); + (void)!write(fd, "garbage!", 8); + close(fd); + + if (cluster_xid_authority_read(&got)) + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + /* read failing entirely (no trustworthy copy) is also acceptable: + * joiners fail closed on unavailable exactly like on unsealed. */ +} + +UT_TEST(test_mark_cluster_era_survives_primary_corruption_via_bak) +{ + ClusterXidAuthorityHeader got; + char path[MAXPGPATH]; + int fd; + + /* F1 variant: after mark_cluster_era, no on-disk copy may lack the + * one-way CLUSTER_ERA flag -- a stale .bak without it would let an + * enabled=off boot re-enter the native era on a formed tree. */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + cluster_xid_authority_mark_cluster_era(); + + snprintf(path, sizeof(path), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_REL_PATH); + fd = open(path, O_RDWR, 0600); + (void)!write(fd, "garbage!", 8); + close(fd); + + if (cluster_xid_authority_read(&got)) + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, + CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); +} + +/* + * write_raw_image / read_raw_image -- install / fetch one fixed-size + * authority image at `path` (raw open/write, no CRC games: the images + * moved around here are byte copies of real, valid on-disk images). + */ +static void +write_raw_image(const char *path, const char *image, size_t len) +{ + int fd; + + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0600); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(write(fd, image, len), (long long)len); + close(fd); +} + +static void +read_raw_image(const char *path, char *image, size_t len) +{ + int fd; + + fd = open(path, O_RDONLY, 0); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(read(fd, image, len), (long long)len); + close(fd); +} + +UT_TEST(test_unseal_torn_crash_repairs_stale_sealed_bak) +{ + ClusterXidAuthorityHeader got; + char primary_path[MAXPGPATH]; + char bak_path[MAXPGPATH]; + char old_image[sizeof(ClusterXidAuthorityHeader)]; + char bak_image[sizeof(ClusterXidAuthorityHeader)]; + int fd; + + /* review r3-X1: a crash BETWEEN write_header_both's two renames leaves + * primary=UNSEALED(new) / .bak=SEALED(old high-water). Every native-era + * boot re-runs the transition, which must re-assert BOTH copies; before + * the fix it early-returned on the already-unsealed primary and the + * stale SEALED .bak survived the whole run -- any later primary read + * failure resurrected the previous pass's high-water (8.A). */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + + /* capture the pre-transition (SEALED) primary image */ + snprintf(primary_path, sizeof(primary_path), "%s/%s", test_root, + CLUSTER_XID_AUTHORITY_REL_PATH); + snprintf(bak_path, sizeof(bak_path), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_BAK_REL_PATH); + read_raw_image(primary_path, old_image, sizeof(old_image)); + + /* complete the transition, then re-install the old SEALED image as + * .bak, simulating the crash window between the two durable renames */ + cluster_xid_authority_begin_native_run(); + write_raw_image(bak_path, old_image, sizeof(old_image)); + + /* restart repair path: the next native-era boot re-runs the transition */ + cluster_xid_authority_begin_native_run(); + + /* the .bak copy itself must have been repaired to an unsealed image */ + read_raw_image(bak_path, bak_image, sizeof(bak_image)); + UT_ASSERT_EQ(cluster_xid_authority_classify(bak_image, sizeof(bak_image)), + CLUSTER_XID_AUTHORITY_VALID); + memcpy(&got, bak_image, sizeof(got)); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + + /* a later primary read failure must fall back to the REPAIRED image, + * not resurrect the previous pass's sealed high-water */ + fd = open(primary_path, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + (void)!write(fd, "garbage!", 8); + close(fd); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 816); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); +} + +UT_TEST(test_mark_cluster_era_torn_crash_repairs_stale_bak) +{ + ClusterXidAuthorityHeader got; + char primary_path[MAXPGPATH]; + char bak_path[MAXPGPATH]; + char old_image[sizeof(ClusterXidAuthorityHeader)]; + char bak_image[sizeof(ClusterXidAuthorityHeader)]; + int fd; + + /* review r3-X1 variant: the same torn-crash window across the + * CLUSTER_ERA stamp leaves a .bak without the one-way flag; a .bak + * fallback would then let an enabled=off boot re-enter the native era + * on a formed tree. The stamp is re-run by every cluster boot and + * must repair the lagging copy. */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + + snprintf(primary_path, sizeof(primary_path), "%s/%s", test_root, + CLUSTER_XID_AUTHORITY_REL_PATH); + snprintf(bak_path, sizeof(bak_path), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_BAK_REL_PATH); + read_raw_image(primary_path, old_image, sizeof(old_image)); + + cluster_xid_authority_mark_cluster_era(); + write_raw_image(bak_path, old_image, sizeof(old_image)); + + /* restart repair path: the next cluster boot re-runs the stamp */ + cluster_xid_authority_mark_cluster_era(); + + read_raw_image(bak_path, bak_image, sizeof(bak_image)); + UT_ASSERT_EQ(cluster_xid_authority_classify(bak_image, sizeof(bak_image)), + CLUSTER_XID_AUTHORITY_VALID); + memcpy(&got, bak_image, sizeof(got)); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, + CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); + + fd = open(primary_path, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + (void)!write(fd, "garbage!", 8); + close(fd); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, + CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); +} + +UT_TEST(test_prefix_check_divergence_truth_table) +{ + const uint64 xids_per_page = (uint64)BLCKSZ * 4; + char pgdata_seed[MAXPGPATH]; + char pgdata_join[MAXPGPATH]; + char seg[MAXPGPATH]; + int fd; + + /* F2: identical prefix -> CONSISTENT (both trees one page of 0xAA) */ + unlink_files(); + make_fake_pgdata(pgdata_seed, sizeof(pgdata_seed), "f2seed", 1, 0xAA); + make_fake_pgdata(pgdata_join, sizeof(pgdata_join), "f2join", 1, 0xAA); + cluster_xid_prehistory_publish(pgdata_seed, 816); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816, 0), + CLUSTER_XID_PREFIX_CONSISTENT); + + /* flip one status byte INSIDE the limit -> DIVERGED */ + snprintf(seg, sizeof(seg), "%s/pg_xact/0000", pgdata_join); + fd = open(seg, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(lseek(fd, 100, SEEK_SET), 100); /* byte 100 = xids 400..403 */ + (void)!write(fd, "X", 1); + close(fd); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816, 0), + CLUSTER_XID_PREFIX_DIVERGED); + + /* the same flipped byte OUTSIDE the limit -> CONSISTENT (adopt arm: + * bytes at/after own_next belong to the seed alone) */ + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 400, 0), + CLUSTER_XID_PREFIX_CONSISTENT); + + /* partial-byte boundary: xid 400's 2-bit slot enters scope at limit 401 */ + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 401, 0), + CLUSTER_XID_PREFIX_DIVERGED); + + /* missing local page inside the comparable range -> fail-closed + * UNAVAILABLE, never CONSISTENT: with oldestXid=0 nothing below the + * hole is frozen, so "shorter clone" is not a possible innocent + * explanation (review r3-X2) */ + unlink(seg); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816, 0), + CLUSTER_XID_PREFIX_UNAVAILABLE); + + /* no trustworthy blob -> UNAVAILABLE */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816, 0), + CLUSTER_XID_PREFIX_UNAVAILABLE); + + /* multi-segment divergence: 33-page trees, flip a byte in segment 0001 */ + { + const int n_pages = 33; + const uint64 hw = (uint64)n_pages * xids_per_page; + + unlink_files(); + make_fake_pgdata_multiseg(pgdata_seed, sizeof(pgdata_seed), "f2seedms", n_pages); + make_fake_pgdata_multiseg(pgdata_join, sizeof(pgdata_join), "f2joinms", n_pages); + cluster_xid_prehistory_publish(pgdata_seed, hw); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, 0), + CLUSTER_XID_PREFIX_CONSISTENT); + snprintf(seg, sizeof(seg), "%s/pg_xact/0001", pgdata_join); + fd = open(seg, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + (void)!write(fd, "Z", 1); + close(fd); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, 0), + CLUSTER_XID_PREFIX_DIVERGED); + } +} + +UT_TEST(test_prefix_check_front_truncation) +{ + const uint64 xids_per_page = (uint64)BLCKSZ * 4; + const int n_pages = 33; + const uint64 hw = (uint64)n_pages * xids_per_page; + const uint64 page32_first = 32 * xids_per_page; + char pgdata_seed[MAXPGPATH]; + char pgdata_join[MAXPGPATH]; + char seg[MAXPGPATH]; + unsigned char b; + int fd; + + /* review r3-X2 (a): front truncation is no alibi. SimpleLruTruncate + * removed the joiner's segment 0000 (oldestXid frozen past it), but the + * surviving page 32 diverges inside the comparable range -> DIVERGED. + * Pre-fix, break-on-first-missing-local-page returned CONSISTENT and + * the adopt arm then overwrote the joiner's own live-xid outcomes. */ + unlink_files(); + make_fake_pgdata_multiseg(pgdata_seed, sizeof(pgdata_seed), "ftseed", n_pages); + make_fake_pgdata_multiseg(pgdata_join, sizeof(pgdata_join), "ftjoin", n_pages); + cluster_xid_prehistory_publish(pgdata_seed, hw); + snprintf(seg, sizeof(seg), "%s/pg_xact/0000", pgdata_join); + UT_ASSERT_EQ(unlink(seg), 0); + snprintf(seg, sizeof(seg), "%s/pg_xact/0001", pgdata_join); + fd = open(seg, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(lseek(fd, 600, SEEK_SET), 600); /* byte 600 = slots 2400..2403 */ + (void)!write(fd, "X", 1); + close(fd); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, page32_first), + CLUSTER_XID_PREFIX_DIVERGED); + + /* (b) frozen exemption: restore byte 600 (page fill 0x20), then flip + * ONLY slot 2400's two low bits (0x20 -> 0x21). With oldestXid at slot + * 2401 the divergence sits strictly below the boundary, mid-byte: the + * lead mask must exclude it -> CONSISTENT (missing segment 0000 below + * page(oldestXid) is equally exempt). */ + fd = open(seg, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(lseek(fd, 600, SEEK_SET), 600); + b = 0x21; + (void)!write(fd, &b, 1); + close(fd); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, page32_first + 2401), + CLUSTER_XID_PREFIX_CONSISTENT); + + /* same bytes with oldestXid AT the divergent slot -> DIVERGED (the + * boundary slot itself is comparable) */ + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, page32_first + 2400), + CLUSTER_XID_PREFIX_DIVERGED); + + /* review r3-X2 (c): a hole AT/ABOVE page(oldestXid) inside the + * comparable range is an anomaly -> UNAVAILABLE (fail-closed), never + * CONSISTENT */ + UT_ASSERT_EQ(unlink(seg), 0); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, page32_first + 2400), + CLUSTER_XID_PREFIX_UNAVAILABLE); +} + +UT_TEST(test_prehistory_classify_corrupt) +{ + char buf[64]; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_prehistory_classify(NULL, 0), CLUSTER_XID_AUTHORITY_INVALID_SHORT); + + /* publish a real blob, then flip a payload bit -> CRC catches it */ + { + char pgdata_seed[MAXPGPATH]; + char p[MAXPGPATH]; + char *image; + struct stat st; + int fd; + int r; + + make_fake_pgdata(pgdata_seed, sizeof(pgdata_seed), "seed2", 1, 0x55); + cluster_xid_prehistory_publish(pgdata_seed, 816); + + snprintf(p, sizeof(p), "%s/%s", test_root, CLUSTER_XID_PREHISTORY_REL_PATH); + UT_ASSERT_EQ(stat(p, &st), 0); + image = malloc(st.st_size); + fd = open(p, O_RDONLY, 0); + r = (int)read(fd, image, st.st_size); + close(fd); + UT_ASSERT_EQ(r, (int)st.st_size); + UT_ASSERT_EQ(cluster_xid_prehistory_classify(image, st.st_size), + CLUSTER_XID_AUTHORITY_VALID); + + image[sizeof(ClusterXidPrehistoryHeader) + 100] ^= 0x01; + UT_ASSERT_EQ(cluster_xid_prehistory_classify(image, st.st_size), + CLUSTER_XID_AUTHORITY_INVALID_CRC); + free(image); + } + + memset(buf, 0, sizeof(buf)); + UT_ASSERT_EQ(cluster_xid_prehistory_classify(buf, sizeof(buf)), + CLUSTER_XID_AUTHORITY_INVALID_MAGIC); +} + +UT_TEST(test_begin_native_run_unseals_before_cluster_era) +{ + ClusterXidAuthorityHeader got; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + + /* a follow-up native run clears SEALED but keeps the high-water */ + cluster_xid_authority_begin_native_run(); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 816); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + + /* idempotent while open */ + cluster_xid_authority_begin_native_run(); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + + /* the run's clean shutdown re-publishes and re-seals monotonically */ + cluster_xid_authority_publish_native(900, 1, true); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 900); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, CLUSTER_XID_AUTHORITY_FLAG_SEALED); +} + +int +main(void) +{ + setup_shared_dir(); + + UT_PLAN(18); + UT_RUN(test_layout_offsets_locked); + UT_RUN(test_classify_short_magic_crc_valid); + UT_RUN(test_payload_bytes_boundaries); + UT_RUN(test_seed_if_absent_creates_unsealed); + UT_RUN(test_publish_monotone_and_seal); + UT_RUN(test_mark_cluster_era_one_way); + UT_RUN(test_begin_native_run_unseals_before_cluster_era); + UT_RUN(test_unseal_survives_primary_corruption_via_bak); + UT_RUN(test_mark_cluster_era_survives_primary_corruption_via_bak); + UT_RUN(test_unseal_torn_crash_repairs_stale_sealed_bak); + UT_RUN(test_mark_cluster_era_torn_crash_repairs_stale_bak); + UT_RUN(test_primary_corrupt_falls_back_to_bak); + UT_RUN(test_present_distinguishes_corrupt_from_absent); + UT_RUN(test_prehistory_publish_adopt_round_trip); + UT_RUN(test_prehistory_multi_segment_round_trip); + UT_RUN(test_prefix_check_divergence_truth_table); + UT_RUN(test_prefix_check_front_truncation); + UT_RUN(test_prehistory_classify_corrupt); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/perl/PostgreSQL/Test/ClusterQuad.pm b/src/test/perl/PostgreSQL/Test/ClusterQuad.pm index 3b3a2bb29f..873852fd76 100644 --- a/src/test/perl/PostgreSQL/Test/ClusterQuad.pm +++ b/src/test/perl/PostgreSQL/Test/ClusterQuad.pm @@ -55,6 +55,29 @@ use PostgreSQL::Test::Utils; our $NODES = 4; + +# Relocate an init_from_backup node's copied pg_wal into its shared WAL thread +# directory, matching the layout initdb -X produces for the seed node. +sub _relocate_backup_pg_wal +{ + my ($node, $wal_threads_root, $thread_id) = @_; + my $pgwal = $node->data_dir . '/pg_wal'; + my $wal_thread = "$wal_threads_root/thread_$thread_id"; + + mkdir $wal_thread or die "mkdir $wal_thread: $!"; + opendir(my $dh, $pgwal) or die "opendir $pgwal: $!"; + for my $e (readdir $dh) + { + next if $e eq '.' || $e eq '..'; + rename("$pgwal/$e", "$wal_thread/$e") + or die "rename $pgwal/$e -> $wal_thread/$e: $!"; + } + closedir $dh; + rmdir $pgwal or die "rmdir $pgwal: $!"; + symlink($wal_thread, $pgwal) or die "symlink $pgwal -> $wal_thread: $!"; + return; +} + #----------------------------------------------------------------------- # new_quad($class, $cluster_name, %opts) # @@ -66,6 +89,8 @@ our $NODES = 4; # voting-disk files (QVOTEC reaches quorum OK) # wal_threads_root : 1 -> shared per-thread WAL root # shared_data : 1 -> shared data root (cluster_fs backend) +# shared_catalog : 1 -> t/337-style shared-catalog formation; +# implies shared_data + wal_threads_root #----------------------------------------------------------------------- sub new_quad { @@ -91,6 +116,12 @@ sub new_quad "${cluster_name}_node$i", port => $pg_ports[$i]); } + if ($opts{shared_catalog}) + { + $opts{shared_data} = 1; + $opts{wal_threads_root} = 1; + } + # spec-4.6: strict-mode opt-in (mirror ClusterTriple) -- pre-allocate N # shared voting-disk files so QVOTEC reaches quorum_state=OK and the GES # inbound validation (in_quorum, check 4) accepts cross-node traffic. @@ -127,23 +158,75 @@ sub new_quad if ($opts{shared_data}) { $shared_data_root = PostgreSQL::Test::Utils::tempdir(); + if ($opts{shared_catalog}) + { + mkdir "$shared_data_root/global" + or die "mkdir $shared_data_root/global: $!"; + } + } + + if ($opts{shared_catalog}) + { + my $sc_common = <init(allows_streaming => 1, + extra => [ '-X', "$wal_threads_root/thread_1" ]); + $nodes[0]->start; + $nodes[0]->backup('clusterquad_scb'); + $nodes[0]->stop; + + for my $i (1 .. $NODES - 1) + { + $nodes[$i]->init_from_backup($nodes[0], 'clusterquad_scb'); + _relocate_backup_pg_wal($nodes[$i], $wal_threads_root, $i + 1); + } + + # Seed the shared catalog/controlfile/OID authorities in a single-node era, + # then append the real cluster config below. Last GUC value wins. + $nodes[0]->append_conf('postgresql.conf', $sc_common); + $nodes[0]->append_conf('postgresql.conf', <start; + die "shared_catalog seed did not create catalog authority" + unless -e "$shared_data_root/global/pgrac_catalog_authority"; + $nodes[0]->stop; + } + else + { + my $wal_node_index = 0; + for my $node (@nodes) + { + if (defined $wal_threads_root) + { + my $thread_id = $wal_node_index + 1; + $node->init(extra => [ '-X', "$wal_threads_root/thread_$thread_id" ]); + } + else + { + $node->init; + } + $wal_node_index++; + } } - my $wal_node_index = 0; for my $node (@nodes) { if (defined $wal_threads_root) { - my $thread_id = $wal_node_index + 1; - $node->init(extra => [ '-X', "$wal_threads_root/thread_$thread_id" ]); $node->append_conf('postgresql.conf', "cluster.wal_threads_dir = '$wal_threads_root'\n"); } - else - { - $node->init; - } - $wal_node_index++; if (defined $shared_data_root) { @@ -153,6 +236,15 @@ sub new_quad "cluster.shared_data_dir = '$shared_data_root'\n"); $node->append_conf('postgresql.conf', "cluster.smgr_user_relations = on\n"); + if ($opts{shared_catalog}) + { + $node->append_conf('postgresql.conf', + "cluster.controlfile_shared_authority = on\n"); + $node->append_conf('postgresql.conf', + "cluster.shared_catalog = on\n"); + $node->append_conf('postgresql.conf', + "cluster.merged_recovery = on\n"); + } } # Enable cluster + tier1, same baseline as ClusterTriple (spec-2.2).