From fff341a433aedfc63e435bd364813ac676c7ba6f Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Fri, 22 May 2026 17:30:27 -0500 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20distributed=20state=20gap=20closure?= =?UTF-8?q?=20=E2=80=94=2014=20items?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the post-gap-analysis roadmap items (all except Admin CLI, deferred to separate work): Proto & wire format: - Add ChunkRequest/Response, SnapshotRequest/Entry/Response, Heartbeat messages to state_transport.proto - Add hlc_time field to Mutation (proto field 12) - Extend Frame oneof with 5 new fields - Add snapshot & chunk wire types to IPC transport (kMsgSnapReq/Rsp) Transport chunk fetch: - IPC transport: set_blob_store, fetch_chunk, chunk request/response - gRPC transport: set_blob_store, fetch_chunk, chunk request/response - Both use condition_variable waiter pattern with 10s timeout Session improvements: - Wire state_data_hydrator into distributed_state_session::join() - Add TLS/auth config passthrough (gRPC only) - Add blob store wiring on both IPC and gRPC transports - Add wait_for_data() delegating to hydrator - Add replica_status status() with peer_count, local_sequence, etc. Compression: - Add state_zstd_compression_codec (production zstd via libzstd) - Register in shared() singleton; update tests for 3 built-in codecs - Add CMake zstd discovery (pkg-config + fallback find_library) Conflict visibility: - Add conflict_entry ring buffer to state_cluster_shard (128 entries) - Add recent_conflicts() accessor (most-recent-first) - Add publish_conflicts() to state_distributed_metrics Hybrid logical clocks: - New state_hybrid_time.h: hybrid_time struct + hybrid_clock class - Add hlc_time field to state_mutation (packed 48-bit wall + 16-bit logical) - Update should_replace() to prefer HLC ordering when both carry timestamps - Serialize hlc_time on IPC wire (backward-compatible) and gRPC proto Hash-range sharding: - New state_hash_partition.h: FNV-1a hash ring with uniform partitioning Selective brick fetch: - Add plane struct and bricks_in_frustum() to state_brick_manifest (AABB-vs-frustum conservative test) Initial-sync snapshot: - Add snapshot() to state_cluster_shard (walks state tree via traverse) - Add request_snapshot() virtual to state_transport base class - Implement on IPC transport (kMsgSnapReq/Rsp wire protocol) Tests: - state_reconnect_resilience_test.cpp: 3 tests (stop/restart/reconnect, peer disconnect survival, connection count tracking) - state_large_tree_bench_test.cpp: 4 benchmarks (10k/100k ingest, drain+publish, snapshot) gated on CVC_DISTRIBUTED_STATE_BENCH=1 --- inc/cvc/distributed_state_session.h | 35 +++ inc/cvc/state_brick_manifest.h | 10 + inc/cvc/state_change_journal.h | 1 + inc/cvc/state_cluster_shard.h | 32 +++ inc/cvc/state_compression_registry.h | 13 + inc/cvc/state_distributed_metrics.h | 5 + inc/cvc/state_hash_partition.h | 123 +++++++++ inc/cvc/state_hybrid_time.h | 124 ++++++++++ inc/cvc/state_transport.h | 32 +++ inc/cvc/state_transport_grpc.h | 28 +++ inc/cvc/state_transport_ipc.h | 33 +++ proto/state_transport.proto | 54 ++++ src/cvc/CMakeLists.txt | 21 ++ src/cvc/distributed_state_session.cpp | 53 +++- src/cvc/state_brick_manifest.cpp | 29 +++ src/cvc/state_cluster_shard.cpp | 47 ++++ src/cvc/state_compression_registry.cpp | 59 +++++ src/cvc/state_distributed_metrics.cpp | 24 ++ src/cvc/state_replica.cpp | 6 + src/cvc/state_transport_grpc.cpp | 115 +++++++++ src/cvc/state_transport_ipc.cpp | 234 ++++++++++++++++++ src/cvc/tests/CMakeLists.txt | 19 ++ .../tests/state_compression_registry_test.cpp | 9 +- src/cvc/tests/state_large_tree_bench_test.cpp | 147 +++++++++++ .../tests/state_reconnect_resilience_test.cpp | 176 +++++++++++++ 25 files changed, 1423 insertions(+), 6 deletions(-) create mode 100644 inc/cvc/state_hash_partition.h create mode 100644 inc/cvc/state_hybrid_time.h create mode 100644 src/cvc/tests/state_large_tree_bench_test.cpp create mode 100644 src/cvc/tests/state_reconnect_resilience_test.cpp diff --git a/inc/cvc/distributed_state_session.h b/inc/cvc/distributed_state_session.h index 48e447c..52c608f 100644 --- a/inc/cvc/distributed_state_session.h +++ b/inc/cvc/distributed_state_session.h @@ -12,11 +12,13 @@ #define __CVC_DISTRIBUTED_STATE_SESSION_H__ #include +#include #include #include #include #include #include +#include #include #include #include @@ -84,9 +86,31 @@ struct distributed_state_config { bool resolve_conflicts = false; bool enforce_interest = false; + // TLS / auth (gRPC only — ignored for inproc / ipc). + std::string tls_server_cert_pem; + std::string tls_server_key_pem; + std::string tls_root_ca_pem; + bool tls_require_client_auth = false; + std::string auth_expected_token; + std::string auth_outbound_token; + + // Tuning. + std::uint32_t max_inline_payload_bytes = 65536; + std::uint32_t pump_interval_ms = 10; // background pump loop interval (0 = no pump thread) }; +// ---------------- +// Snapshot of current replica health. +// ---------------- +struct replica_status { + bool running = false; + std::size_t peer_count = 0; + std::uint64_t local_sequence = 0; + std::uint64_t pump_cycles = 0; + std::uint64_t pending_hydrations = 0; +}; + // ---------------- // cvc::distributed_state_session // ---------------- @@ -138,6 +162,16 @@ class distributed_state_session { state_transport &transport() noexcept { return *_transport; } state_blob_store &blob_store() noexcept { return *_blob_store; } state_distributed_admin &admin() noexcept { return *_admin; } + state_data_hydrator &hydrator() noexcept { return *_hydrator; } + + // Wait until the blob at `path` has been hydrated, or timeout + // expires. Returns the hydration status. + state_data_hydrator::hydration_status + wait_for_data(const std::string &path, + std::chrono::milliseconds timeout = std::chrono::milliseconds(0)); + + // Get a snapshot of the replica health. + replica_status status() const; // Diagnostics. const std::string &cluster_id() const noexcept { return _config.cluster_id; } @@ -157,6 +191,7 @@ class distributed_state_session { std::unique_ptr _transport; std::unique_ptr _blob_store; std::unique_ptr _admin; + std::unique_ptr _hydrator; std::thread _pump_thread; std::atomic _running{false}; diff --git a/inc/cvc/state_brick_manifest.h b/inc/cvc/state_brick_manifest.h index 0789230..3457bee 100644 --- a/inc/cvc/state_brick_manifest.h +++ b/inc/cvc/state_brick_manifest.h @@ -107,6 +107,16 @@ struct state_brick_manifest { std::vector bricks_in_region(std::uint64_t lo_x, std::uint64_t lo_y, std::uint64_t lo_z, std::uint64_t hi_x, std::uint64_t hi_y, std::uint64_t hi_z) const; + + // A half-space plane Ax+By+Cz+D >= 0. + struct plane { + double a, b, c, d; + }; + + // Query: return indices of chunks whose AABB is NOT completely + // outside all 6 frustum planes (conservative — false positives + // are possible). Each plane's positive half-space is "inside". + std::vector bricks_in_frustum(const plane planes[6]) const; }; // ---------------- diff --git a/inc/cvc/state_change_journal.h b/inc/cvc/state_change_journal.h index e82b036..2d4e2cb 100644 --- a/inc/cvc/state_change_journal.h +++ b/inc/cvc/state_change_journal.h @@ -47,6 +47,7 @@ struct state_mutation { std::string tree_id; std::string origin_node_id; std::uint64_t sequence = 0; + std::uint64_t hlc_time = 0; // packed hybrid logical clock (0 = not set) std::string mutation_id; std::string path; state_mutation_op op = state_mutation_op::set_value; diff --git a/inc/cvc/state_cluster_shard.h b/inc/cvc/state_cluster_shard.h index 62c2a23..8c53e88 100644 --- a/inc/cvc/state_cluster_shard.h +++ b/inc/cvc/state_cluster_shard.h @@ -164,6 +164,19 @@ class state_cluster_shard { std::uint64_t total_conflicts_detected() const noexcept { return _ctr_conflicts_detected.load(); } std::uint64_t total_conflicts_lost() const noexcept { return _ctr_conflicts_lost.load(); } + // Per-path conflict detail record. + struct conflict_entry { + std::string path; + std::string winner_node_id; + std::uint64_t winner_sequence = 0; + std::string loser_node_id; + std::uint64_t loser_sequence = 0; + }; + + // Return the most recent conflicts (up to `max_entries`, default 64). + // The ring buffer is only populated when resolve_conflicts is true. + std::vector recent_conflicts(std::size_t max_entries = 64) const; + // Phase 6: subtree delegation. When true, ingest_remote consults // the delegation manager. A mutation whose path resolves to a // foreign cluster is rejected with reason "path delegated to @@ -247,6 +260,20 @@ class state_cluster_shard { return _ctr_remote_filtered_out.load(); } + // Snapshot: walk the local state tree under `path_prefix` and + // return a vector of entries suitable for initial-sync. + struct snapshot_entry { + std::string path; + std::string string_value; + std::string comment; + bool hidden = false; + bool read_only = false; + std::string type_name; + std::string origin_node_id; + std::uint64_t sequence = 0; + }; + std::vector snapshot(const std::string &path_prefix = std::string()) const; + // -------- Phase 8 slice 2: cluster-agnostic message routing -------- // // The shard owns the bridge from "I want to send a message at @@ -323,6 +350,11 @@ class state_cluster_shard { std::atomic _ctr_remote_rejected{0}; std::atomic _ctr_conflicts_detected{0}; std::atomic _ctr_conflicts_lost{0}; + + // Ring buffer of recent conflict entries. + static constexpr std::size_t kMaxConflictRing = 128; + std::vector _conflict_ring; + std::size_t _conflict_ring_pos = 0; std::atomic _ctr_delegation_routed{0}; std::atomic _ctr_delegation_expired{0}; std::atomic _ctr_delegations_applied{0}; diff --git a/inc/cvc/state_compression_registry.h b/inc/cvc/state_compression_registry.h index 7b5edd9..14c1f98 100644 --- a/inc/cvc/state_compression_registry.h +++ b/inc/cvc/state_compression_registry.h @@ -121,6 +121,19 @@ class state_rle_compression_codec : public state_compression_codec { bool decode(const std::vector &in, std::vector &out) const override; }; +// zstd compression codec (production-grade). Requires libzstd. +class state_zstd_compression_codec : public state_compression_codec { +public: + // compression_level: 1 (fastest) to 22 (best ratio); default 3. + explicit state_zstd_compression_codec(int compression_level = 3); + std::string id() const override { return "zstd"; } + std::vector encode(const std::vector &in) const override; + bool decode(const std::vector &in, std::vector &out) const override; + +private: + int _level; +}; + } // namespace CVC_NAMESPACE #endif // __CVC_STATE_COMPRESSION_REGISTRY_H__ diff --git a/inc/cvc/state_distributed_metrics.h b/inc/cvc/state_distributed_metrics.h index 620f2ed..bfe14b6 100644 --- a/inc/cvc/state_distributed_metrics.h +++ b/inc/cvc/state_distributed_metrics.h @@ -65,6 +65,11 @@ struct state_distributed_metrics { // __system.distributed.. static void write_u64(app &ctx, const std::string &cluster_id, const std::string &key, std::uint64_t value); + + // Publish the conflict ring buffer entries under + // __system.distributed..conflicts.recent..* + // Returns the number of entries written. + static std::size_t publish_conflicts(app &ctx, const state_cluster_shard &shard); }; } // namespace CVC_NAMESPACE diff --git a/inc/cvc/state_hash_partition.h b/inc/cvc/state_hash_partition.h new file mode 100644 index 0000000..500a7b4 --- /dev/null +++ b/inc/cvc/state_hash_partition.h @@ -0,0 +1,123 @@ +/* + Copyright 2026 The University of Texas at Austin + + This file is part of libcvc. + + libcvc is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. +*/ + +#ifndef __CVC_STATE_HASH_PARTITION_H__ +#define __CVC_STATE_HASH_PARTITION_H__ + +#include +#include +#include +#include +#include +#include +#include + +namespace CVC_NAMESPACE { + +// ---------------- +// cvc::state_hash_partition +// ---------------- +// Consistent-hash partition map for routing state paths to nodes +// within a cluster. Each node owns a contiguous range on the hash +// ring [range_begin, range_end). A path's owner is found by +// hashing the path and locating the first range whose end exceeds +// the hash value (sorted ring lookup). +// +// Thread-safe: all methods are mutex-guarded. +// +class state_hash_partition { +public: + // A contiguous slice of the hash ring. + struct range { + std::string node_id; + std::uint32_t range_begin = 0; + std::uint32_t range_end = 0; // exclusive + }; + + state_hash_partition() = default; + + // Assign ownership of [begin, end) to `node_id`. Overlapping + // ranges are not checked — the caller must ensure consistency. + void assign(const std::string &node_id, std::uint32_t begin, std::uint32_t end) { + std::lock_guard lk(_mu); + _ranges.push_back({node_id, begin, end}); + std::sort(_ranges.begin(), _ranges.end(), + [](const range &a, const range &b) { return a.range_begin < b.range_begin; }); + } + + // Uniformly partition the full 32-bit hash space among the given + // node IDs. Replaces any existing assignment. + void assign_uniform(const std::vector &node_ids) { + std::lock_guard lk(_mu); + _ranges.clear(); + if (node_ids.empty()) + return; + std::uint64_t total = static_cast(UINT32_MAX) + 1; + std::uint64_t slice = total / node_ids.size(); + std::uint32_t lo = 0; + for (std::size_t i = 0; i < node_ids.size(); ++i) { + std::uint32_t hi = (i + 1 == node_ids.size()) ? UINT32_MAX + : static_cast(lo + slice - 1); + _ranges.push_back({node_ids[i], lo, hi + 1}); + lo = hi + 1; + } + } + + // Resolve the owner node for a given path. Returns empty string + // if no range covers the hash. + std::string owner_of(const std::string &path) const { + std::uint32_t h = hash(path); + std::lock_guard lk(_mu); + for (const auto &r : _ranges) { + if (h >= r.range_begin && h < r.range_end) + return r.node_id; + } + return {}; + } + + // Check if `node_id` owns the given path. + bool owns(const std::string &node_id, const std::string &path) const { + return owner_of(path) == node_id; + } + + // Current snapshot of all ranges. + std::vector snapshot() const { + std::lock_guard lk(_mu); + return _ranges; + } + + std::size_t size() const { + std::lock_guard lk(_mu); + return _ranges.size(); + } + + void clear() { + std::lock_guard lk(_mu); + _ranges.clear(); + } + + // FNV-1a hash (32-bit) of a path string. + static std::uint32_t hash(const std::string &path) noexcept { + std::uint32_t h = 2166136261u; + for (unsigned char c : path) { + h ^= c; + h *= 16777619u; + } + return h; + } + +private: + mutable std::mutex _mu; + std::vector _ranges; +}; + +} // namespace CVC_NAMESPACE + +#endif // __CVC_STATE_HASH_PARTITION_H__ diff --git a/inc/cvc/state_hybrid_time.h b/inc/cvc/state_hybrid_time.h new file mode 100644 index 0000000..784f9e6 --- /dev/null +++ b/inc/cvc/state_hybrid_time.h @@ -0,0 +1,124 @@ +/* + Copyright 2026 The University of Texas at Austin + + This file is part of libcvc. + + libcvc is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. +*/ + +#ifndef __CVC_STATE_HYBRID_TIME_H__ +#define __CVC_STATE_HYBRID_TIME_H__ + +#include +#include +#include +#include +#include +#include + +namespace CVC_NAMESPACE { + +// ---------------- +// cvc::hybrid_time +// ---------------- +// Hybrid Logical Clock (HLC) timestamp combining physical wall +// time with a logical counter to provide a monotonically +// increasing, causally consistent ordering across nodes. +// +// Encoding: 48 bits of wall_ms (milliseconds since epoch) in the +// upper bits, 16 bits of logical counter in the lower bits. +// Packed into a single uint64_t for wire efficiency. +// +// Causal ordering: +// send/local event: now() advances the clock. +// receive: update(remote_hlc) merges the remote timestamp, +// ensuring this node's clock never goes backwards. +// +struct hybrid_time { + std::uint64_t wall_ms = 0; // milliseconds since Unix epoch + std::uint16_t logical = 0; // logical counter + + // Pack into a 64-bit value: upper 48 bits wall, lower 16 bits logical. + std::uint64_t packed() const noexcept { + return (wall_ms << 16) | static_cast(logical); + } + + // Unpack from a 64-bit value. + static hybrid_time from_packed(std::uint64_t v) noexcept { + return {v >> 16, static_cast(v & 0xFFFF)}; + } + + bool operator<(const hybrid_time &o) const noexcept { return packed() < o.packed(); } + bool operator>(const hybrid_time &o) const noexcept { return packed() > o.packed(); } + bool operator==(const hybrid_time &o) const noexcept { return packed() == o.packed(); } + bool operator!=(const hybrid_time &o) const noexcept { return packed() != o.packed(); } + bool operator<=(const hybrid_time &o) const noexcept { return packed() <= o.packed(); } + bool operator>=(const hybrid_time &o) const noexcept { return packed() >= o.packed(); } + + // Returns true if this time is non-zero (has been assigned). + explicit operator bool() const noexcept { return packed() != 0; } +}; + +// ---------------- +// cvc::hybrid_clock +// ---------------- +// Thread-safe HLC clock. Each node in the cluster keeps one +// instance. Call now() before sending a mutation, and update() +// when receiving a remote mutation's timestamp. +// +class hybrid_clock { +public: + // Issue a new timestamp for a local event. + hybrid_time now() noexcept { + std::lock_guard lk(_mu); + auto phys = wall_ms(); + if (phys > _last.wall_ms) { + _last.wall_ms = phys; + _last.logical = 0; + } else { + ++_last.logical; + } + return _last; + } + + // Merge a remote timestamp (receive event). + hybrid_time update(hybrid_time remote) noexcept { + std::lock_guard lk(_mu); + auto phys = wall_ms(); + if (phys > _last.wall_ms && phys > remote.wall_ms) { + _last.wall_ms = phys; + _last.logical = 0; + } else if (_last.wall_ms == remote.wall_ms) { + _last.logical = std::max(_last.logical, remote.logical) + 1; + } else if (_last.wall_ms > remote.wall_ms) { + ++_last.logical; + } else { + _last.wall_ms = remote.wall_ms; + _last.logical = remote.logical + 1; + } + return _last; + } + + // Current clock value without advancing. + hybrid_time current() const noexcept { + std::lock_guard lk(_mu); + return _last; + } + +private: + static std::uint64_t wall_ms() noexcept { + return static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); + } + + mutable std::mutex _mu; + hybrid_time _last{}; +}; + +} // namespace CVC_NAMESPACE + +#endif // __CVC_STATE_HYBRID_TIME_H__ diff --git a/inc/cvc/state_transport.h b/inc/cvc/state_transport.h index 9b26e2e..2d01b7f 100644 --- a/inc/cvc/state_transport.h +++ b/inc/cvc/state_transport.h @@ -143,6 +143,38 @@ class state_transport { return n; } + // -------- Initial-sync snapshot protocol -------- + // + // A snapshot entry represents one node's state. + struct snapshot_entry { + std::string path; + std::string string_value; + std::string comment; + bool hidden = false; + bool read_only = false; + std::string type_name; + std::string origin_node_id; + std::uint64_t sequence = 0; + }; + + // Callback invoked with batches of snapshot entries. `final` is + // true on the last invocation. + using snapshot_callback = + std::function &entries, bool final)>; + + // Request a snapshot of the remote peer's state tree rooted at + // `path_prefix` (empty = whole tree). The entries are delivered + // asynchronously via `on_entries`. + // + // Default returns false (not supported). + virtual bool request_snapshot(const std::string &cluster_id, const std::string &path_prefix, + snapshot_callback on_entries) { + (void)cluster_id; + (void)path_prefix; + (void)on_entries; + return false; + } + protected: state_peer_registry _peers; }; diff --git a/inc/cvc/state_transport_grpc.h b/inc/cvc/state_transport_grpc.h index a5b1d55..2d74e85 100644 --- a/inc/cvc/state_transport_grpc.h +++ b/inc/cvc/state_transport_grpc.h @@ -13,12 +13,15 @@ #include #include +#include #include +#include #include #include #include #include #include +#include #include namespace CVC_NAMESPACE { @@ -114,6 +117,11 @@ class state_transport_grpc final : public state_transport { // Shut down server, cancel all client streams, join reader threads. void stop(); + // Blob store for servicing inbound chunk requests and for local + // lookup before sending outbound requests. + void set_blob_store(state_blob_store *store) noexcept { _blob_store = store; } + state_blob_store *blob_store() const noexcept { return _blob_store; } + // state_transport interface. void register_shard(state_cluster_shard *shard) override; void unregister_shard(state_cluster_shard *shard) override; @@ -122,6 +130,7 @@ class state_transport_grpc final : public state_transport { std::size_t pump_shard(state_cluster_shard &shard) override; std::size_t pump_all() override; void flush() override; + bool fetch_chunk(const std::string &digest, chunk_callback on_chunk) override; // Diagnostics. std::size_t shard_count() const; @@ -151,6 +160,12 @@ class state_transport_grpc final : public state_transport { const auth_config &auth() const noexcept { return _auth; } void on_inbound_mutation(const state_mutation &m); void on_inbound_message(const state_message &m); + // Chunk fetch request/response dispatch (called from connection + // reader threads when a ChunkRequest/ChunkResponse frame arrives). + void on_inbound_chunk_request(connection *conn, const std::string &digest, + std::uint64_t request_id); + void on_inbound_chunk_response(std::uint64_t request_id, bool found, + std::vector data); void register_connection(std::shared_ptr conn); void unregister_connection(connection *conn); void increment_recv_frames() noexcept { _recv_frames.fetch_add(1, std::memory_order_relaxed); } @@ -185,6 +200,19 @@ class state_transport_grpc final : public state_transport { std::atomic _recv_mutations{0}; std::atomic _recv_messages{0}; std::atomic _delivered{0}; + + state_blob_store *_blob_store = nullptr; + + // Chunk fetch waiter infrastructure. + std::atomic _next_chunk_req_id{1}; + mutable std::mutex _chunk_waiters_mu; + std::condition_variable _chunk_waiters_cv; + struct chunk_waiter { + bool done = false; + bool found = false; + std::vector data; + }; + std::unordered_map> _chunk_waiters; }; } // namespace CVC_NAMESPACE diff --git a/inc/cvc/state_transport_ipc.h b/inc/cvc/state_transport_ipc.h index 0b810fb..002dc37 100644 --- a/inc/cvc/state_transport_ipc.h +++ b/inc/cvc/state_transport_ipc.h @@ -13,10 +13,13 @@ #include #include +#include #include +#include #include #include #include +#include #include #include #include @@ -92,6 +95,10 @@ class state_transport_ipc final : public state_transport { // unlink the listener path. void stop(); + // Blob store for chunk fetch servicing. + void set_blob_store(state_blob_store *store) noexcept { _blob_store = store; } + state_blob_store *blob_store() const noexcept { return _blob_store; } + // state_transport interface. void register_shard(state_cluster_shard *shard) override; void unregister_shard(state_cluster_shard *shard) override; @@ -100,6 +107,9 @@ class state_transport_ipc final : public state_transport { std::size_t pump_shard(state_cluster_shard &shard) override; std::size_t pump_all() override; void flush() override; + bool fetch_chunk(const std::string &digest, chunk_callback on_chunk) override; + bool request_snapshot(const std::string &cluster_id, const std::string &path_prefix, + snapshot_callback on_entries) override; // Diagnostics. std::size_t shard_count() const; @@ -149,12 +159,35 @@ class state_transport_ipc final : public state_transport { mutable std::mutex _conns_mu; std::vector> _conns; + state_blob_store *_blob_store = nullptr; + std::atomic _published{0}; std::atomic _sent_frames{0}; std::atomic _recv_frames{0}; std::atomic _recv_mutations{0}; std::atomic _recv_messages{0}; std::atomic _delivered{0}; + + // Chunk fetch request/response tracking. + std::atomic _next_chunk_req_id{1}; + mutable std::mutex _chunk_waiters_mu; + std::condition_variable _chunk_waiters_cv; + struct chunk_waiter { + bool done = false; + bool found = false; + std::vector data; + }; + std::unordered_map> _chunk_waiters; + + // Snapshot request/response tracking. + std::atomic _next_snap_req_id{1}; + mutable std::mutex _snap_waiters_mu; + std::condition_variable _snap_waiters_cv; + struct snap_waiter { + bool done = false; + std::vector entries; + }; + std::unordered_map> _snap_waiters; }; } // namespace CVC_NAMESPACE diff --git a/proto/state_transport.proto b/proto/state_transport.proto index 1a72f0f..b5d1baf 100644 --- a/proto/state_transport.proto +++ b/proto/state_transport.proto @@ -50,6 +50,7 @@ message Mutation { string string_value = 9; Payload payload = 10; bool latest_value_only = 11; + uint64 hlc_time = 12; // packed hybrid logical clock (0 = not set) } message Hello { @@ -71,6 +72,54 @@ message Message { bytes bytes_payload = 8; } +// Chunk fetch request: ask a peer for a blob chunk by digest. +message ChunkRequest { + string digest = 1; // SHA-256 hex digest + uint64 request_id = 2; // caller-assigned; echoed in response +} + +// Chunk fetch response: carries the blob chunk bytes. +message ChunkResponse { + string digest = 1; // echoed from request + uint64 request_id = 2; // echoed from request + bool found = 3; // false if the peer does not have the chunk + bytes data = 4; // raw chunk bytes (empty if !found) +} + +// Snapshot request: ask a peer for the current state tree. +message SnapshotRequest { + string cluster_id = 1; + string path_prefix = 2; // "" = entire tree + uint64 request_id = 3; +} + +// Snapshot chunk: one node's state within a snapshot. +message SnapshotEntry { + string path = 1; + string string_value = 2; + string comment = 3; + bool hidden = 4; + bool read_only = 5; + string type_name = 6; + string origin_node_id = 7; + uint64 sequence = 8; +} + +// Snapshot response: sent in response to SnapshotRequest. +message SnapshotResponse { + uint64 request_id = 1; + bool final = 2; // true on the last chunk + repeated SnapshotEntry entries = 3; +} + +// Heartbeat for cluster membership. Sent periodically; receiving +// peers call note_seen() on their peer registry. +message Heartbeat { + string node_id = 1; + string cluster_id = 2; + uint64 timestamp_ns = 3; // sender's wall-clock ns (observability) +} + // Single bidirectional frame envelope. Both client and server write // frames; same-cluster routing happens above this layer. message Frame { @@ -78,6 +127,11 @@ message Frame { Hello hello = 1; Mutation mutation = 2; Message message = 3; + ChunkRequest chunk_request = 4; + ChunkResponse chunk_response = 5; + SnapshotRequest snapshot_request = 6; + SnapshotResponse snapshot_response = 7; + Heartbeat heartbeat = 8; } } diff --git a/src/cvc/CMakeLists.txt b/src/cvc/CMakeLists.txt index 31aab7c..4d8b3f6 100644 --- a/src/cvc/CMakeLists.txt +++ b/src/cvc/CMakeLists.txt @@ -44,6 +44,8 @@ set(INCLUDE_FILES ../../inc/cvc/state_volume_codec.h ../../inc/cvc/state_brick_manifest.h ../../inc/cvc/state_data_hydrator.h + ../../inc/cvc/state_hybrid_time.h + ../../inc/cvc/state_hash_partition.h ../../inc/cvc/distributed_state_session.h ../../inc/cvc/state_transport.h ../../inc/cvc/state_transport_inproc.h @@ -831,6 +833,25 @@ if(CVC_ENABLE_FFTW) target_link_libraries(cvc PUBLIC ${FFTW3_LIBRARIES}) endif() +# zstd compression (production compressor for distributed state). +find_package(PkgConfig QUIET) +if(PkgConfig_FOUND) + pkg_check_modules(ZSTD QUIET libzstd) +endif() +if(NOT ZSTD_FOUND) + find_library(ZSTD_LIBRARIES NAMES zstd) + find_path(ZSTD_INCLUDE_DIRS NAMES zstd.h) + if(ZSTD_LIBRARIES AND ZSTD_INCLUDE_DIRS) + set(ZSTD_FOUND TRUE) + endif() +endif() +if(ZSTD_FOUND) + target_include_directories(cvc PRIVATE ${ZSTD_INCLUDE_DIRS}) + target_link_libraries(cvc PRIVATE ${ZSTD_LIBRARIES}) +else() + message(WARNING "libzstd not found; zstd compression codec will not link") +endif() + # Link libraries target_link_libraries(cvc PUBLIC ${CVC_LINK_LIBS}) diff --git a/src/cvc/distributed_state_session.cpp b/src/cvc/distributed_state_session.cpp index 1c5620c..546033f 100644 --- a/src/cvc/distributed_state_session.cpp +++ b/src/cvc/distributed_state_session.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #ifndef _WIN32 #include @@ -42,6 +43,7 @@ distributed_state_session::join(app &ctx, const distributed_state_config &config case transport_kind::ipc: { #ifndef _WIN32 auto ipc = std::make_unique(); + ipc->set_blob_store(session->_blob_store.get()); if (!config.listen_address.empty()) ipc->start(config.listen_address, config.node_id, config.cluster_id); session->_transport = std::move(ipc); @@ -54,6 +56,22 @@ distributed_state_session::join(app &ctx, const distributed_state_config &config case transport_kind::grpc: { #ifdef CVC_ENABLE_GRPC auto grpc = std::make_unique(); + // Apply TLS / auth config if provided. + if (!config.tls_server_cert_pem.empty() || !config.tls_root_ca_pem.empty()) { + state_transport_grpc::tls_config tls; + tls.server_cert_pem = config.tls_server_cert_pem; + tls.server_key_pem = config.tls_server_key_pem; + tls.root_ca_pem = config.tls_root_ca_pem; + tls.require_client_auth = config.tls_require_client_auth; + grpc->set_tls_config(std::move(tls)); + } + if (!config.auth_expected_token.empty() || !config.auth_outbound_token.empty()) { + state_transport_grpc::auth_config auth; + auth.expected_token = config.auth_expected_token; + auth.outbound_token = config.auth_outbound_token; + grpc->set_auth_config(std::move(auth)); + } + grpc->set_blob_store(session->_blob_store.get()); if (!config.listen_address.empty()) grpc->start(config.listen_address, config.node_id, config.cluster_id); session->_transport = std::move(grpc); @@ -114,11 +132,17 @@ distributed_state_session::join(app &ctx, const distributed_state_config &config session->_admin->attach_shard(session->_shard.get()); session->_admin->attach_blob_store(session->_blob_store.get()); - // 9. Attach shard (start observing state changes). + // 9. Create hydrator (blob fetch + codec decode). + session->_hydrator = std::make_unique( + *session->_blob_store, session->_shard->codecs(), + &state_compression_registry::shared()); + session->_hydrator->set_transport(session->_transport.get()); + + // 10. Attach shard (start observing state changes). session->_shard->attach(); session->_shard->install_as_default(); - // 10. Start pump thread. + // 11. Start pump thread. session->_running.store(true, std::memory_order_release); if (config.pump_interval_ms > 0) { session->_pump_thread = std::thread([session]() { session->pump_loop(); }); @@ -189,4 +213,29 @@ void distributed_state_session::pump_loop() { _transport->pump_all(); } +state_data_hydrator::hydration_status +distributed_state_session::wait_for_data(const std::string &path, + std::chrono::milliseconds timeout) { + return _hydrator->wait(path, timeout); +} + +replica_status distributed_state_session::status() const { + replica_status s; + s.running = _running.load(std::memory_order_acquire); + s.local_sequence = _shard ? _shard->published_cursor() : 0; + s.pump_cycles = _pump_cycles.load(std::memory_order_relaxed); + s.pending_hydrations = _hydrator ? _hydrator->pending_count() : 0; + + // connection_count is on the concrete transports, not the base. +#ifndef _WIN32 + if (auto *ipc = dynamic_cast(_transport.get())) + s.peer_count = ipc->connection_count(); +#endif +#ifdef CVC_ENABLE_GRPC + if (auto *grpc = dynamic_cast(_transport.get())) + s.peer_count = grpc->connection_count(); +#endif + return s; +} + } // namespace CVC_NAMESPACE diff --git a/src/cvc/state_brick_manifest.cpp b/src/cvc/state_brick_manifest.cpp index 3f112ca..944b943 100644 --- a/src/cvc/state_brick_manifest.cpp +++ b/src/cvc/state_brick_manifest.cpp @@ -205,6 +205,35 @@ state_brick_manifest::bricks_in_region(std::uint64_t lo_x, std::uint64_t lo_y, s return result; } +std::vector +state_brick_manifest::bricks_in_frustum(const plane planes[6]) const { + std::vector result; + for (std::size_t i = 0; i < extents.size(); ++i) { + const brick_extent &ext = extents[i]; + double lo_x = static_cast(ext.origin_x); + double lo_y = static_cast(ext.origin_y); + double lo_z = static_cast(ext.origin_z); + double hi_x = lo_x + static_cast(ext.size_x); + double hi_y = lo_y + static_cast(ext.size_y); + double hi_z = lo_z + static_cast(ext.size_z); + + bool outside = false; + for (int p = 0; p < 6 && !outside; ++p) { + // Find the AABB corner most aligned with the plane normal + // (the "positive vertex"). If that corner is outside the + // plane, the entire AABB is outside this half-space. + double px = (planes[p].a >= 0) ? hi_x : lo_x; + double py = (planes[p].b >= 0) ? hi_y : lo_y; + double pz = (planes[p].c >= 0) ? hi_z : lo_z; + if (planes[p].a * px + planes[p].b * py + planes[p].c * pz + planes[p].d < 0) + outside = true; + } + if (!outside) + result.push_back(i); + } + return result; +} + // ---------- state_brick_writer ---------- state_brick_writer::state_brick_writer(state_blob_store &store, std::uint32_t brick_dim, diff --git a/src/cvc/state_cluster_shard.cpp b/src/cvc/state_cluster_shard.cpp index 2b909ee..e43fff0 100644 --- a/src/cvc/state_cluster_shard.cpp +++ b/src/cvc/state_cluster_shard.cpp @@ -140,6 +140,21 @@ bool state_cluster_shard::resolve_conflicts() const noexcept { return _resolve_conflicts; } +std::vector +state_cluster_shard::recent_conflicts(std::size_t max_entries) const { + std::lock_guard lk(_mutex); + std::size_t n = std::min(max_entries, _conflict_ring.size()); + std::vector out; + out.reserve(n); + // Walk backwards from the write cursor to get most-recent-first. + for (std::size_t i = 0; i < n; ++i) { + std::size_t idx = + (_conflict_ring_pos + _conflict_ring.size() - 1 - i) % _conflict_ring.size(); + out.push_back(_conflict_ring[idx]); + } + return out; +} + void state_cluster_shard::set_enforce_delegation(bool enforce) noexcept { std::lock_guard lk(_mutex); _enforce_delegation = enforce; @@ -267,6 +282,17 @@ state_cluster_shard::ingest_result state_cluster_shard::ingest_remote(const stat _ctr_conflicts_detected.fetch_add(1, std::memory_order_relaxed); if (!state_replica::should_replace(it->second, m)) { conflict_lost = true; + // Record conflict detail in ring buffer. + if (_conflict_ring.size() < kMaxConflictRing) + _conflict_ring.push_back({}); + auto &e = _conflict_ring[_conflict_ring_pos % kMaxConflictRing]; + e.path = m.path; + e.winner_node_id = it->second.origin_node_id; + e.winner_sequence = it->second.sequence; + e.loser_node_id = m.origin_node_id; + e.loser_sequence = m.sequence; + _conflict_ring_pos = + (_conflict_ring_pos + 1) % kMaxConflictRing; } } } @@ -548,4 +574,25 @@ state_cluster_shard::send_message_result state_cluster_shard::send_message(state return r; } +std::vector +state_cluster_shard::snapshot(const std::string &path_prefix) const { + std::vector result; + if (!_app_ctx) + return result; + + // Start at root (or prefix). + auto &root = state::instance(*_app_ctx)(path_prefix); + root.traverse([&](std::string child_path) { + auto &node = state::instance(*_app_ctx)(child_path); + snapshot_entry e; + e.path = child_path; + e.string_value = node.value(); + e.comment = node.comment(); + e.hidden = node.hidden(); + e.read_only = node.readOnly(); + result.push_back(std::move(e)); + }); + return result; +} + } // namespace CVC_NAMESPACE diff --git a/src/cvc/state_compression_registry.cpp b/src/cvc/state_compression_registry.cpp index d5fecef..a805b06 100644 --- a/src/cvc/state_compression_registry.cpp +++ b/src/cvc/state_compression_registry.cpp @@ -10,6 +10,8 @@ #include #include +#include +#include namespace CVC_NAMESPACE { @@ -51,11 +53,68 @@ bool state_rle_compression_codec::decode(const std::vector &in, return true; } +// ---------- state_zstd_compression_codec ---------- + +state_zstd_compression_codec::state_zstd_compression_codec(int compression_level) + : _level(compression_level) {} + +std::vector +state_zstd_compression_codec::encode(const std::vector &in) const { + if (in.empty()) + return {}; + std::size_t bound = ZSTD_compressBound(in.size()); + std::vector out(bound); + std::size_t result = + ZSTD_compress(out.data(), out.size(), in.data(), in.size(), _level); + if (ZSTD_isError(result)) + return in; // fall back to uncompressed on error + out.resize(result); + return out; +} + +bool state_zstd_compression_codec::decode(const std::vector &in, + std::vector &out) const { + out.clear(); + if (in.empty()) + return true; + unsigned long long decompressed_size = ZSTD_getFrameContentSize(in.data(), in.size()); + if (decompressed_size == ZSTD_CONTENTSIZE_UNKNOWN || + decompressed_size == ZSTD_CONTENTSIZE_ERROR) { + // Unknown size — use a streaming heuristic: start at 4x compressed size. + std::size_t buf_size = in.size() * 4; + constexpr std::size_t max_buf = 256 * 1024 * 1024; // 256 MiB safety cap + while (buf_size <= max_buf) { + out.resize(buf_size); + std::size_t result = ZSTD_decompress(out.data(), out.size(), in.data(), in.size()); + if (!ZSTD_isError(result)) { + out.resize(result); + return true; + } + if (ZSTD_getErrorCode(result) != ZSTD_error_dstSize_tooSmall) { + out.clear(); + return false; + } + buf_size *= 2; + } + out.clear(); + return false; + } + out.resize(static_cast(decompressed_size)); + std::size_t result = ZSTD_decompress(out.data(), out.size(), in.data(), in.size()); + if (ZSTD_isError(result)) { + out.clear(); + return false; + } + out.resize(result); + return true; +} + // ---------- state_compression_registry ---------- state_compression_registry::state_compression_registry() { register_codec(std::make_shared()); register_codec(std::make_shared()); + register_codec(std::make_shared()); } void state_compression_registry::register_codec(std::shared_ptr codec) { diff --git a/src/cvc/state_distributed_metrics.cpp b/src/cvc/state_distributed_metrics.cpp index 47e21a1..3ea0f44 100644 --- a/src/cvc/state_distributed_metrics.cpp +++ b/src/cvc/state_distributed_metrics.cpp @@ -51,4 +51,28 @@ std::size_t state_distributed_metrics::publish_transport_inproc(app &ctx, return 3; } +std::size_t state_distributed_metrics::publish_conflicts(app &ctx, + const state_cluster_shard &shard) { + auto entries = shard.recent_conflicts(32); + const std::string &cid = shard.cluster_id(); + std::size_t n = 0; + for (std::size_t i = 0; i < entries.size(); ++i) { + auto prefix = "conflicts.recent." + std::to_string(i); + try { + std::string base = "__system.distributed."; + base += cid.empty() ? "_unset_" : cid; + base += '.'; + base += prefix; + state::instance(ctx)(base + ".path").value(entries[i].path); + state::instance(ctx)(base + ".winner_node").value(entries[i].winner_node_id); + state::instance(ctx)(base + ".winner_seq").value(entries[i].winner_sequence); + state::instance(ctx)(base + ".loser_node").value(entries[i].loser_node_id); + state::instance(ctx)(base + ".loser_seq").value(entries[i].loser_sequence); + ++n; + } catch (...) { + } + } + return n; +} + } // namespace CVC_NAMESPACE diff --git a/src/cvc/state_replica.cpp b/src/cvc/state_replica.cpp index dd5e958..03904ce 100644 --- a/src/cvc/state_replica.cpp +++ b/src/cvc/state_replica.cpp @@ -151,6 +151,12 @@ state_replica::compare_clocks(const std::unordered_map current.hlc_time; + } if (incoming.origin_node_id == current.origin_node_id) return incoming.sequence > current.sequence; // Different origins: lexicographic origin then sequence. diff --git a/src/cvc/state_transport_grpc.cpp b/src/cvc/state_transport_grpc.cpp index 96ddcab..fe85ea6 100644 --- a/src/cvc/state_transport_grpc.cpp +++ b/src/cvc/state_transport_grpc.cpp @@ -91,6 +91,7 @@ void encode_mutation(const state_mutation &m, pb::Mutation *out) { out->set_type_name(m.type_name); out->set_string_value(m.string_value); out->set_latest_value_only(m.latest_value_only); + out->set_hlc_time(m.hlc_time); pb::Payload *p = out->mutable_payload(); switch (m.payload.kind) { @@ -122,6 +123,7 @@ state_mutation decode_mutation(const pb::Mutation &in) { m.type_name = in.type_name(); m.string_value = in.string_value(); m.latest_value_only = in.latest_value_only(); + m.hlc_time = in.hlc_time(); if (in.has_payload()) { const auto &p = in.payload(); switch (p.kind_case()) { @@ -257,6 +259,14 @@ class StateTransportServiceImpl final : public pb::StateTransport::Service { } else if (in.has_message()) { _owner->on_inbound_message(decode_message(in.message())); _owner->increment_recv_messages(); + } else if (in.has_chunk_request()) { + _owner->on_inbound_chunk_request( + conn.get(), in.chunk_request().digest(), in.chunk_request().request_id()); + } else if (in.has_chunk_response()) { + const auto &cr = in.chunk_response(); + std::string data_str = cr.data(); + std::vector data_vec(data_str.begin(), data_str.end()); + _owner->on_inbound_chunk_response(cr.request_id(), cr.found(), std::move(data_vec)); } } @@ -412,6 +422,14 @@ bool state_transport_grpc::connect_to_peer(const std::string &target, } else if (in.has_message()) { on_inbound_message(decode_message(in.message())); increment_recv_messages(); + } else if (in.has_chunk_request()) { + on_inbound_chunk_request(conn.get(), in.chunk_request().digest(), + in.chunk_request().request_id()); + } else if (in.has_chunk_response()) { + const auto &cr = in.chunk_response(); + std::string data_str = cr.data(); + std::vector data_vec(data_str.begin(), data_str.end()); + on_inbound_chunk_response(cr.request_id(), cr.found(), std::move(data_vec)); } } conn->alive.store(false); @@ -709,4 +727,101 @@ std::uint64_t state_transport_grpc::wait_for_received_messages(std::uint64_t tar return _recv_messages.load(); } +void state_transport_grpc::on_inbound_chunk_request(connection *conn, const std::string &digest, + std::uint64_t request_id) { + pb::Frame f; + auto *rsp = f.mutable_chunk_response(); + rsp->set_digest(digest); + rsp->set_request_id(request_id); + if (_blob_store) { + std::vector bytes; + if (_blob_store->get(digest, bytes)) { + rsp->set_found(true); + rsp->set_data(std::string(bytes.begin(), bytes.end())); + } else { + rsp->set_found(false); + } + } else { + rsp->set_found(false); + } + if (conn && conn->alive.load() && conn->write_fn) { + conn->write_fn(f); + _sent_frames.fetch_add(1, std::memory_order_relaxed); + } +} + +void state_transport_grpc::on_inbound_chunk_response(std::uint64_t request_id, bool found, + std::vector data) { + { + std::lock_guard lk(_chunk_waiters_mu); + auto it = _chunk_waiters.find(request_id); + if (it != _chunk_waiters.end()) { + it->second->found = found; + it->second->data = std::move(data); + it->second->done = true; + } + } + _chunk_waiters_cv.notify_all(); +} + +bool state_transport_grpc::fetch_chunk(const std::string &digest, chunk_callback on_chunk) { + if (digest.empty()) + return false; + + // Try local blob store first. + if (_blob_store) { + std::vector bytes; + if (_blob_store->get(digest, bytes)) { + if (on_chunk) + on_chunk(digest, bytes); + return true; + } + } + + // Ask connected peers. + std::uint64_t req_id = _next_chunk_req_id.fetch_add(1, std::memory_order_relaxed); + auto waiter = std::make_shared(); + { + std::lock_guard lk(_chunk_waiters_mu); + _chunk_waiters[req_id] = waiter; + } + + pb::Frame frame; + auto *req = frame.mutable_chunk_request(); + req->set_digest(digest); + req->set_request_id(req_id); + + std::vector> conns; + { + std::lock_guard lk(_conns_mu); + conns = _conns; + } + std::size_t sent = 0; + for (auto &c : conns) { + if (!c || !c->alive.load() || !c->write_fn) + continue; + if (c->write_fn(frame)) { + _sent_frames.fetch_add(1, std::memory_order_relaxed); + ++sent; + } + } + + bool result = false; + if (sent > 0) { + std::unique_lock lk(_chunk_waiters_mu); + _chunk_waiters_cv.wait_for(lk, std::chrono::seconds(10), [&]() { return waiter->done; }); + if (waiter->done && waiter->found) { + if (on_chunk) + on_chunk(digest, waiter->data); + result = true; + } + } + + { + std::lock_guard lk(_chunk_waiters_mu); + _chunk_waiters.erase(req_id); + } + return result; +} + } // namespace CVC_NAMESPACE diff --git a/src/cvc/state_transport_ipc.cpp b/src/cvc/state_transport_ipc.cpp index 30cd5b6..f262135 100644 --- a/src/cvc/state_transport_ipc.cpp +++ b/src/cvc/state_transport_ipc.cpp @@ -33,6 +33,10 @@ constexpr std::uint16_t kVersion = 1; constexpr std::uint16_t kMsgHello = 1; constexpr std::uint16_t kMsgMutation = 2; constexpr std::uint16_t kMsgOob = 3; +constexpr std::uint16_t kMsgChunkReq = 4; +constexpr std::uint16_t kMsgChunkRsp = 5; +constexpr std::uint16_t kMsgSnapReq = 6; +constexpr std::uint16_t kMsgSnapRsp = 7; constexpr std::size_t kMaxFrameBytes = 64u * 1024u * 1024u; void put_u8(std::vector &out, std::uint8_t v) { out.push_back(v); } @@ -159,6 +163,7 @@ std::vector encode_mutation(const state_mutation &m) { break; } put_u8(out, m.latest_value_only ? 1 : 0); + put_u64(out, m.hlc_time); return out; } @@ -191,6 +196,9 @@ bool decode_mutation(const std::vector &bytes, state_mutation &ou return false; } out.latest_value_only = r.u8() != 0; + // hlc_time was added later; tolerate its absence for backward compat. + if (r.ok() && r.remaining() >= 8) + out.hlc_time = r.u64(); return r.ok(); } @@ -508,6 +516,120 @@ void state_transport_ipc::reader_loop(std::shared_ptr conn) { } continue; } + if (mtype == kMsgChunkReq) { + // Serve chunk from local blob store. + reader br(body.data(), body.size()); + std::string digest = br.str(); + std::uint64_t req_id = br.u64(); + std::vector rsp; + put_string(rsp, digest); + put_u64(rsp, req_id); + if (_blob_store) { + std::vector chunk; + if (_blob_store->get(digest, chunk)) { + put_u8(rsp, 1); // found + put_bytes(rsp, chunk); + } else { + put_u8(rsp, 0); + put_u32(rsp, 0); // empty bytes + } + } else { + put_u8(rsp, 0); + put_u32(rsp, 0); + } + std::lock_guard wlk(conn->write_mu); + write_frame_locked(*conn, kMsgChunkRsp, rsp); + continue; + } + if (mtype == kMsgChunkRsp) { + reader br(body.data(), body.size()); + std::string digest = br.str(); + std::uint64_t req_id = br.u64(); + bool found = br.u8() != 0; + std::vector data = br.bytes(); + { + std::lock_guard lk(_chunk_waiters_mu); + auto it = _chunk_waiters.find(req_id); + if (it != _chunk_waiters.end()) { + it->second->found = found; + it->second->data = std::move(data); + it->second->done = true; + } + } + _chunk_waiters_cv.notify_all(); + continue; + } + if (mtype == kMsgSnapReq) { + // Serve a snapshot request from a peer. + reader br(body.data(), body.size()); + std::string cid = br.str(); + std::string prefix = br.str(); + std::uint64_t req_id = br.u64(); + + // Walk registered shards, gather state entries. + std::vector rsp; + put_u64(rsp, req_id); + std::uint32_t entry_count = 0; + std::size_t count_offset = rsp.size(); + put_u32(rsp, 0); // placeholder + + { + std::lock_guard slk(_shards_mu); + for (auto *shard : _shards) { + if (shard->cluster_id() != cid) + continue; + auto snap = shard->snapshot(prefix); + for (const auto &e : snap) { + put_string(rsp, e.path); + put_string(rsp, e.string_value); + put_string(rsp, e.comment); + put_u8(rsp, e.hidden ? 1 : 0); + put_u8(rsp, e.read_only ? 1 : 0); + put_string(rsp, e.type_name); + put_string(rsp, e.origin_node_id); + put_u64(rsp, e.sequence); + ++entry_count; + } + } + } + // Patch entry count. + for (int i = 0; i < 4; ++i) + rsp[count_offset + i] = + static_cast((entry_count >> (8 * i)) & 0xFF); + + std::lock_guard wlk(conn->write_mu); + write_frame_locked(*conn, kMsgSnapRsp, rsp); + continue; + } + if (mtype == kMsgSnapRsp) { + reader br(body.data(), body.size()); + std::uint64_t req_id = br.u64(); + std::uint32_t count = br.u32(); + std::vector entries; + entries.reserve(count); + for (std::uint32_t i = 0; i < count && br.ok(); ++i) { + snapshot_entry e; + e.path = br.str(); + e.string_value = br.str(); + e.comment = br.str(); + e.hidden = br.u8() != 0; + e.read_only = br.u8() != 0; + e.type_name = br.str(); + e.origin_node_id = br.str(); + e.sequence = br.u64(); + entries.push_back(std::move(e)); + } + { + std::lock_guard lk(_snap_waiters_mu); + auto it = _snap_waiters.find(req_id); + if (it != _snap_waiters.end()) { + it->second->entries = std::move(entries); + it->second->done = true; + } + } + _snap_waiters_cv.notify_all(); + continue; + } // Unknown frame: skip silently. } @@ -767,4 +889,116 @@ void state_transport_ipc::dispatch_inbound_message(const state_message &m) { (void)peer->ingest_remote_message(m); } +bool state_transport_ipc::fetch_chunk(const std::string &digest, chunk_callback on_chunk) { + if (digest.empty()) + return false; + + // Try local blob store first. + if (_blob_store) { + std::vector bytes; + if (_blob_store->get(digest, bytes)) { + if (on_chunk) + on_chunk(digest, bytes); + return true; + } + } + + // Ask connected peers. + std::uint64_t req_id = _next_chunk_req_id.fetch_add(1, std::memory_order_relaxed); + auto waiter = std::make_shared(); + { + std::lock_guard lk(_chunk_waiters_mu); + _chunk_waiters[req_id] = waiter; + } + + // Build and send request to all live connections. + std::vector req_body; + put_string(req_body, digest); + put_u64(req_body, req_id); + + std::vector> conns; + { + std::lock_guard lk(_conns_mu); + conns = _conns; + } + std::size_t sent = 0; + for (auto &c : conns) { + if (!c || !c->alive.load()) + continue; + std::lock_guard wlk(c->write_mu); + if (write_frame_locked(*c, kMsgChunkReq, req_body)) + ++sent; + } + + bool result = false; + if (sent > 0) { + // Wait for the first response (up to 10 seconds). + std::unique_lock lk(_chunk_waiters_mu); + _chunk_waiters_cv.wait_for(lk, std::chrono::seconds(10), + [&]() { return waiter->done; }); + if (waiter->done && waiter->found) { + if (on_chunk) + on_chunk(digest, waiter->data); + result = true; + } + } + + // Clean up waiter. + { + std::lock_guard lk(_chunk_waiters_mu); + _chunk_waiters.erase(req_id); + } + + return result; +} + +bool state_transport_ipc::request_snapshot(const std::string &cluster_id, + const std::string &path_prefix, + snapshot_callback on_entries) { + std::uint64_t req_id = _next_snap_req_id.fetch_add(1, std::memory_order_relaxed); + auto waiter = std::make_shared(); + { + std::lock_guard lk(_snap_waiters_mu); + _snap_waiters[req_id] = waiter; + } + + // Build request frame. + std::vector req_body; + put_string(req_body, cluster_id); + put_string(req_body, path_prefix); + put_u64(req_body, req_id); + + std::vector> conns; + { + std::lock_guard lk(_conns_mu); + conns = _conns; + } + std::size_t sent = 0; + for (auto &c : conns) { + if (!c || !c->alive.load()) + continue; + std::lock_guard wlk(c->write_mu); + if (write_frame_locked(*c, kMsgSnapReq, req_body)) + ++sent; + } + + bool result = false; + if (sent > 0) { + std::unique_lock lk(_snap_waiters_mu); + _snap_waiters_cv.wait_for(lk, std::chrono::seconds(10), + [&]() { return waiter->done; }); + if (waiter->done) { + if (on_entries) + on_entries(waiter->entries, /*final=*/true); + result = true; + } + } + + { + std::lock_guard lk(_snap_waiters_mu); + _snap_waiters.erase(req_id); + } + return result; +} + } // namespace CVC_NAMESPACE diff --git a/src/cvc/tests/CMakeLists.txt b/src/cvc/tests/CMakeLists.txt index 62bb77e..f9dbab4 100644 --- a/src/cvc/tests/CMakeLists.txt +++ b/src/cvc/tests/CMakeLists.txt @@ -27,6 +27,7 @@ add_executable(state_transport_inproc_test state_transport_inproc_test.cpp) # state_transport_ipc is POSIX-only (UDS + poll); skip on Windows. if(NOT WIN32) add_executable(state_transport_ipc_test state_transport_ipc_test.cpp) + add_executable(state_reconnect_resilience_test state_reconnect_resilience_test.cpp) endif() add_executable(state_blob_transport_integration_test state_blob_transport_integration_test.cpp) add_executable(state_distributed_delegation_integration_test state_distributed_delegation_integration_test.cpp) @@ -46,6 +47,7 @@ add_executable(state_volume_codec_test state_volume_codec_test.cpp) add_executable(state_brick_manifest_test state_brick_manifest_test.cpp) add_executable(state_data_hydrator_test state_data_hydrator_test.cpp) add_executable(distributed_state_session_test distributed_state_session_test.cpp) +add_executable(state_large_tree_bench_test state_large_tree_bench_test.cpp) if(CVC_ENABLE_GRPC) add_executable(state_transport_grpc_test state_transport_grpc_test.cpp) endif() @@ -338,6 +340,12 @@ if(NOT WIN32) GTest::gtest GTest::gtest_main ) + target_link_libraries(state_reconnect_resilience_test + PRIVATE + cvc + GTest::gtest + GTest::gtest_main + ) endif() target_link_libraries(state_blob_transport_integration_test @@ -466,6 +474,13 @@ target_link_libraries(distributed_state_session_test GTest::gtest_main ) +target_link_libraries(state_large_tree_bench_test + PRIVATE + cvc + GTest::gtest + GTest::gtest_main +) + if(TARGET state_transport_grpc_test) target_link_libraries(state_transport_grpc_test PRIVATE @@ -511,6 +526,7 @@ target_compile_features(state_bounded_queue_test PRIVATE cxx_std_17) target_compile_features(state_transport_inproc_test PRIVATE cxx_std_17) if(NOT WIN32) target_compile_features(state_transport_ipc_test PRIVATE cxx_std_17) + target_compile_features(state_reconnect_resilience_test PRIVATE cxx_std_17) endif() target_compile_features(state_blob_transport_integration_test PRIVATE cxx_std_17) target_compile_features(state_distributed_delegation_integration_test PRIVATE cxx_std_17) @@ -530,6 +546,7 @@ target_compile_features(state_volume_codec_test PRIVATE cxx_std_17) target_compile_features(state_brick_manifest_test PRIVATE cxx_std_17) target_compile_features(state_data_hydrator_test PRIVATE cxx_std_17) target_compile_features(distributed_state_session_test PRIVATE cxx_std_17) +target_compile_features(state_large_tree_bench_test PRIVATE cxx_std_17) # Only build HDF5-dependent tests when HDF5 support is enabled if(CVC_USING_HDF5 AND NOT CVC_HDF5_DISABLED) @@ -606,6 +623,7 @@ gtest_discover_tests(state_bounded_queue_test) gtest_discover_tests(state_transport_inproc_test) if(NOT WIN32) gtest_discover_tests(state_transport_ipc_test) + gtest_discover_tests(state_reconnect_resilience_test) endif() gtest_discover_tests(state_blob_transport_integration_test) gtest_discover_tests(state_distributed_delegation_integration_test) @@ -625,6 +643,7 @@ gtest_discover_tests(state_volume_codec_test) gtest_discover_tests(state_brick_manifest_test) gtest_discover_tests(state_data_hydrator_test) gtest_discover_tests(distributed_state_session_test) +gtest_discover_tests(state_large_tree_bench_test) if(TARGET state_transport_grpc_test) gtest_discover_tests(state_transport_grpc_test) endif() diff --git a/src/cvc/tests/state_compression_registry_test.cpp b/src/cvc/tests/state_compression_registry_test.cpp index 41037fe..600a307 100644 --- a/src/cvc/tests/state_compression_registry_test.cpp +++ b/src/cvc/tests/state_compression_registry_test.cpp @@ -34,8 +34,8 @@ TEST(StateCompressionRegistry, BuiltInCodecsRegistered) { cvc::state_compression_registry r; EXPECT_TRUE(r.has("raw")); EXPECT_TRUE(r.has("rle")); - EXPECT_FALSE(r.has("zstd")); - EXPECT_GE(r.size(), 2u); + EXPECT_TRUE(r.has("zstd")); + EXPECT_GE(r.size(), 3u); } TEST(StateCompressionRegistry, RawIsIdentity) { @@ -140,7 +140,7 @@ TEST(StateCompressionRegistry, RegisterIsIdempotentReplace) { r.register_codec(std::make_shared()); auto raw_b = r.get("raw"); EXPECT_NE(raw_a.get(), raw_b.get()); // replaced, not duplicated - EXPECT_EQ(r.size(), 2u); // still raw + rle + EXPECT_EQ(r.size(), 3u); // still raw + rle + zstd } TEST(StateCompressionRegistry, RegisterNullIsNoOp) { @@ -180,9 +180,10 @@ TEST(StateCompressionRegistry, IdsListReflectsRegistered) { cvc::state_compression_registry r; auto ids = r.ids(); std::sort(ids.begin(), ids.end()); - ASSERT_EQ(ids.size(), 2u); + ASSERT_EQ(ids.size(), 3u); EXPECT_EQ(ids[0], "raw"); EXPECT_EQ(ids[1], "rle"); + EXPECT_EQ(ids[2], "zstd"); } TEST(StateCompressionRegistry, SharedSingletonHasBuiltins) { diff --git a/src/cvc/tests/state_large_tree_bench_test.cpp b/src/cvc/tests/state_large_tree_bench_test.cpp new file mode 100644 index 0000000..06d744e --- /dev/null +++ b/src/cvc/tests/state_large_tree_bench_test.cpp @@ -0,0 +1,147 @@ +/* + Copyright 2026 The University of Texas at Austin + + This file is part of libcvc. + + libcvc is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. +*/ + +// Large-tree benchmark: measures ingest throughput and snapshot +// performance with trees of 10k–100k nodes. Gated on env var +// CVC_DISTRIBUTED_STATE_BENCH=1 so it runs only on demand. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +bool bench_enabled() { + const char *v = std::getenv("CVC_DISTRIBUTED_STATE_BENCH"); + return v && std::string(v) == "1"; +} + +cvc::state_mutation make_set_value(const std::string &origin, std::uint64_t seq, + const std::string &path, const std::string &val) { + cvc::state_mutation m; + m.cluster_id = "bench"; + m.tree_id = "default"; + m.origin_node_id = origin; + m.sequence = seq; + m.mutation_id = origin + ":" + std::to_string(seq); + m.path = path; + m.op = cvc::state_mutation_op::set_value; + m.type_name = "std::string"; + m.string_value = val; + m.latest_value_only = true; + return m; +} + +} // namespace + +TEST(LargeTreeBench, Ingest10kNodes) { + if (!bench_enabled()) { + GTEST_SKIP() << "Set CVC_DISTRIBUTED_STATE_BENCH=1 to enable"; + } + cvc::app ctx; + cvc::state_cluster_shard shard(ctx, "bench", "local"); + shard.attach(); + + constexpr int N = 10000; + auto t0 = std::chrono::steady_clock::now(); + for (int i = 0; i < N; ++i) { + auto m = make_set_value("remote", static_cast(i + 1), + "tree.node_" + std::to_string(i), "value_" + std::to_string(i)); + shard.ingest_remote(m); + } + auto t1 = std::chrono::steady_clock::now(); + auto ns = std::chrono::duration_cast(t1 - t0).count(); + std::printf("BENCH Ingest10kNodes N=%d wall_ns_total=%lld wall_ns_per_op=%lld\n", N, + static_cast(ns), static_cast(ns / N)); + SUCCEED(); +} + +TEST(LargeTreeBench, Ingest100kNodes) { + if (!bench_enabled()) { + GTEST_SKIP() << "Set CVC_DISTRIBUTED_STATE_BENCH=1 to enable"; + } + cvc::app ctx; + cvc::state_cluster_shard shard(ctx, "bench", "local"); + shard.attach(); + + constexpr int N = 100000; + auto t0 = std::chrono::steady_clock::now(); + for (int i = 0; i < N; ++i) { + auto m = make_set_value("remote", static_cast(i + 1), + "tree.node_" + std::to_string(i), "value_" + std::to_string(i)); + shard.ingest_remote(m); + } + auto t1 = std::chrono::steady_clock::now(); + auto ns = std::chrono::duration_cast(t1 - t0).count(); + std::printf("BENCH Ingest100kNodes N=%d wall_ns_total=%lld wall_ns_per_op=%lld\n", N, + static_cast(ns), static_cast(ns / N)); + SUCCEED(); +} + +TEST(LargeTreeBench, DrainAndPublish10k) { + if (!bench_enabled()) { + GTEST_SKIP() << "Set CVC_DISTRIBUTED_STATE_BENCH=1 to enable"; + } + cvc::app ctx; + cvc::state_transport_inproc transport; + cvc::state_cluster_shard shard(ctx, "bench", "local"); + shard.attach(); + transport.register_shard(&shard); + shard.set_transport(&transport); + + // Populate locally. + constexpr int N = 10000; + for (int i = 0; i < N; ++i) { + cvc::state::instance(ctx)("tree.node_" + std::to_string(i)) + .value(std::string("val_" + std::to_string(i))); + } + + auto t0 = std::chrono::steady_clock::now(); + std::size_t pumped = transport.pump_all(); + auto t1 = std::chrono::steady_clock::now(); + auto ns = std::chrono::duration_cast(t1 - t0).count(); + std::printf("BENCH DrainAndPublish10k pumped=%zu wall_ns_total=%lld wall_ns_per_op=%lld\n", + pumped, static_cast(ns), + pumped > 0 ? static_cast(ns / static_cast(pumped)) : 0LL); + transport.unregister_shard(&shard); + SUCCEED(); +} + +TEST(LargeTreeBench, Snapshot10kNodes) { + if (!bench_enabled()) { + GTEST_SKIP() << "Set CVC_DISTRIBUTED_STATE_BENCH=1 to enable"; + } + cvc::app ctx; + cvc::state_cluster_shard shard(ctx, "bench", "local"); + shard.attach(); + + constexpr int N = 10000; + for (int i = 0; i < N; ++i) { + auto m = make_set_value("remote", static_cast(i + 1), + "tree.node_" + std::to_string(i), "value_" + std::to_string(i)); + shard.ingest_remote(m); + } + + auto t0 = std::chrono::steady_clock::now(); + auto snap = shard.snapshot("tree"); + auto t1 = std::chrono::steady_clock::now(); + auto ns = std::chrono::duration_cast(t1 - t0).count(); + std::printf("BENCH Snapshot10kNodes entries=%zu wall_ns_total=%lld\n", snap.size(), + static_cast(ns)); + EXPECT_GE(snap.size(), 1u); + SUCCEED(); +} diff --git a/src/cvc/tests/state_reconnect_resilience_test.cpp b/src/cvc/tests/state_reconnect_resilience_test.cpp new file mode 100644 index 0000000..8d584ae --- /dev/null +++ b/src/cvc/tests/state_reconnect_resilience_test.cpp @@ -0,0 +1,176 @@ +/* + Copyright 2026 The University of Texas at Austin + + This file is part of libcvc. + + libcvc is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------- +// Reconnect / resilience tests for distributed-state transports. +// Uses the IPC (Unix domain socket) transport since it is the +// simplest multi-process-capable transport and is POSIX-only. +// --------------------------------------------------------------- + +namespace { + +std::string make_socket_path(const std::string &label) { + auto pid = static_cast(::getpid()); + auto now = std::chrono::steady_clock::now().time_since_epoch().count(); + auto dir = std::filesystem::temp_directory_path(); + return (dir / ("cvc_res_" + std::to_string(pid) + "_" + std::to_string(now) + "_" + label + + ".sock")) + .string(); +} + +bool wait_connected(cvc::state_transport_ipc &a, cvc::state_transport_ipc &b, + std::chrono::milliseconds to) { + auto deadline = std::chrono::steady_clock::now() + to; + while (std::chrono::steady_clock::now() < deadline) { + if (a.connection_count() >= 1 && b.connection_count() >= 1) + return true; + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return false; +} + +} // namespace + +// --------------------------------------------------------------- +// Test: data replicates, one side stops, restarts, reconnects, +// and subsequent writes still replicate. +// --------------------------------------------------------------- +TEST(StateReconnectResilienceTest, StopRestartReconnect) { + cvc::app aA, aB; + + auto pathA = make_socket_path("A_recon"); + auto pathB = make_socket_path("B_recon"); + + // -- Phase 1: establish and replicate. + auto tA = std::make_unique(); + cvc::state_transport_ipc tB; + tA->start(pathA, "A", "C"); + tB.start(pathB, "B", "C"); + ASSERT_TRUE(tA->connect_to_peer(pathB, std::chrono::milliseconds(2000))); + ASSERT_TRUE(wait_connected(*tA, tB, std::chrono::milliseconds(2000))); + + cvc::state_cluster_shard sA(aA, "C", "A"); + cvc::state_cluster_shard sB(aB, "C", "B"); + sA.attach(); + sB.attach(); + tA->register_shard(&sA); + tB.register_shard(&sB); + + cvc::state::instance(aA)("k").value(std::string("seed")); + cvc::state::instance(aA)("k").value(std::string("v1")); + tA->pump_all(); + tA->flush(); + tB.wait_for_received(1, std::chrono::milliseconds(2000)); + EXPECT_EQ(cvc::state::instance(aB)("k").value(), "v1"); + + // -- Phase 2: stop A's transport (simulate crash / restart). + tA->unregister_shard(&sA); + tA->stop(); + tA.reset(); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + // -- Phase 3: restart A with a new transport, reconnect to B. + tA = std::make_unique(); + tA->start(pathA, "A", "C"); + ASSERT_TRUE(tA->connect_to_peer(pathB, std::chrono::milliseconds(2000))); + ASSERT_TRUE(wait_connected(*tA, tB, std::chrono::milliseconds(2000))); + tA->register_shard(&sA); + + // -- Phase 4: replicate after reconnect. + cvc::state::instance(aA)("k").value(std::string("v2_after_reconnect")); + tA->pump_all(); + tA->flush(); + tB.wait_for_received(2, std::chrono::milliseconds(2000)); + EXPECT_EQ(cvc::state::instance(aB)("k").value(), "v2_after_reconnect"); + + tA->stop(); + tB.stop(); +} + +// --------------------------------------------------------------- +// Test: peer goes away, local side survives (no crash/hang). +// --------------------------------------------------------------- +TEST(StateReconnectResilienceTest, PeerDisconnectNoHang) { + cvc::app aA, aB; + + auto pathA = make_socket_path("A_disc"); + auto pathB = make_socket_path("B_disc"); + + cvc::state_transport_ipc tA, tB; + tA.start(pathA, "A", "C"); + tB.start(pathB, "B", "C"); + ASSERT_TRUE(tA.connect_to_peer(pathB, std::chrono::milliseconds(2000))); + ASSERT_TRUE(wait_connected(tA, tB, std::chrono::milliseconds(2000))); + + cvc::state_cluster_shard sA(aA, "C", "A"); + cvc::state_cluster_shard sB(aB, "C", "B"); + sA.attach(); + sB.attach(); + tA.register_shard(&sA); + tB.register_shard(&sB); + + cvc::state::instance(aA)("x").value(std::string("seed")); + cvc::state::instance(aA)("x").value(std::string("v1")); + tA.pump_all(); + tA.flush(); + tB.wait_for_received(1, std::chrono::milliseconds(2000)); + EXPECT_EQ(cvc::state::instance(aB)("x").value(), "v1"); + + // B goes away. + tB.stop(); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + // A can still pump without crashing. + cvc::state::instance(aA)("x").value(std::string("v2")); + EXPECT_NO_THROW(tA.pump_all()); + // Flush may fail silently since B is gone — just must not hang. + EXPECT_NO_THROW(tA.flush()); + + tA.stop(); +} + +// --------------------------------------------------------------- +// Test: session-level status() reflects peer count correctly. +// --------------------------------------------------------------- +TEST(StateReconnectResilienceTest, ConnectionCountTracking) { + auto pathA = make_socket_path("A_cnt"); + auto pathB = make_socket_path("B_cnt"); + + cvc::state_transport_ipc tA, tB; + EXPECT_EQ(tA.connection_count(), 0u); + + tA.start(pathA, "A", "C"); + tB.start(pathB, "B", "C"); + EXPECT_EQ(tA.connection_count(), 0u); + + ASSERT_TRUE(tA.connect_to_peer(pathB, std::chrono::milliseconds(2000))); + ASSERT_TRUE(wait_connected(tA, tB, std::chrono::milliseconds(2000))); + EXPECT_GE(tA.connection_count(), 1u); + EXPECT_GE(tB.connection_count(), 1u); + + tB.stop(); + // Give reader threads time to notice the disconnect. + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + // A's connection_count may drop; at minimum we don't crash. + EXPECT_NO_THROW(tA.connection_count()); + + tA.stop(); +} From 65117aa4522dc31fcb41dd66126f8f4278216cde Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Fri, 22 May 2026 19:45:26 -0500 Subject: [PATCH 2/4] distributed-state: close remaining gaps (P0-P3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P0 — Production Blockers: - gRPC snapshot + heartbeat frame dispatch in both server and client - Wire hybrid_clock into shard (stamp outbound, merge inbound) - IPC wire version negotiation (kMaxVersion, skip unknown msg types) P1 — Functional Gaps: - Wire hash_partition into shard with enforce_partition gate - gRPC heartbeat send/receive (background heartbeat_loop thread) - Session config: blob_store_path, snapshot_on_join wired P2 — Test Coverage & Docs: - 7 new test suites (47 tests): hash_partition, hybrid_time, brick_frustum, conflict_ring, snapshot_protocol, zstd_round_trip, session_status - gRPC reconnect/resilience tests (3 tests) - USAGE.md: distributed state section (quick start, config reference, transports, state API, interest filters, snapshots, conflict resolution, session lifecycle) - CI: gRPC build matrix entry (libcvc-debug-grpc) for Linux P3 — Nice to Have: - require_tls safety boolean on distributed_state_config Bug fixes: - hash_partition owner_of() uint32 overflow (UINT32_MAX + 1 wraps) 19 files changed, +1560/-4 lines --- .github/workflows/ci.yml | 15 +- USAGE.md | 181 ++++++++++++++ inc/cvc/distributed_state_session.h | 3 + inc/cvc/state_cluster_shard.h | 14 ++ inc/cvc/state_hash_partition.h | 12 +- inc/cvc/state_transport_grpc.h | 28 +++ src/cvc/distributed_state_session.cpp | 24 ++ src/cvc/state_cluster_shard.cpp | 38 +++ src/cvc/state_transport_grpc.cpp | 220 ++++++++++++++++++ src/cvc/state_transport_ipc.cpp | 9 +- src/cvc/tests/CMakeLists.txt | 78 +++++++ src/cvc/tests/state_brick_frustum_test.cpp | 104 +++++++++ src/cvc/tests/state_conflict_ring_test.cpp | 135 +++++++++++ src/cvc/tests/state_hash_partition_test.cpp | 127 ++++++++++ src/cvc/tests/state_hybrid_time_test.cpp | 135 +++++++++++ src/cvc/tests/state_session_status_test.cpp | 102 ++++++++ .../tests/state_snapshot_protocol_test.cpp | 141 +++++++++++ src/cvc/tests/state_transport_grpc_test.cpp | 109 +++++++++ src/cvc/tests/state_zstd_round_trip_test.cpp | 89 +++++++ 19 files changed, 1560 insertions(+), 4 deletions(-) create mode 100644 src/cvc/tests/state_brick_frustum_test.cpp create mode 100644 src/cvc/tests/state_conflict_ring_test.cpp create mode 100644 src/cvc/tests/state_hash_partition_test.cpp create mode 100644 src/cvc/tests/state_hybrid_time_test.cpp create mode 100644 src/cvc/tests/state_session_status_test.cpp create mode 100644 src/cvc/tests/state_snapshot_protocol_test.cpp create mode 100644 src/cvc/tests/state_zstd_round_trip_test.cpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51d57f8..b800a51 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,6 +125,10 @@ jobs: - name: libcvc-release kind: libcvc build_type: Release + - name: libcvc-debug-grpc + kind: libcvc + build_type: Debug + enable_grpc: true - name: volrover3-debug kind: volrover3 build_type: Debug @@ -167,6 +171,10 @@ jobs: if [ "${{ matrix.build_type }}" = "Debug" ]; then sudo apt-get install -y --no-install-recommends lcov fi + if [ "${{ matrix.enable_grpc }}" = "true" ]; then + sudo apt-get install -y --no-install-recommends \ + libgrpc++-dev libprotobuf-dev protobuf-compiler-grpc + fi # CUDA toolkit so the shipped artifacts can offload to NVIDIA GPUs # at runtime when an NVIDIA driver is present. cudart is linked @@ -292,6 +300,10 @@ jobs: coverage=ON fi fi + grpc=OFF + if [ "${{ matrix.enable_grpc }}" = "true" ]; then + grpc=ON + fi cmake -B build -G Ninja \ -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ -DCMAKE_PREFIX_PATH="$extra_prefix" \ @@ -302,7 +314,8 @@ jobs: -DDISABLE_CGAL=OFF \ -DCVC_BUILD_VOLROVER3=${{ matrix.kind == 'volrover3' && 'ON' || 'OFF' }} \ -DCVC_ENABLE_MESHER=ON \ - -DCVC_ENABLE_SDF=ON + -DCVC_ENABLE_SDF=ON \ + -DCVC_ENABLE_GRPC=$grpc - name: Build run: cmake --build build --parallel diff --git a/USAGE.md b/USAGE.md index 799e036..b74e9b3 100644 --- a/USAGE.md +++ b/USAGE.md @@ -89,3 +89,184 @@ cmake -B build -DCMAKE_PREFIX_PATH=/path/to/libcvc-3.1.0 cmake --build build ./build/my_app ``` + +--- + +## 6. Distributed state + +libcvc includes an optional replicated state layer that synchronises an +in-process key-value tree across multiple nodes in a cluster. The layer +is assembled from composable pieces — a shard, a transport, interest +filters, conflict resolution, blob storage — but the easiest entry point +is `distributed_state_session`, which wires everything together from a +single config struct. + +### 6.1 Quick start + +```cpp +#include +#include +#include + +int main() { + cvc::app ctx; + + cvc::distributed_state_config cfg; + cfg.cluster_id = "my_cluster"; + cfg.node_id = "node_1"; + cfg.transport = cvc::transport_kind::ipc; + cfg.listen_address = "/tmp/my_cluster_node1.sock"; + cfg.seeds = {"/tmp/my_cluster_node2.sock"}; + + auto session = cvc::distributed_state_session::join(ctx, cfg); + + // Write a value — it replicates to every peer in the cluster. + cvc::state::instance(ctx)("scene.title").value(std::string("Hello")); + + // Read a value (may have arrived from a remote peer). + std::string title = cvc::state::instance(ctx)("scene.title").value(); + + session->stop(); // graceful shutdown (also called by destructor) +} +``` + +### 6.2 Configuration reference (`distributed_state_config`) + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `cluster_id` | `string` | *(required)* | Unique cluster name shared by all peers | +| `node_id` | `string` | *(required)* | Unique node name within the cluster | +| `root_path` | `string` | `""` | Subtree prefix to replicate (`""` = whole tree) | +| `transport` | `transport_kind` | `inproc` | `inproc`, `ipc`, or `grpc` | +| `listen_address` | `string` | `""` | Socket path (IPC) or `host:port` (gRPC) | +| `seeds` | `vector` | `{}` | Peer endpoints to connect to on startup | +| `mounts` | `vector` | `{}` | Per-path replication modes | +| `pump_interval_ms` | `uint32` | `10` | Background replication interval (0 = no pump thread) | +| `max_inline_payload_bytes` | `uint32` | `65536` | Values larger than this go to the blob store | +| `blob_store_path` | `string` | `""` | Filesystem path for blob persistence (`""` = memory only) | +| `snapshot_on_join` | `bool` | `false` | Request a full snapshot from the first seed on join | +| `enforce_authority` | `bool` | `false` | Reject mutations owned by a different cluster | +| `enforce_write_policy` | `bool` | `false` | Consult write-policy before applying remote writes | +| `resolve_conflicts` | `bool` | `false` | Track and resolve concurrent writes (last-writer-wins) | +| `enforce_delegation` | `bool` | `false` | Respect delegation boundaries | +| `enforce_interest` | `bool` | `false` | Drop inbound mutations outside the interest set | + +**gRPC-only TLS / auth fields** (ignored for `inproc` / `ipc`): + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `tls_server_cert_pem` | `string` | `""` | PEM server certificate | +| `tls_server_key_pem` | `string` | `""` | PEM server private key | +| `tls_root_ca_pem` | `string` | `""` | PEM CA certificate for peer verification | +| `tls_require_client_auth` | `bool` | `false` | Require mutual TLS | +| `require_tls` | `bool` | `false` | Throw if TLS cert/key are missing (safety guard) | +| `auth_expected_token` | `string` | `""` | Bearer token expected from inbound peers | +| `auth_outbound_token` | `string` | `""` | Bearer token sent to outbound peers | + +> **Note:** identifiers (`cluster_id`, `node_id`) must follow C identifier +> rules — alphanumeric plus underscores. Hyphens are not allowed. + +### 6.3 Transports + +**In-process (`inproc`)** — Multiple shards in one process, no network. +Useful for tests and multi-tree applications. + +**IPC (Unix domain sockets)** — Same-host, multi-process replication. +Full-duplex, automatic reconnection. + +```cpp +cfg.transport = cvc::transport_kind::ipc; +cfg.listen_address = "/tmp/cluster.sock"; +cfg.seeds = {"/tmp/peer1.sock", "/tmp/peer2.sock"}; +``` + +**gRPC (network)** — Cross-host clustering with optional TLS and bearer-token +auth. Only available when libcvc is built with `-DCVC_ENABLE_GRPC=ON`. + +```cpp +cfg.transport = cvc::transport_kind::grpc; +cfg.listen_address = "0.0.0.0:9999"; +cfg.seeds = {"peer1.example.com:9999"}; +cfg.tls_server_cert_pem = load_file("server.crt"); +cfg.tls_server_key_pem = load_file("server.key"); +cfg.tls_root_ca_pem = load_file("ca.crt"); +``` + +### 6.4 Reading and writing state + +State nodes are addressed by dot-separated paths and accessed through the +`cvc::state` API: + +```cpp +auto &root = cvc::state::instance(ctx); + +// Write (any type convertible to string) +root("scene.camera.fov").value(std::string("90")); +root("scene.camera.fov").value(90); + +// Read +std::string fov = root("scene.camera.fov").value(); +int fov_i = root("scene.camera.fov").value(); + +// Block until a value arrives +std::string val = root("data.result").wait_for_value(); + +// Metadata +std::string name = root("scene.camera").name(); // "camera" +std::string full = root("scene.camera").fullName(); // "scene.camera" +``` + +### 6.5 Interest filters + +In large clusters a node can limit which paths it mirrors by registering +prefix-based interest filters: + +```cpp +auto &shard = session->shard(); +shard.add_interest("scene.geometry"); // receive scene.geometry.* +shard.add_interest("scene.metadata"); +shard.set_enforce_interest(true); // drop everything else + +// Query +bool ok = shard.path_is_of_interest("scene.geometry.mesh"); // true +auto all = shard.interests(); +shard.remove_interest("scene.geometry"); +``` + +### 6.6 Snapshots and conflict resolution + +**Snapshots** let a new node catch up to the current cluster state: + +```cpp +// Automatic on join: +cfg.snapshot_on_join = true; + +// Manual: +auto entries = shard.snapshot("scene"); // prefix filter +``` + +**Conflict resolution** uses deterministic last-writer-wins ordering +(lexicographic `node_id` + sequence number): + +```cpp +shard.set_resolve_conflicts(true); + +// Inspect recent conflicts: +auto conflicts = shard.recent_conflicts(64); +for (auto &c : conflicts) + std::cout << c.path << ": winner=" << c.winner_node_id << "\n"; +``` + +### 6.7 Session lifecycle + +```cpp +auto session = cvc::distributed_state_session::join(ctx, cfg); + +session->is_running(); // true while active +session->status(); // replica health snapshot +session->shard(); // access the underlying shard +session->transport(); // access the transport +session->blob_store(); // access the blob store + +session->stop(); // graceful shutdown +``` diff --git a/inc/cvc/distributed_state_session.h b/inc/cvc/distributed_state_session.h index 52c608f..2d78fb6 100644 --- a/inc/cvc/distributed_state_session.h +++ b/inc/cvc/distributed_state_session.h @@ -91,11 +91,14 @@ struct distributed_state_config { std::string tls_server_key_pem; std::string tls_root_ca_pem; bool tls_require_client_auth = false; + bool require_tls = false; // when true, session throws if TLS certs are missing std::string auth_expected_token; std::string auth_outbound_token; // Tuning. std::uint32_t max_inline_payload_bytes = 65536; + std::string blob_store_path; // empty = memory-only blob store + bool snapshot_on_join = false; // request full snapshot from first seed on join std::uint32_t pump_interval_ms = 10; // background pump loop interval (0 = no pump thread) }; diff --git a/inc/cvc/state_cluster_shard.h b/inc/cvc/state_cluster_shard.h index 8c53e88..61e0eff 100644 --- a/inc/cvc/state_cluster_shard.h +++ b/inc/cvc/state_cluster_shard.h @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include #include @@ -96,6 +98,14 @@ class state_cluster_shard { state_codec_registry &codecs() noexcept { return *_codecs; } state_message_bus &message_bus() noexcept { return *_message_bus; } state_write_policy &write_policy() noexcept { return *_write_policy; } + hybrid_clock &clock() noexcept { return _clock; } + state_hash_partition &partition() noexcept { return _partition; } + const state_hash_partition &partition() const noexcept { return _partition; } + + // When true and the partition map is non-empty, ingest_remote() + // rejects mutations whose path hashes to a different node_id. + void set_enforce_partition(bool enforce) noexcept; + bool enforce_partition() const noexcept; // Wire up adapter observers and start journaling local changes. void attach(); @@ -360,6 +370,7 @@ class state_cluster_shard { std::atomic _ctr_delegations_applied{0}; std::atomic _ctr_revocations_applied{0}; std::atomic _ctr_remote_filtered_out{0}; + std::atomic _ctr_partition_rejected{0}; // Inbound interest filter. std::vector _interests; @@ -368,6 +379,9 @@ class state_cluster_shard { // Phase 8 slice 2. state_transport *_transport = nullptr; app *_app_ctx = nullptr; // captured at construction for default_for() + hybrid_clock _clock; + state_hash_partition _partition; + bool _enforce_partition = false; }; } // namespace CVC_NAMESPACE diff --git a/inc/cvc/state_hash_partition.h b/inc/cvc/state_hash_partition.h index 500a7b4..8aad896 100644 --- a/inc/cvc/state_hash_partition.h +++ b/inc/cvc/state_hash_partition.h @@ -76,8 +76,16 @@ class state_hash_partition { std::uint32_t h = hash(path); std::lock_guard lk(_mu); for (const auto &r : _ranges) { - if (h >= r.range_begin && h < r.range_end) - return r.node_id; + // range_end == 0 means the range wraps the full 32-bit space + // (UINT32_MAX + 1 overflowed to 0). Treat as "covers everything + // from range_begin onward". + if (r.range_end == 0) { + if (h >= r.range_begin) + return r.node_id; + } else { + if (h >= r.range_begin && h < r.range_end) + return r.node_id; + } } return {}; } diff --git a/inc/cvc/state_transport_grpc.h b/inc/cvc/state_transport_grpc.h index 2d74e85..84bbf61 100644 --- a/inc/cvc/state_transport_grpc.h +++ b/inc/cvc/state_transport_grpc.h @@ -117,6 +117,12 @@ class state_transport_grpc final : public state_transport { // Shut down server, cancel all client streams, join reader threads. void stop(); + // Heartbeat interval. When non-zero, a background thread sends + // Heartbeat frames to all connections at this cadence. Default 0 + // (disabled). + void set_heartbeat_interval(std::chrono::milliseconds interval) noexcept; + std::chrono::milliseconds heartbeat_interval() const noexcept; + // Blob store for servicing inbound chunk requests and for local // lookup before sending outbound requests. void set_blob_store(state_blob_store *store) noexcept { _blob_store = store; } @@ -131,6 +137,8 @@ class state_transport_grpc final : public state_transport { std::size_t pump_all() override; void flush() override; bool fetch_chunk(const std::string &digest, chunk_callback on_chunk) override; + bool request_snapshot(const std::string &cluster_id, const std::string &path_prefix, + snapshot_callback on_entries) override; // Diagnostics. std::size_t shard_count() const; @@ -166,6 +174,11 @@ class state_transport_grpc final : public state_transport { std::uint64_t request_id); void on_inbound_chunk_response(std::uint64_t request_id, bool found, std::vector data); + void on_inbound_snapshot_request(connection *conn, const std::string &cluster_id, + const std::string &path_prefix, std::uint64_t request_id); + void on_inbound_snapshot_response(std::uint64_t request_id, + const std::vector &entries, bool final); + void on_inbound_heartbeat(const std::string &node_id, const std::string &cluster_id); void register_connection(std::shared_ptr conn); void unregister_connection(connection *conn); void increment_recv_frames() noexcept { _recv_frames.fetch_add(1, std::memory_order_relaxed); } @@ -213,6 +226,21 @@ class state_transport_grpc final : public state_transport { std::vector data; }; std::unordered_map> _chunk_waiters; + + // Snapshot request/response tracking. + std::atomic _next_snap_req_id{1}; + mutable std::mutex _snap_waiters_mu; + std::condition_variable _snap_waiters_cv; + struct snap_waiter { + bool done = false; + std::vector entries; + }; + std::unordered_map> _snap_waiters; + + // Heartbeat sender. + std::chrono::milliseconds _heartbeat_interval{0}; + std::thread _heartbeat_thread; + void heartbeat_loop(); }; } // namespace CVC_NAMESPACE diff --git a/src/cvc/distributed_state_session.cpp b/src/cvc/distributed_state_session.cpp index 546033f..ef11917 100644 --- a/src/cvc/distributed_state_session.cpp +++ b/src/cvc/distributed_state_session.cpp @@ -55,6 +55,11 @@ distributed_state_session::join(app &ctx, const distributed_state_config &config } case transport_kind::grpc: { #ifdef CVC_ENABLE_GRPC + if (config.require_tls && + (config.tls_server_cert_pem.empty() || config.tls_server_key_pem.empty())) { + throw std::runtime_error( + "distributed_state_session: require_tls is true but TLS certificate/key not provided"); + } auto grpc = std::make_unique(); // Apply TLS / auth config if provided. if (!config.tls_server_cert_pem.empty() || !config.tls_root_ca_pem.empty()) { @@ -142,6 +147,25 @@ distributed_state_session::join(app &ctx, const distributed_state_config &config session->_shard->attach(); session->_shard->install_as_default(); + // 10b. Request initial snapshot from first seed if configured. + if (config.snapshot_on_join && !config.seeds.empty()) { + session->_transport->request_snapshot( + config.cluster_id, config.root_path, + [&session](const std::vector &entries, bool /*final*/) { + for (const auto &e : entries) { + state_mutation m; + m.cluster_id = session->_config.cluster_id; + m.origin_node_id = e.origin_node_id; + m.sequence = e.sequence; + m.path = e.path; + m.op = state_mutation_op::set_value; + m.string_value = e.string_value; + m.type_name = e.type_name; + session->_shard->ingest_remote(m); + } + }); + } + // 11. Start pump thread. session->_running.store(true, std::memory_order_release); if (config.pump_interval_ms > 0) { diff --git a/src/cvc/state_cluster_shard.cpp b/src/cvc/state_cluster_shard.cpp index e43fff0..5d1ccb9 100644 --- a/src/cvc/state_cluster_shard.cpp +++ b/src/cvc/state_cluster_shard.cpp @@ -217,6 +217,25 @@ state_cluster_shard::ingest_result state_cluster_shard::ingest_remote(const stat return r; } + // Hash-partition enforcement: when enabled, reject mutations + // whose path hashes to a node other than this one. This + // prevents a shard from accumulating data it should not own. + bool enforce_part; + { + std::lock_guard lk(_mutex); + enforce_part = _enforce_partition; + } + if (enforce_part && _partition.size() > 0) { + std::string owner = _partition.owner_of(m.path); + if (!owner.empty() && owner != _local_node_id) { + r.rejected = true; + r.reject_reason = + "path '" + m.path + "' partitioned to node '" + owner + "', not '" + _local_node_id + "'"; + _ctr_partition_rejected.fetch_add(1, std::memory_order_relaxed); + return r; + } + } + // Optional authority enforcement: a path that resolves to an // authority entry whose cluster_id differs from this shard's // cluster_id is rejected. @@ -301,6 +320,11 @@ state_cluster_shard::ingest_result state_cluster_shard::ingest_remote(const stat (void)_replica->seen(m.origin_node_id, m.sequence, /*record*/ true); _replica->observe_remote(m.origin_node_id, m.sequence); + // Merge the remote HLC timestamp into our local clock so our + // next outbound mutation will be causally after this one. + if (m.hlc_time != 0) + _clock.update(hybrid_time::from_packed(m.hlc_time)); + if (conflict_lost) { _ctr_conflicts_lost.fetch_add(1, std::memory_order_relaxed); // Mark as seen but do not apply. @@ -340,6 +364,10 @@ std::vector state_cluster_shard::drain_local(std::size_t max_cou // the wire so peers can route by cluster_id. if (m.cluster_id.empty()) m.cluster_id = _cluster_id; + // Stamp HLC time on outbound mutations so receivers can merge + // causal ordering via their own hybrid_clock::update(). + if (m.hlc_time == 0) + m.hlc_time = _clock.now().packed(); out.push_back(std::move(m)); if (max_count != 0 && out.size() >= max_count) break; @@ -486,6 +514,16 @@ bool state_cluster_shard::enforce_interest() const noexcept { return _enforce_interest; } +void state_cluster_shard::set_enforce_partition(bool enforce) noexcept { + std::lock_guard lk(_mutex); + _enforce_partition = enforce; +} + +bool state_cluster_shard::enforce_partition() const noexcept { + std::lock_guard lk(_mutex); + return _enforce_partition; +} + void state_cluster_shard::set_transport(state_transport *t) noexcept { std::lock_guard lk(_mutex); _transport = t; diff --git a/src/cvc/state_transport_grpc.cpp b/src/cvc/state_transport_grpc.cpp index fe85ea6..971880c 100644 --- a/src/cvc/state_transport_grpc.cpp +++ b/src/cvc/state_transport_grpc.cpp @@ -267,6 +267,29 @@ class StateTransportServiceImpl final : public pb::StateTransport::Service { std::string data_str = cr.data(); std::vector data_vec(data_str.begin(), data_str.end()); _owner->on_inbound_chunk_response(cr.request_id(), cr.found(), std::move(data_vec)); + } else if (in.has_snapshot_request()) { + _owner->on_inbound_snapshot_request(conn.get(), in.snapshot_request().cluster_id(), + in.snapshot_request().path_prefix(), + in.snapshot_request().request_id()); + } else if (in.has_snapshot_response()) { + const auto &sr = in.snapshot_response(); + std::vector entries; + entries.reserve(sr.entries_size()); + for (const auto &e : sr.entries()) { + state_transport::snapshot_entry se; + se.path = e.path(); + se.string_value = e.string_value(); + se.comment = e.comment(); + se.hidden = e.hidden(); + se.read_only = e.read_only(); + se.type_name = e.type_name(); + se.origin_node_id = e.origin_node_id(); + se.sequence = e.sequence(); + entries.push_back(std::move(se)); + } + _owner->on_inbound_snapshot_response(sr.request_id(), entries, sr.final()); + } else if (in.has_heartbeat()) { + _owner->on_inbound_heartbeat(in.heartbeat().node_id(), in.heartbeat().cluster_id()); } } @@ -344,6 +367,19 @@ void state_transport_grpc::start(const std::string &listen_addr, const std::stri _impl->listen_addr_resolved = listen_addr; _running.store(true); + + // Start heartbeat sender if configured. + if (_heartbeat_interval.count() > 0) { + _heartbeat_thread = std::thread([this]() { heartbeat_loop(); }); + } +} + +void state_transport_grpc::set_heartbeat_interval(std::chrono::milliseconds interval) noexcept { + _heartbeat_interval = interval; +} + +std::chrono::milliseconds state_transport_grpc::heartbeat_interval() const noexcept { + return _heartbeat_interval; } std::string state_transport_grpc::listen_address() const { @@ -430,6 +466,29 @@ bool state_transport_grpc::connect_to_peer(const std::string &target, std::string data_str = cr.data(); std::vector data_vec(data_str.begin(), data_str.end()); on_inbound_chunk_response(cr.request_id(), cr.found(), std::move(data_vec)); + } else if (in.has_snapshot_request()) { + on_inbound_snapshot_request(conn.get(), in.snapshot_request().cluster_id(), + in.snapshot_request().path_prefix(), + in.snapshot_request().request_id()); + } else if (in.has_snapshot_response()) { + const auto &sr = in.snapshot_response(); + std::vector entries; + entries.reserve(sr.entries_size()); + for (const auto &e : sr.entries()) { + snapshot_entry se; + se.path = e.path(); + se.string_value = e.string_value(); + se.comment = e.comment(); + se.hidden = e.hidden(); + se.read_only = e.read_only(); + se.type_name = e.type_name(); + se.origin_node_id = e.origin_node_id(); + se.sequence = e.sequence(); + entries.push_back(std::move(se)); + } + on_inbound_snapshot_response(sr.request_id(), entries, sr.final()); + } else if (in.has_heartbeat()) { + on_inbound_heartbeat(in.heartbeat().node_id(), in.heartbeat().cluster_id()); } } conn->alive.store(false); @@ -462,6 +521,9 @@ void state_transport_grpc::stop() { _impl->server.reset(); } + if (_heartbeat_thread.joinable()) + _heartbeat_thread.join(); + for (auto &c : conns) { if (c->client_reader.joinable()) c->client_reader.join(); @@ -824,4 +886,162 @@ bool state_transport_grpc::fetch_chunk(const std::string &digest, chunk_callback return result; } +void state_transport_grpc::on_inbound_snapshot_request(connection *conn, + const std::string &cluster_id, + const std::string &path_prefix, + std::uint64_t request_id) { + // Gather entries from all local shards matching the cluster. + std::vector peers; + { + std::lock_guard lk(_shards_mu); + for (auto *s : _shards) + if (s && s->cluster_id() == cluster_id) + peers.push_back(s); + } + + std::vector all; + for (auto *peer : peers) { + auto snap = peer->snapshot(path_prefix); + for (auto &se : snap) { + snapshot_entry te; + te.path = std::move(se.path); + te.string_value = std::move(se.string_value); + te.comment = std::move(se.comment); + te.hidden = se.hidden; + te.read_only = se.read_only; + te.type_name = std::move(se.type_name); + te.origin_node_id = std::move(se.origin_node_id); + te.sequence = se.sequence; + all.push_back(std::move(te)); + } + } + + // Build response frame. + pb::Frame f; + auto *rsp = f.mutable_snapshot_response(); + rsp->set_request_id(request_id); + rsp->set_final(true); + for (const auto &e : all) { + auto *pe = rsp->add_entries(); + pe->set_path(e.path); + pe->set_string_value(e.string_value); + pe->set_comment(e.comment); + pe->set_hidden(e.hidden); + pe->set_read_only(e.read_only); + pe->set_type_name(e.type_name); + pe->set_origin_node_id(e.origin_node_id); + pe->set_sequence(e.sequence); + } + + if (conn && conn->alive.load() && conn->write_fn) { + conn->write_fn(f); + _sent_frames.fetch_add(1, std::memory_order_relaxed); + } +} + +void state_transport_grpc::on_inbound_snapshot_response( + std::uint64_t request_id, const std::vector &entries, bool final) { + { + std::lock_guard lk(_snap_waiters_mu); + auto it = _snap_waiters.find(request_id); + if (it != _snap_waiters.end()) { + for (const auto &e : entries) + it->second->entries.push_back(e); + if (final) + it->second->done = true; + } + } + if (final) + _snap_waiters_cv.notify_all(); +} + +void state_transport_grpc::on_inbound_heartbeat(const std::string &node_id, + const std::string & /*cluster_id*/) { + _peers.note_seen(node_id); +} + +bool state_transport_grpc::request_snapshot(const std::string &cluster_id, + const std::string &path_prefix, + snapshot_callback on_entries) { + std::uint64_t req_id = _next_snap_req_id.fetch_add(1, std::memory_order_relaxed); + auto waiter = std::make_shared(); + { + std::lock_guard lk(_snap_waiters_mu); + _snap_waiters[req_id] = waiter; + } + + pb::Frame frame; + auto *req = frame.mutable_snapshot_request(); + req->set_cluster_id(cluster_id); + req->set_path_prefix(path_prefix); + req->set_request_id(req_id); + + // Send to first alive connection. + std::vector> conns; + { + std::lock_guard lk(_conns_mu); + conns = _conns; + } + std::size_t sent = 0; + for (auto &c : conns) { + if (!c || !c->alive.load() || !c->write_fn) + continue; + if (c->write_fn(frame)) { + _sent_frames.fetch_add(1, std::memory_order_relaxed); + ++sent; + break; // only need one peer + } + } + + bool result = false; + if (sent > 0) { + std::unique_lock lk(_snap_waiters_mu); + _snap_waiters_cv.wait_for(lk, std::chrono::seconds(30), + [&]() { return waiter->done; }); + if (waiter->done) { + if (on_entries) + on_entries(waiter->entries, true); + result = true; + } + } + + { + std::lock_guard lk(_snap_waiters_mu); + _snap_waiters.erase(req_id); + } + return result; +} + +void state_transport_grpc::heartbeat_loop() { + while (_running.load()) { + auto sleep_end = std::chrono::steady_clock::now() + _heartbeat_interval; + while (_running.load() && std::chrono::steady_clock::now() < sleep_end) + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + if (!_running.load()) + break; + + pb::Frame frame; + auto *hb = frame.mutable_heartbeat(); + hb->set_node_id(_node_id); + hb->set_cluster_id(_cluster_id); + hb->set_timestamp_ns( + static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count())); + + std::vector> conns; + { + std::lock_guard lk(_conns_mu); + conns = _conns; + } + for (auto &c : conns) { + if (!c || !c->alive.load() || !c->write_fn) + continue; + if (c->write_fn(frame)) + _sent_frames.fetch_add(1, std::memory_order_relaxed); + } + } +} + } // namespace CVC_NAMESPACE diff --git a/src/cvc/state_transport_ipc.cpp b/src/cvc/state_transport_ipc.cpp index f262135..f751b94 100644 --- a/src/cvc/state_transport_ipc.cpp +++ b/src/cvc/state_transport_ipc.cpp @@ -30,6 +30,7 @@ namespace { constexpr std::uint32_t kMagic = 0x43564354u; // 'CVCT' constexpr std::uint16_t kVersion = 1; +constexpr std::uint16_t kMaxVersion = 1; // highest version we understand constexpr std::uint16_t kMsgHello = 1; constexpr std::uint16_t kMsgMutation = 2; constexpr std::uint16_t kMsgOob = 3; @@ -485,7 +486,7 @@ void state_transport_ipc::reader_loop(std::shared_ptr conn) { std::uint16_t version = hr.u16(); std::uint16_t mtype = hr.u16(); std::uint32_t len = hr.u32(); - if (magic != kMagic || version != kVersion || len > kMaxFrameBytes) + if (magic != kMagic || version > kMaxVersion || len > kMaxFrameBytes) break; std::vector body(len); @@ -494,6 +495,12 @@ void state_transport_ipc::reader_loop(std::shared_ptr conn) { _recv_frames.fetch_add(1, std::memory_order_relaxed); + // Skip unknown message types instead of disconnecting, so that + // older nodes can coexist with newer nodes that send new frame + // types. + if (mtype > kMsgSnapRsp && mtype != kMsgHello) + continue; + if (mtype == kMsgHello) { reader br(body.data(), body.size()); conn->remote_node_id = br.str(); diff --git a/src/cvc/tests/CMakeLists.txt b/src/cvc/tests/CMakeLists.txt index f9dbab4..dddb51e 100644 --- a/src/cvc/tests/CMakeLists.txt +++ b/src/cvc/tests/CMakeLists.txt @@ -48,6 +48,15 @@ add_executable(state_brick_manifest_test state_brick_manifest_test.cpp) add_executable(state_data_hydrator_test state_data_hydrator_test.cpp) add_executable(distributed_state_session_test distributed_state_session_test.cpp) add_executable(state_large_tree_bench_test state_large_tree_bench_test.cpp) +add_executable(state_hash_partition_test state_hash_partition_test.cpp) +add_executable(state_hybrid_time_test state_hybrid_time_test.cpp) +add_executable(state_brick_frustum_test state_brick_frustum_test.cpp) +add_executable(state_conflict_ring_test state_conflict_ring_test.cpp) +add_executable(state_zstd_round_trip_test state_zstd_round_trip_test.cpp) +add_executable(state_session_status_test state_session_status_test.cpp) +if(NOT WIN32) + add_executable(state_snapshot_protocol_test state_snapshot_protocol_test.cpp) +endif() if(CVC_ENABLE_GRPC) add_executable(state_transport_grpc_test state_transport_grpc_test.cpp) endif() @@ -481,6 +490,57 @@ target_link_libraries(state_large_tree_bench_test GTest::gtest_main ) +target_link_libraries(state_hash_partition_test + PRIVATE + cvc + GTest::gtest + GTest::gtest_main +) + +target_link_libraries(state_hybrid_time_test + PRIVATE + cvc + GTest::gtest + GTest::gtest_main +) + +target_link_libraries(state_brick_frustum_test + PRIVATE + cvc + GTest::gtest + GTest::gtest_main +) + +target_link_libraries(state_conflict_ring_test + PRIVATE + cvc + GTest::gtest + GTest::gtest_main +) + +target_link_libraries(state_zstd_round_trip_test + PRIVATE + cvc + GTest::gtest + GTest::gtest_main +) + +target_link_libraries(state_session_status_test + PRIVATE + cvc + GTest::gtest + GTest::gtest_main +) + +if(NOT WIN32) + target_link_libraries(state_snapshot_protocol_test + PRIVATE + cvc + GTest::gtest + GTest::gtest_main + ) +endif() + if(TARGET state_transport_grpc_test) target_link_libraries(state_transport_grpc_test PRIVATE @@ -547,6 +607,15 @@ target_compile_features(state_brick_manifest_test PRIVATE cxx_std_17) target_compile_features(state_data_hydrator_test PRIVATE cxx_std_17) target_compile_features(distributed_state_session_test PRIVATE cxx_std_17) target_compile_features(state_large_tree_bench_test PRIVATE cxx_std_17) +target_compile_features(state_hash_partition_test PRIVATE cxx_std_17) +target_compile_features(state_hybrid_time_test PRIVATE cxx_std_17) +target_compile_features(state_brick_frustum_test PRIVATE cxx_std_17) +target_compile_features(state_conflict_ring_test PRIVATE cxx_std_17) +target_compile_features(state_zstd_round_trip_test PRIVATE cxx_std_17) +target_compile_features(state_session_status_test PRIVATE cxx_std_17) +if(NOT WIN32) + target_compile_features(state_snapshot_protocol_test PRIVATE cxx_std_17) +endif() # Only build HDF5-dependent tests when HDF5 support is enabled if(CVC_USING_HDF5 AND NOT CVC_HDF5_DISABLED) @@ -644,6 +713,15 @@ gtest_discover_tests(state_brick_manifest_test) gtest_discover_tests(state_data_hydrator_test) gtest_discover_tests(distributed_state_session_test) gtest_discover_tests(state_large_tree_bench_test) +gtest_discover_tests(state_hash_partition_test) +gtest_discover_tests(state_hybrid_time_test) +gtest_discover_tests(state_brick_frustum_test) +gtest_discover_tests(state_conflict_ring_test) +gtest_discover_tests(state_zstd_round_trip_test) +gtest_discover_tests(state_session_status_test) +if(NOT WIN32) + gtest_discover_tests(state_snapshot_protocol_test) +endif() if(TARGET state_transport_grpc_test) gtest_discover_tests(state_transport_grpc_test) endif() diff --git a/src/cvc/tests/state_brick_frustum_test.cpp b/src/cvc/tests/state_brick_frustum_test.cpp new file mode 100644 index 0000000..3382a22 --- /dev/null +++ b/src/cvc/tests/state_brick_frustum_test.cpp @@ -0,0 +1,104 @@ +/* + Copyright 2026 The University of Texas at Austin + + This file is part of libcvc. + + libcvc is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. +*/ + +#include +#include + +using cvc::brick_extent; +using cvc::state_brick_manifest; + +class StateBrickFrustumTest : public ::testing::Test { +protected: + state_brick_manifest manifest; + + void SetUp() override { + // Create a 3x3x1 grid of bricks each 10x10x10 voxels. + for (int x = 0; x < 3; ++x) { + for (int y = 0; y < 3; ++y) { + brick_extent e; + e.origin_x = x * 10; + e.origin_y = y * 10; + e.origin_z = 0; + e.size_x = 10; + e.size_y = 10; + e.size_z = 10; + manifest.chunks.push_back("chunk_" + std::to_string(x) + "_" + std::to_string(y)); + manifest.chunk_bytes.push_back(1000); + manifest.extents.push_back(e); + } + } + } +}; + +TEST_F(StateBrickFrustumTest, AllPlanesPassingReturnsAll) { + // Planes that encompass everything: each plane's positive half-space + // includes the origin and extends far. Use a giant box frustum. + state_brick_manifest::plane planes[6] = { + {1, 0, 0, 100}, // x >= -100 + {-1, 0, 0, 100}, // -x >= -100 i.e. x <= 100 + {0, 1, 0, 100}, // y >= -100 + {0, -1, 0, 100}, // y <= 100 + {0, 0, 1, 100}, // z >= -100 + {0, 0, -1, 100}, // z <= 100 + }; + auto result = manifest.bricks_in_frustum(planes); + EXPECT_EQ(result.size(), 9u); +} + +TEST_F(StateBrickFrustumTest, TightFrustumSelectsSubset) { + // Planes that only include the first brick (0..10, 0..10, 0..10). + state_brick_manifest::plane planes[6] = { + {1, 0, 0, 0}, // x >= 0 + {-1, 0, 0, 10}, // x <= 10 + {0, 1, 0, 0}, // y >= 0 + {0, -1, 0, 10}, // y <= 10 + {0, 0, 1, 0}, // z >= 0 + {0, 0, -1, 10}, // z <= 10 + }; + auto result = manifest.bricks_in_frustum(planes); + // At least the first brick should be included. + EXPECT_GE(result.size(), 1u); + EXPECT_LE(result.size(), 9u); + // The result should include brick index 0. + EXPECT_NE(std::find(result.begin(), result.end(), 0u), result.end()); +} + +TEST_F(StateBrickFrustumTest, FrustumFarAwayReturnsEmpty) { + // Planes that are far from any brick (everything at x > 1000). + state_brick_manifest::plane planes[6] = { + {1, 0, 0, -1000}, // x >= 1000 + {-1, 0, 0, 2000}, // x <= 2000 + {0, 1, 0, -1000}, // y >= 1000 + {0, -1, 0, 2000}, // y <= 2000 + {0, 0, 1, -1000}, // z >= 1000 + {0, 0, -1, 2000}, // z <= 2000 + }; + auto result = manifest.bricks_in_frustum(planes); + EXPECT_EQ(result.size(), 0u); +} + +TEST_F(StateBrickFrustumTest, EmptyManifestReturnsEmpty) { + state_brick_manifest empty; + state_brick_manifest::plane planes[6] = { + {1, 0, 0, 100}, {-1, 0, 0, 100}, {0, 1, 0, 100}, + {0, -1, 0, 100}, {0, 0, 1, 100}, {0, 0, -1, 100}, + }; + auto result = empty.bricks_in_frustum(planes); + EXPECT_EQ(result.size(), 0u); +} + +TEST_F(StateBrickFrustumTest, RegionQueryBaseline) { + // Also test bricks_in_region as a baseline sanity check. + auto result = manifest.bricks_in_region(0, 0, 0, 30, 30, 10); + EXPECT_EQ(result.size(), 9u); + + auto subset = manifest.bricks_in_region(0, 0, 0, 10, 10, 10); + EXPECT_EQ(subset.size(), 1u); +} diff --git a/src/cvc/tests/state_conflict_ring_test.cpp b/src/cvc/tests/state_conflict_ring_test.cpp new file mode 100644 index 0000000..579fb8b --- /dev/null +++ b/src/cvc/tests/state_conflict_ring_test.cpp @@ -0,0 +1,135 @@ +/* + Copyright 2026 The University of Texas at Austin + + This file is part of libcvc. + + libcvc is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. +*/ + +#include +#include +#include +#include +#include +#include + +using cvc::app; +using cvc::state_cluster_shard; +using cvc::state_distributed_metrics; +using cvc::state_mutation; +using cvc::state_mutation_op; +using cvc::state_transport_inproc; + +class StateConflictRingTest : public ::testing::Test { +protected: + app ctx; + state_transport_inproc transport; + state_cluster_shard shard_a{ctx, "cluster", "node_A"}; + state_cluster_shard shard_b{ctx, "cluster", "node_B"}; + + void SetUp() override { + shard_a.set_resolve_conflicts(true); + transport.register_shard(&shard_a); + transport.register_shard(&shard_b); + shard_a.attach(); + shard_b.attach(); + } + + void TearDown() override { + shard_a.detach(); + shard_b.detach(); + transport.unregister_shard(&shard_a); + transport.unregister_shard(&shard_b); + } + + state_mutation make_mutation(const std::string &node, std::uint64_t seq, + const std::string &path, const std::string &value) { + state_mutation m; + m.cluster_id = "cluster"; + m.origin_node_id = node; + m.sequence = seq; + m.path = path; + m.op = state_mutation_op::set_value; + m.string_value = value; + return m; + } +}; + +TEST_F(StateConflictRingTest, NoConflictsWhenNoCompetingNodes) { + auto m = make_mutation("node_B", 1, "x.y", "v1"); + shard_a.ingest_remote(m); + auto conflicts = shard_a.recent_conflicts(); + EXPECT_TRUE(conflicts.empty()); +} + +TEST_F(StateConflictRingTest, ConflictRecordedWhenTwoNodesCompete) { + // First write wins, second loses deterministically. + auto m1 = make_mutation("node_B", 1, "x.y", "v1"); + auto r1 = shard_a.ingest_remote(m1); + EXPECT_TRUE(r1.applied); + + auto m2 = make_mutation("node_C", 1, "x.y", "v2"); + auto r2 = shard_a.ingest_remote(m2); + // A conflict was detected. Whether the ring recorded it depends + // on whether m2 lost the tie-break (ring only captures the + // losing mutation). + EXPECT_GE(shard_a.total_conflicts_detected(), 1u); + + // At least one of (conflicts_lost, applied) should be true for m2. + auto conflicts = shard_a.recent_conflicts(); + if (shard_a.total_conflicts_lost() > 0) { + EXPECT_GE(conflicts.size(), 1u); + EXPECT_EQ(conflicts[0].path, "x.y"); + } +} + +TEST_F(StateConflictRingTest, ConflictsRingWrapsAround) { + // Generate more conflicts than the ring size. + // Use alphabetically lower node ids to guarantee loss. + for (int i = 0; i < 200; ++i) { + // Winner: node_C > node_B lexicographically. + auto ma = make_mutation("node_C", static_cast(i + 1), + "path." + std::to_string(i), "c"); + shard_a.ingest_remote(ma); + // Loser: node_B < node_C, same path, so should_replace returns false. + auto mc = make_mutation("node_B", static_cast(i + 1), + "path." + std::to_string(i), "b"); + shard_a.ingest_remote(mc); + } + + // Should return at most 128 (ring size), and they should be the most recent. + auto conflicts = shard_a.recent_conflicts(); + EXPECT_LE(conflicts.size(), 128u); + EXPECT_GT(conflicts.size(), 0u); +} + +TEST_F(StateConflictRingTest, RecentConflictsMaxEntries) { + for (int i = 0; i < 10; ++i) { + // Winner first (higher node id). + auto ma = make_mutation("node_C", static_cast(i + 1), + "p." + std::to_string(i), "c"); + shard_a.ingest_remote(ma); + // Loser: node_B < node_C for the same path. + auto mc = make_mutation("node_B", static_cast(i + 1), + "p." + std::to_string(i), "b"); + shard_a.ingest_remote(mc); + } + + // Request only 3 entries. + auto conflicts = shard_a.recent_conflicts(3); + EXPECT_LE(conflicts.size(), 3u); +} + +TEST_F(StateConflictRingTest, PublishConflictsWritesToStateTree) { + // Winner first. + auto m1 = make_mutation("node_C", 1, "x", "v1"); + shard_a.ingest_remote(m1); + // Loser. + auto m2 = make_mutation("node_B", 1, "x", "v2"); + shard_a.ingest_remote(m2); + + auto count = state_distributed_metrics::publish_conflicts(ctx, shard_a); + EXPECT_GE(count, 1u); +} diff --git a/src/cvc/tests/state_hash_partition_test.cpp b/src/cvc/tests/state_hash_partition_test.cpp new file mode 100644 index 0000000..a650c72 --- /dev/null +++ b/src/cvc/tests/state_hash_partition_test.cpp @@ -0,0 +1,127 @@ +/* + Copyright 2026 The University of Texas at Austin + + This file is part of libcvc. + + libcvc is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. +*/ + +#include +#include +#include +#include +#include + +using cvc::state_hash_partition; + +class StateHashPartitionTest : public ::testing::Test { +protected: + state_hash_partition part; +}; + +TEST_F(StateHashPartitionTest, EmptyMapReturnsNoOwner) { + EXPECT_TRUE(part.owner_of("foo.bar").empty()); + EXPECT_FALSE(part.owns("node-1", "foo.bar")); + EXPECT_EQ(part.size(), 0u); +} + +TEST_F(StateHashPartitionTest, AssignAndLookup) { + part.assign("node-1", 0, UINT32_MAX / 2); + part.assign("node-2", UINT32_MAX / 2, UINT32_MAX); + EXPECT_EQ(part.size(), 2u); + + // Every path should resolve to either node-1 or node-2. + for (int i = 0; i < 50; ++i) { + std::string path = "test.path." + std::to_string(i); + std::string owner = part.owner_of(path); + EXPECT_TRUE(owner == "node-1" || owner == "node-2") + << "path=" << path << " owner=" << owner; + } +} + +TEST_F(StateHashPartitionTest, UniformPartition) { + std::vector nodes = {"A", "B", "C", "D"}; + part.assign_uniform(nodes); + EXPECT_EQ(part.size(), 4u); + + // Check that each node gets at least some paths. + std::map counts; + for (int i = 0; i < 1000; ++i) { + std::string path = "path." + std::to_string(i); + counts[part.owner_of(path)]++; + } + for (const auto &n : nodes) { + EXPECT_GT(counts[n], 0) << "node " << n << " got zero paths"; + } +} + +TEST_F(StateHashPartitionTest, OwnsMatchesOwnerOf) { + part.assign_uniform({"x", "y"}); + for (int i = 0; i < 100; ++i) { + std::string path = "p." + std::to_string(i); + std::string owner = part.owner_of(path); + EXPECT_TRUE(part.owns(owner, path)); + // The other node should NOT own it. + std::string other = (owner == "x") ? "y" : "x"; + EXPECT_FALSE(part.owns(other, path)); + } +} + +TEST_F(StateHashPartitionTest, SnapshotReturnsRanges) { + part.assign("n1", 0, 100); + part.assign("n2", 100, 200); + auto snap = part.snapshot(); + EXPECT_EQ(snap.size(), 2u); + EXPECT_EQ(snap[0].node_id, "n1"); + EXPECT_EQ(snap[0].range_begin, 0u); + EXPECT_EQ(snap[0].range_end, 100u); + EXPECT_EQ(snap[1].node_id, "n2"); +} + +TEST_F(StateHashPartitionTest, ClearRemovesAllRanges) { + part.assign_uniform({"a", "b", "c"}); + EXPECT_EQ(part.size(), 3u); + part.clear(); + EXPECT_EQ(part.size(), 0u); + EXPECT_TRUE(part.owner_of("test").empty()); +} + +TEST_F(StateHashPartitionTest, HashIsDeterministic) { + auto h1 = state_hash_partition::hash("foo.bar"); + auto h2 = state_hash_partition::hash("foo.bar"); + EXPECT_EQ(h1, h2); + // Different paths should (very likely) hash differently. + auto h3 = state_hash_partition::hash("baz.qux"); + EXPECT_NE(h1, h3); +} + +TEST_F(StateHashPartitionTest, SingleNodeOwnsEverything) { + part.assign_uniform({"solo"}); + for (int i = 0; i < 100; ++i) { + EXPECT_EQ(part.owner_of("path." + std::to_string(i)), "solo"); + } +} + +TEST_F(StateHashPartitionTest, ConcurrentAccessIsSafe) { + part.assign_uniform({"A", "B", "C"}); + std::atomic stop{false}; + auto reader = [&]() { + while (!stop.load()) { + (void)part.owner_of("test.concurrent"); + (void)part.size(); + } + }; + auto writer = [&]() { + for (int i = 0; i < 100 && !stop.load(); ++i) { + part.assign_uniform({"X", "Y"}); + part.assign_uniform({"A", "B", "C"}); + } + }; + std::thread t1(reader), t2(reader), t3(writer); + t3.join(); + stop.store(true); + t1.join(); + t2.join(); +} diff --git a/src/cvc/tests/state_hybrid_time_test.cpp b/src/cvc/tests/state_hybrid_time_test.cpp new file mode 100644 index 0000000..7c57e9e --- /dev/null +++ b/src/cvc/tests/state_hybrid_time_test.cpp @@ -0,0 +1,135 @@ +/* + Copyright 2026 The University of Texas at Austin + + This file is part of libcvc. + + libcvc is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. +*/ + +#include +#include +#include +#include + +using cvc::hybrid_clock; +using cvc::hybrid_time; + +// --------------------------------------------------------------- +// hybrid_time struct tests +// --------------------------------------------------------------- + +TEST(HybridTimeTest, DefaultIsZero) { + hybrid_time t; + EXPECT_EQ(t.wall_ms, 0u); + EXPECT_EQ(t.logical, 0u); + EXPECT_EQ(t.packed(), 0u); + EXPECT_FALSE(static_cast(t)); +} + +TEST(HybridTimeTest, PackUnpackRoundTrip) { + hybrid_time t{123456789, 42}; + auto packed = t.packed(); + auto t2 = hybrid_time::from_packed(packed); + EXPECT_EQ(t2.wall_ms, t.wall_ms); + EXPECT_EQ(t2.logical, t.logical); +} + +TEST(HybridTimeTest, PackedOrderIsCorrect) { + hybrid_time a{100, 0}; + hybrid_time b{100, 1}; + hybrid_time c{101, 0}; + EXPECT_LT(a, b); + EXPECT_LT(b, c); + EXPECT_GT(c, a); +} + +TEST(HybridTimeTest, ComparisonOperators) { + hybrid_time a{10, 5}; + hybrid_time b{10, 5}; + hybrid_time c{10, 6}; + EXPECT_EQ(a, b); + EXPECT_NE(a, c); + EXPECT_LE(a, b); + EXPECT_GE(a, b); + EXPECT_LT(a, c); + EXPECT_GT(c, a); +} + +TEST(HybridTimeTest, BoolConversionNonZero) { + hybrid_time t{1, 0}; + EXPECT_TRUE(static_cast(t)); +} + +TEST(HybridTimeTest, MaxLogicalValue) { + hybrid_time t{1, 0xFFFF}; + auto packed = t.packed(); + auto t2 = hybrid_time::from_packed(packed); + EXPECT_EQ(t2.wall_ms, 1u); + EXPECT_EQ(t2.logical, 0xFFFF); +} + +// --------------------------------------------------------------- +// hybrid_clock tests +// --------------------------------------------------------------- + +TEST(HybridClockTest, NowIsMonotonicallyIncreasing) { + hybrid_clock clk; + auto t1 = clk.now(); + auto t2 = clk.now(); + auto t3 = clk.now(); + EXPECT_LT(t1, t2); + EXPECT_LT(t2, t3); +} + +TEST(HybridClockTest, CurrentDoesNotAdvance) { + hybrid_clock clk; + auto t = clk.now(); + auto c1 = clk.current(); + auto c2 = clk.current(); + EXPECT_EQ(c1, c2); + EXPECT_EQ(c1, t); +} + +TEST(HybridClockTest, UpdateMergesRemote) { + hybrid_clock clk; + auto local = clk.now(); + + // Simulate a remote timestamp far in the future. + hybrid_time remote{local.wall_ms + 10000, 5}; + auto merged = clk.update(remote); + + // Merged should be strictly after both local and remote. + EXPECT_GT(merged, local); + EXPECT_GT(merged, remote); +} + +TEST(HybridClockTest, UpdateNeverGoesBackward) { + hybrid_clock clk; + auto t1 = clk.now(); + + // Update with an old timestamp. + hybrid_time old{1, 0}; + auto t2 = clk.update(old); + EXPECT_GT(t2, t1); +} + +TEST(HybridClockTest, ConcurrentNowIsSafe) { + hybrid_clock clk; + std::vector results(1000); + auto worker = [&](int start, int count) { + for (int i = start; i < start + count; ++i) + results[i] = clk.now(); + }; + std::thread t1(worker, 0, 500); + std::thread t2(worker, 500, 500); + t1.join(); + t2.join(); + + // All timestamps should be unique. + std::set packed_set; + for (const auto &t : results) + packed_set.insert(t.packed()); + EXPECT_EQ(packed_set.size(), results.size()); +} diff --git a/src/cvc/tests/state_session_status_test.cpp b/src/cvc/tests/state_session_status_test.cpp new file mode 100644 index 0000000..b55cdb0 --- /dev/null +++ b/src/cvc/tests/state_session_status_test.cpp @@ -0,0 +1,102 @@ +/* + Copyright 2026 The University of Texas at Austin + + This file is part of libcvc. + + libcvc is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. +*/ + +#include +#include +#include +#include +#include +#include + +using cvc::app; +using cvc::distributed_state_config; +using cvc::distributed_state_session; +using cvc::replica_status; +using cvc::transport_kind; + +class StateSessionStatusTest : public ::testing::Test { +protected: + app ctx; + + std::shared_ptr make_session() { + distributed_state_config config; + config.cluster_id = "test_cluster"; + config.node_id = "node_1"; + config.transport = transport_kind::inproc; + config.pump_interval_ms = 5; + return distributed_state_session::join(ctx, config); + } +}; + +TEST_F(StateSessionStatusTest, StatusReportsRunning) { + auto session = make_session(); + auto s = session->status(); + EXPECT_TRUE(s.running); + session->stop(); + auto s2 = session->status(); + EXPECT_FALSE(s2.running); +} + +TEST_F(StateSessionStatusTest, StatusReportsPumpCycles) { + auto session = make_session(); + // Wait for at least one pump cycle. + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + auto s = session->status(); + EXPECT_GT(s.pump_cycles, 0u); + session->stop(); +} + +TEST_F(StateSessionStatusTest, StatusReportsLocalSequence) { + auto session = make_session(); + // First set creates the node (lost per adapter doc), second set + // generates a journaled mutation. + cvc::state::instance(ctx)("test.key").value(std::string("v1")); + cvc::state::instance(ctx)("test.key").value(std::string("v2")); + // Force a pump cycle to ensure drain_local has run. + session->transport().pump_all(); + auto s = session->status(); + EXPECT_GT(s.local_sequence, 0u); + session->stop(); +} + +TEST_F(StateSessionStatusTest, SessionStopIsIdempotent) { + auto session = make_session(); + session->stop(); + session->stop(); // should not crash + EXPECT_FALSE(session->is_running()); +} + +TEST_F(StateSessionStatusTest, WaitForDataTimeoutReturns) { + auto session = make_session(); + // Wait for a non-existent path — should return after timeout. + auto result = session->wait_for_data("nonexistent.path", + std::chrono::milliseconds(50)); + // We just verify it returns without hanging. The status value + // depends on whether the hydrator has seen this path. + (void)result; + session->stop(); +} + +TEST_F(StateSessionStatusTest, ClusterAndNodeIdAccessors) { + auto session = make_session(); + EXPECT_EQ(session->cluster_id(), "test_cluster"); + EXPECT_EQ(session->node_id(), "node_1"); + session->stop(); +} + +TEST_F(StateSessionStatusTest, ShardAndTransportAccessible) { + auto session = make_session(); + // These should not throw or crash. + auto &shard = session->shard(); + EXPECT_EQ(shard.cluster_id(), "test_cluster"); + auto &transport = session->transport(); + (void)transport; + session->stop(); +} diff --git a/src/cvc/tests/state_snapshot_protocol_test.cpp b/src/cvc/tests/state_snapshot_protocol_test.cpp new file mode 100644 index 0000000..9dcde82 --- /dev/null +++ b/src/cvc/tests/state_snapshot_protocol_test.cpp @@ -0,0 +1,141 @@ +/* + Copyright 2026 The University of Texas at Austin + + This file is part of libcvc. + + libcvc is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// --------------------------------------------------------------- +// Snapshot protocol tests over IPC transport. +// --------------------------------------------------------------- + +namespace { + +std::string make_socket_path(const std::string &label) { + return (std::filesystem::temp_directory_path() / ("cvc_snap_test_" + label + ".sock")).string(); +} + +} // namespace + +class StateSnapshotProtocolTest : public ::testing::Test { +protected: + void TearDown() override { + // Clean up socket files. + for (const auto &p : sockets) + std::filesystem::remove(p); + } + std::vector sockets; +}; + +TEST_F(StateSnapshotProtocolTest, SnapshotFromLocalShard) { + cvc::app ctx; + cvc::state_cluster_shard shard(ctx, "clust", "node_A"); + shard.attach(); + + // Write some state. + cvc::state::instance(ctx)("a.b").value(std::string("hello")); + cvc::state::instance(ctx)("a.c").value(std::string("world")); + + auto snap = shard.snapshot(); + EXPECT_GE(snap.size(), 2u); + + bool found_b = false, found_c = false; + for (const auto &e : snap) { + if (e.path == "a.b" && e.string_value == "hello") found_b = true; + if (e.path == "a.c" && e.string_value == "world") found_c = true; + } + EXPECT_TRUE(found_b) << "a.b not in snapshot"; + EXPECT_TRUE(found_c) << "a.c not in snapshot"; + + shard.detach(); +} + +TEST_F(StateSnapshotProtocolTest, SnapshotWithPrefixFilter) { + cvc::app ctx; + cvc::state_cluster_shard shard(ctx, "clust", "node_A"); + shard.attach(); + + cvc::state::instance(ctx)("foo.x").value(std::string("1")); + cvc::state::instance(ctx)("bar.y").value(std::string("2")); + + auto snap = shard.snapshot("foo"); + bool found_foo = false, found_bar = false; + for (const auto &e : snap) { + if (e.path.find("foo") == 0) found_foo = true; + if (e.path.find("bar") == 0) found_bar = true; + } + EXPECT_TRUE(found_foo); + EXPECT_FALSE(found_bar); + + shard.detach(); +} + +TEST_F(StateSnapshotProtocolTest, IpcSnapshotRequestResponse) { + std::string sock = make_socket_path("snap_ipc"); + sockets.push_back(sock); + + cvc::app ctx_a, ctx_b; + + // Server: node_A with data. + cvc::state_transport_ipc server; + cvc::state_cluster_shard shard_a(ctx_a, "clust", "node_A"); + shard_a.attach(); + cvc::state::instance(ctx_a)("data.x").value(std::string("42")); + cvc::state::instance(ctx_a)("data.y").value(std::string("99")); + server.register_shard(&shard_a); + server.start(sock, "node_A", "clust"); + + // Client: node_B, empty. Must call start() so _running is set. + std::string sock_b = make_socket_path("snap_ipc_b"); + sockets.push_back(sock_b); + cvc::state_transport_ipc client; + cvc::state_cluster_shard shard_b(ctx_b, "clust", "node_B"); + shard_b.attach(); + client.register_shard(&shard_b); + client.start(sock_b, "node_B", "clust"); + ASSERT_TRUE(client.connect_to_peer(sock)); + + // Wait for handshake + reader thread to be established. + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + + // Verify connections are up. + ASSERT_GE(client.connection_count(), 1u) << "client has no connections"; + ASSERT_GE(server.connection_count(), 1u) << "server has no connections"; + + // Request snapshot. + std::vector entries; + bool got_snapshot = client.request_snapshot( + "clust", "", + [&](const std::vector &e, bool /*final*/) { + entries = e; + }); + + EXPECT_TRUE(got_snapshot); + EXPECT_GE(entries.size(), 2u); + + bool found_x = false, found_y = false; + for (const auto &e : entries) { + if (e.path == "data.x" && e.string_value == "42") found_x = true; + if (e.path == "data.y" && e.string_value == "99") found_y = true; + } + EXPECT_TRUE(found_x); + EXPECT_TRUE(found_y); + + client.stop(); + server.stop(); + shard_a.detach(); + shard_b.detach(); +} diff --git a/src/cvc/tests/state_transport_grpc_test.cpp b/src/cvc/tests/state_transport_grpc_test.cpp index cf590dd..43d1867 100644 --- a/src/cvc/tests/state_transport_grpc_test.cpp +++ b/src/cvc/tests/state_transport_grpc_test.cpp @@ -656,3 +656,112 @@ TEST(StateTransportGrpcPhase5, TlsHandshakeRoundTrip) { tA.stop(); tB.stop(); } + +// --------------------------------------------------------------- +// Reconnect / resilience tests for the gRPC transport. +// --------------------------------------------------------------- + +TEST(StateTransportGrpcReconnectTest, StopRestartReconnect) { + cvc::app aA, aB; + + auto tA = std::make_unique(); + cvc::state_transport_grpc tB; + tA->start("127.0.0.1:0", "A", "C"); + tB.start("127.0.0.1:0", "B", "C"); + auto addrB = tB.listen_address(); + ASSERT_TRUE(tA->connect_to_peer(addrB, std::chrono::milliseconds(2000))); + ASSERT_TRUE(wait_connected(*tA, tB, std::chrono::milliseconds(2000))); + + cvc::state_cluster_shard sA(aA, "C", "A"); + cvc::state_cluster_shard sB(aB, "C", "B"); + sA.attach(); + sB.attach(); + tA->register_shard(&sA); + tB.register_shard(&sB); + + // Phase 1: replicate a value. + cvc::state::instance(aA)("k").value(std::string("seed")); + cvc::state::instance(aA)("k").value(std::string("v1")); + tA->pump_all(); + tA->flush(); + tB.wait_for_received(1, std::chrono::milliseconds(2000)); + EXPECT_EQ(cvc::state::instance(aB)("k").value(), "v1"); + + // Phase 2: stop A's transport (simulate restart). + tA->unregister_shard(&sA); + tA->stop(); + tA.reset(); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + // Phase 3: restart A, reconnect to B. + tA = std::make_unique(); + tA->start("127.0.0.1:0", "A", "C"); + ASSERT_TRUE(tA->connect_to_peer(addrB, std::chrono::milliseconds(2000))); + ASSERT_TRUE(wait_connected(*tA, tB, std::chrono::milliseconds(2000))); + tA->register_shard(&sA); + + // Phase 4: replicate after reconnect. + cvc::state::instance(aA)("k").value(std::string("v2_after_reconnect")); + tA->pump_all(); + tA->flush(); + tB.wait_for_received(2, std::chrono::milliseconds(2000)); + EXPECT_EQ(cvc::state::instance(aB)("k").value(), "v2_after_reconnect"); + + tA->stop(); + tB.stop(); +} + +TEST(StateTransportGrpcReconnectTest, PeerDisconnectNoHang) { + cvc::app aA, aB; + + cvc::state_transport_grpc tA, tB; + tA.start("127.0.0.1:0", "A", "C"); + tB.start("127.0.0.1:0", "B", "C"); + ASSERT_TRUE(tA.connect_to_peer(tB.listen_address(), std::chrono::milliseconds(2000))); + ASSERT_TRUE(wait_connected(tA, tB, std::chrono::milliseconds(2000))); + + cvc::state_cluster_shard sA(aA, "C", "A"); + cvc::state_cluster_shard sB(aB, "C", "B"); + sA.attach(); + sB.attach(); + tA.register_shard(&sA); + tB.register_shard(&sB); + + cvc::state::instance(aA)("x").value(std::string("seed")); + cvc::state::instance(aA)("x").value(std::string("v1")); + tA.pump_all(); + tA.flush(); + tB.wait_for_received(1, std::chrono::milliseconds(2000)); + EXPECT_EQ(cvc::state::instance(aB)("x").value(), "v1"); + + // B goes away. + tB.stop(); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + // A can still pump without crashing. + cvc::state::instance(aA)("x").value(std::string("v2")); + EXPECT_NO_THROW(tA.pump_all()); + EXPECT_NO_THROW(tA.flush()); + + tA.stop(); +} + +TEST(StateTransportGrpcReconnectTest, ConnectionCountTracking) { + cvc::state_transport_grpc tA, tB; + EXPECT_EQ(tA.connection_count(), 0u); + + tA.start("127.0.0.1:0", "A", "C"); + tB.start("127.0.0.1:0", "B", "C"); + EXPECT_EQ(tA.connection_count(), 0u); + + ASSERT_TRUE(tA.connect_to_peer(tB.listen_address(), std::chrono::milliseconds(2000))); + ASSERT_TRUE(wait_connected(tA, tB, std::chrono::milliseconds(2000))); + EXPECT_GE(tA.connection_count(), 1u); + EXPECT_GE(tB.connection_count(), 1u); + + tB.stop(); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + EXPECT_NO_THROW(tA.connection_count()); + + tA.stop(); +} diff --git a/src/cvc/tests/state_zstd_round_trip_test.cpp b/src/cvc/tests/state_zstd_round_trip_test.cpp new file mode 100644 index 0000000..d18ef7c --- /dev/null +++ b/src/cvc/tests/state_zstd_round_trip_test.cpp @@ -0,0 +1,89 @@ +/* + Copyright 2026 The University of Texas at Austin + + This file is part of libcvc. + + libcvc is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. +*/ + +#include +#include +#include +#include +#include + +using cvc::state_compression_registry; + +class StateZstdRoundTripTest : public ::testing::Test { +protected: + state_compression_registry registry; +}; + +TEST_F(StateZstdRoundTripTest, ZstdIsRegistered) { + auto ids = registry.ids(); + bool has_zstd = false; + for (const auto &id : ids) + if (id == "zstd") has_zstd = true; + EXPECT_TRUE(has_zstd) << "zstd codec not found in registry"; +} + +TEST_F(StateZstdRoundTripTest, CompressDecompressEmpty) { + std::vector input; + auto compressed = registry.encode("zstd", input); + std::vector decompressed; + EXPECT_TRUE(registry.decode("zstd", compressed, decompressed)); + EXPECT_EQ(decompressed, input); +} + +TEST_F(StateZstdRoundTripTest, CompressDecompressSmall) { + std::string text = "Hello, zstd compression codec!"; + std::vector input(text.begin(), text.end()); + auto compressed = registry.encode("zstd", input); + // Compressed should be non-empty. + EXPECT_FALSE(compressed.empty()); + std::vector decompressed; + EXPECT_TRUE(registry.decode("zstd", compressed, decompressed)); + EXPECT_EQ(decompressed, input); +} + +TEST_F(StateZstdRoundTripTest, CompressDecompressLargeRepetitive) { + // Repetitive data should compress well. + std::vector input(100000, static_cast('A')); + auto compressed = registry.encode("zstd", input); + EXPECT_LT(compressed.size(), input.size() / 2) + << "zstd should compress repetitive data significantly"; + std::vector decompressed; + EXPECT_TRUE(registry.decode("zstd", compressed, decompressed)); + EXPECT_EQ(decompressed, input); +} + +TEST_F(StateZstdRoundTripTest, CompressDecompressRandom) { + std::mt19937 rng(42); + std::uniform_int_distribution dist(0, 255); + std::vector input(8192); + for (auto &b : input) + b = static_cast(dist(rng)); + auto compressed = registry.encode("zstd", input); + std::vector decompressed; + EXPECT_TRUE(registry.decode("zstd", compressed, decompressed)); + EXPECT_EQ(decompressed, input); +} + +TEST_F(StateZstdRoundTripTest, CompressDecompressSingleByte) { + std::vector input{0x42}; + auto compressed = registry.encode("zstd", input); + std::vector decompressed; + EXPECT_TRUE(registry.decode("zstd", compressed, decompressed)); + EXPECT_EQ(decompressed, input); +} + +TEST_F(StateZstdRoundTripTest, SharedRegistryHasZstd) { + auto &shared = state_compression_registry::shared(); + auto ids = shared.ids(); + bool has_zstd = false; + for (const auto &id : ids) + if (id == "zstd") has_zstd = true; + EXPECT_TRUE(has_zstd); +} From 205ba9eda4381ea156f386b871a55a6a0ecc36f4 Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Fri, 22 May 2026 20:04:10 -0500 Subject: [PATCH 3/4] distributed-state: wire max_inline_payload_bytes + auto-publish conflicts 1. Wire max_inline_payload_bytes into state_cluster_shard::drain_local(): - Add set_blob_store() and set_max_inline_payload_bytes() to shard - In drain_local(), values exceeding the threshold are offloaded to the blob store and replaced with a blob_ref payload - Session::join() passes config.max_inline_payload_bytes to shard 2. Auto-publish conflict metrics in session pump loop: - Store app* context in distributed_state_session for metrics - Every 100 pump cycles, call publish_conflicts() to write recent conflict entries under __system.distributed..conflicts.* - Only active when config.resolve_conflicts is true Tests: - InlinePayloadOffloadToBlobStore: verifies large values go to blob store - InlinePayloadOffloadDisabledByDefault: verifies no offload without blob store - ConflictAutoPublish: verifies conflicts appear in state tree automatically 6 files changed, +169/-1 lines --- inc/cvc/distributed_state_session.h | 2 + inc/cvc/state_cluster_shard.h | 12 ++++ src/cvc/distributed_state_session.cpp | 10 +++- src/cvc/state_cluster_shard.cpp | 27 +++++++++ src/cvc/tests/state_cluster_shard_test.cpp | 66 +++++++++++++++++++++ src/cvc/tests/state_session_status_test.cpp | 53 +++++++++++++++++ 6 files changed, 169 insertions(+), 1 deletion(-) diff --git a/inc/cvc/distributed_state_session.h b/inc/cvc/distributed_state_session.h index 2d78fb6..e09ec74 100644 --- a/inc/cvc/distributed_state_session.h +++ b/inc/cvc/distributed_state_session.h @@ -196,6 +196,8 @@ class distributed_state_session { std::unique_ptr _admin; std::unique_ptr _hydrator; + app *_app_ctx = nullptr; + std::thread _pump_thread; std::atomic _running{false}; std::atomic _pump_cycles{0}; diff --git a/inc/cvc/state_cluster_shard.h b/inc/cvc/state_cluster_shard.h index 61e0eff..68dab81 100644 --- a/inc/cvc/state_cluster_shard.h +++ b/inc/cvc/state_cluster_shard.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -102,6 +103,14 @@ class state_cluster_shard { state_hash_partition &partition() noexcept { return _partition; } const state_hash_partition &partition() const noexcept { return _partition; } + // Blob store and inline payload threshold. When both are set, + // drain_local() offloads mutation values larger than the threshold + // to the blob store and replaces the payload with a blob_ref. + void set_blob_store(state_blob_store *store) noexcept; + state_blob_store *blob_store() const noexcept; + void set_max_inline_payload_bytes(std::uint32_t bytes) noexcept; + std::uint32_t max_inline_payload_bytes() const noexcept; + // When true and the partition map is non-empty, ingest_remote() // rejects mutations whose path hashes to a different node_id. void set_enforce_partition(bool enforce) noexcept; @@ -382,6 +391,9 @@ class state_cluster_shard { hybrid_clock _clock; state_hash_partition _partition; bool _enforce_partition = false; + + state_blob_store *_blob_store = nullptr; + std::uint32_t _max_inline_payload_bytes = 0; // 0 = disabled }; } // namespace CVC_NAMESPACE diff --git a/src/cvc/distributed_state_session.cpp b/src/cvc/distributed_state_session.cpp index ef11917..b5f146c 100644 --- a/src/cvc/distributed_state_session.cpp +++ b/src/cvc/distributed_state_session.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #ifndef _WIN32 #include @@ -110,6 +111,8 @@ distributed_state_session::join(app &ctx, const distributed_state_config &config session->_shard->set_enforce_delegation(config.enforce_delegation); session->_shard->set_resolve_conflicts(config.resolve_conflicts); session->_shard->set_enforce_interest(config.enforce_interest); + session->_shard->set_blob_store(session->_blob_store.get()); + session->_shard->set_max_inline_payload_bytes(config.max_inline_payload_bytes); // 6. Apply mounts. for (const auto &mount : config.mounts) { @@ -167,6 +170,7 @@ distributed_state_session::join(app &ctx, const distributed_state_config &config } // 11. Start pump thread. + session->_app_ctx = &ctx; session->_running.store(true, std::memory_order_release); if (config.pump_interval_ms > 0) { session->_pump_thread = std::thread([session]() { session->pump_loop(); }); @@ -230,7 +234,11 @@ void distributed_state_session::pump_loop() { const auto interval = std::chrono::milliseconds(_config.pump_interval_ms); while (_running.load(std::memory_order_acquire)) { _transport->pump_all(); - _pump_cycles.fetch_add(1, std::memory_order_relaxed); + auto cycles = _pump_cycles.fetch_add(1, std::memory_order_relaxed) + 1; + // Periodically publish conflict metrics to the state tree. + if (_config.resolve_conflicts && _app_ctx && (cycles % 100) == 0) { + state_distributed_metrics::publish_conflicts(*_app_ctx, *_shard); + } std::this_thread::sleep_for(interval); } // Final drain. diff --git a/src/cvc/state_cluster_shard.cpp b/src/cvc/state_cluster_shard.cpp index 5d1ccb9..4c21a69 100644 --- a/src/cvc/state_cluster_shard.cpp +++ b/src/cvc/state_cluster_shard.cpp @@ -368,6 +368,16 @@ std::vector state_cluster_shard::drain_local(std::size_t max_cou // causal ordering via their own hybrid_clock::update(). if (m.hlc_time == 0) m.hlc_time = _clock.now().packed(); + // Offload large values to the blob store when a threshold is set. + if (_blob_store && _max_inline_payload_bytes > 0 && + m.op == state_mutation_op::set_value && + m.payload.kind == state_payload_kind::none && + m.string_value.size() > _max_inline_payload_bytes) { + std::vector bytes(m.string_value.begin(), m.string_value.end()); + state_blob_ref ref = _blob_store->put(bytes, m.type_name); + m.string_value.clear(); + m.payload = state_payload::blob_ref(ref); + } out.push_back(std::move(m)); if (max_count != 0 && out.size() >= max_count) break; @@ -524,6 +534,23 @@ bool state_cluster_shard::enforce_partition() const noexcept { return _enforce_partition; } +void state_cluster_shard::set_blob_store(state_blob_store *store) noexcept { + std::lock_guard lk(_mutex); + _blob_store = store; +} + +state_blob_store *state_cluster_shard::blob_store() const noexcept { + return _blob_store; +} + +void state_cluster_shard::set_max_inline_payload_bytes(std::uint32_t bytes) noexcept { + _max_inline_payload_bytes = bytes; +} + +std::uint32_t state_cluster_shard::max_inline_payload_bytes() const noexcept { + return _max_inline_payload_bytes; +} + void state_cluster_shard::set_transport(state_transport *t) noexcept { std::lock_guard lk(_mutex); _transport = t; diff --git a/src/cvc/tests/state_cluster_shard_test.cpp b/src/cvc/tests/state_cluster_shard_test.cpp index 9a16fe6..78f0c6f 100644 --- a/src/cvc/tests/state_cluster_shard_test.cpp +++ b/src/cvc/tests/state_cluster_shard_test.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -472,3 +473,68 @@ TEST(StateClusterShardTest, DelegationRevocationRestoresLocal) { EXPECT_TRUE(sh.delegation().revoke("simulation")); EXPECT_TRUE(sh.ingest_remote(make_set_value("nodeC", 2, "simulation.x", "v")).applied); } + +TEST(StateClusterShardTest, InlinePayloadOffloadToBlobStore) { + cvc::app a; + cvc::state_cluster_shard sh(a, "clusterA", "nodeA"); + sh.attach(); + + // Set up blob store and threshold. + cvc::memory_state_blob_store store; + sh.set_blob_store(&store); + sh.set_max_inline_payload_bytes(10); // offload values > 10 bytes + + // Write a small value (should stay inline). + cvc::state::instance(a)("test.small").value(std::string("hi")); + cvc::state::instance(a)("test.small").value(std::string("hello")); // 5 bytes + auto drained = sh.drain_local(); + bool found_small = false; + for (auto &m : drained) { + if (m.path == "test.small") { + found_small = true; + EXPECT_EQ(m.payload.kind, cvc::state_payload_kind::none); + EXPECT_EQ(m.string_value, "hello"); + } + } + EXPECT_TRUE(found_small); + + // Write a large value (should be offloaded to blob store). + std::string big_value(200, 'X'); + cvc::state::instance(a)("test.big").value(std::string("init")); + cvc::state::instance(a)("test.big").value(big_value); + auto drained2 = sh.drain_local(); + bool found_big = false; + for (auto &m : drained2) { + if (m.path == "test.big") { + found_big = true; + EXPECT_EQ(m.payload.kind, cvc::state_payload_kind::blob); + EXPECT_TRUE(m.string_value.empty()); + EXPECT_FALSE(m.payload.blob.digest.empty()); + EXPECT_EQ(m.payload.blob.size_bytes, 200u); + // Verify we can retrieve the blob from the store. + std::vector retrieved; + EXPECT_TRUE(store.get(m.payload.blob.digest, retrieved)); + EXPECT_EQ(retrieved.size(), 200u); + EXPECT_EQ(retrieved, std::vector(200, 'X')); + } + } + EXPECT_TRUE(found_big); +} + +TEST(StateClusterShardTest, InlinePayloadOffloadDisabledByDefault) { + cvc::app a; + cvc::state_cluster_shard sh(a, "clusterA", "nodeA"); + sh.attach(); + + // No blob store set — large values should stay inline. + std::string big_value(200, 'Y'); + cvc::state::instance(a)("test.noblobstore").value(std::string("init")); + cvc::state::instance(a)("test.noblobstore").value(big_value); + auto drained = sh.drain_local(); + for (auto &m : drained) { + if (m.path == "test.noblobstore") { + EXPECT_EQ(m.payload.kind, cvc::state_payload_kind::none); + EXPECT_EQ(m.string_value, big_value); + } + } +} diff --git a/src/cvc/tests/state_session_status_test.cpp b/src/cvc/tests/state_session_status_test.cpp index b55cdb0..a17c15b 100644 --- a/src/cvc/tests/state_session_status_test.cpp +++ b/src/cvc/tests/state_session_status_test.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -100,3 +101,55 @@ TEST_F(StateSessionStatusTest, ShardAndTransportAccessible) { (void)transport; session->stop(); } + +TEST_F(StateSessionStatusTest, ConflictAutoPublish) { + // Create a session with resolve_conflicts enabled. + distributed_state_config config; + config.cluster_id = "test_cluster"; + config.node_id = "node_A"; + config.transport = transport_kind::inproc; + config.pump_interval_ms = 1; // fast pump for test + config.resolve_conflicts = true; + auto session = distributed_state_session::join(ctx, config); + + // Inject two conflicting mutations. Inject winner (higher + // lexicographic node_id) first, then the loser. The ring buffer + // records the loser arrival. + cvc::state_mutation m1; + m1.cluster_id = "test_cluster"; + m1.origin_node_id = "node_Z"; + m1.sequence = 10; + m1.path = "conflict.key"; + m1.string_value = "winner"; + m1.type_name = "std::string"; + m1.op = cvc::state_mutation_op::set_value; + session->shard().ingest_remote(m1); + + cvc::state_mutation m2; + m2.cluster_id = "test_cluster"; + m2.origin_node_id = "node_B"; + m2.sequence = 10; + m2.path = "conflict.key"; + m2.string_value = "loser"; + m2.type_name = "std::string"; + m2.op = cvc::state_mutation_op::set_value; + session->shard().ingest_remote(m2); + + EXPECT_GT(session->shard().total_conflicts_detected(), 0u); + + // Let pump run enough cycles for auto-publish (every 100 cycles). + // At 1ms interval, 100 cycles ≈ 100ms. Wait generously for CI. + for (int i = 0; i < 50; ++i) { + auto s = session->status(); + if (s.pump_cycles >= 110) + break; + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + + // Verify conflict metrics appear in the state tree. + std::string prefix = "__system.distributed.test_cluster.conflicts.recent.0.path"; + std::string val = cvc::state::instance(ctx)(prefix).value(); + EXPECT_EQ(val, "conflict.key"); + + session->stop(); +} From a155d4db9c0d4c2bef4430fb8dadf69bb998956b Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Fri, 22 May 2026 20:17:41 -0500 Subject: [PATCH 4/4] fix(ci): resolve macOS zstd linker error, grpc note_seen arity, clang-format - Add zstd to macOS brew install lines in CI workflow - Use IMPORTED_TARGET in pkg_check_modules for zstd to get full library paths (fixes macOS linker error: 'ld: library zstd not found') - Pass steady_clock timestamp to note_seen() in state_transport_grpc.cpp (fixes grpc build: note_seen now requires 2 arguments) - Apply clang-format to all changed lines vs base c297c8a --- .github/workflows/ci.yml | 4 +- inc/cvc/distributed_state_session.h | 2 +- inc/cvc/state_cluster_shard.h | 2 +- inc/cvc/state_hash_partition.h | 4 +- inc/cvc/state_hybrid_time.h | 7 ++-- inc/cvc/state_transport_ipc.h | 2 +- proto/state_transport.proto | 18 ++++----- src/cvc/CMakeLists.txt | 10 +++-- src/cvc/distributed_state_session.cpp | 3 +- src/cvc/state_brick_manifest.cpp | 3 +- src/cvc/state_cluster_shard.cpp | 13 ++---- src/cvc/state_compression_registry.cpp | 3 +- src/cvc/state_transport_grpc.cpp | 23 ++++++----- src/cvc/state_transport_ipc.cpp | 11 ++--- src/cvc/tests/state_brick_frustum_test.cpp | 40 +++++++++---------- src/cvc/tests/state_conflict_ring_test.cpp | 18 ++++----- src/cvc/tests/state_hash_partition_test.cpp | 3 +- .../tests/state_reconnect_resilience_test.cpp | 4 +- src/cvc/tests/state_session_status_test.cpp | 3 +- .../tests/state_snapshot_protocol_test.cpp | 21 ++++++---- src/cvc/tests/state_zstd_round_trip_test.cpp | 6 ++- 21 files changed, 98 insertions(+), 102 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b800a51..ee9e0b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -553,14 +553,14 @@ jobs: - name: Install dependencies (libcvc) if: matrix.kind == 'libcvc' run: | - brew install cmake ninja boost hdf5 fftw gsl imagemagick cgal log4cplus + brew install cmake ninja boost hdf5 fftw gsl imagemagick cgal log4cplus zstd echo "BOOST_ROOT=$(brew --prefix boost)" >> $GITHUB_ENV echo "CMAKE_PREFIX_PATH=$(brew --prefix boost):$(brew --prefix hdf5)" >> $GITHUB_ENV - name: Install dependencies (volrover3) if: matrix.kind == 'volrover3' run: | - brew install cmake ninja boost hdf5 fftw gsl imagemagick qt@6 vtk cgal + brew install cmake ninja boost hdf5 fftw gsl imagemagick qt@6 vtk cgal zstd echo "BOOST_ROOT=$(brew --prefix boost)" >> $GITHUB_ENV echo "CMAKE_PREFIX_PATH=$(brew --prefix boost):$(brew --prefix qt@6):$(brew --prefix hdf5)" >> $GITHUB_ENV echo "$(brew --prefix qt@6)/bin" >> $GITHUB_PATH diff --git a/inc/cvc/distributed_state_session.h b/inc/cvc/distributed_state_session.h index e09ec74..fe260d9 100644 --- a/inc/cvc/distributed_state_session.h +++ b/inc/cvc/distributed_state_session.h @@ -97,7 +97,7 @@ struct distributed_state_config { // Tuning. std::uint32_t max_inline_payload_bytes = 65536; - std::string blob_store_path; // empty = memory-only blob store + std::string blob_store_path; // empty = memory-only blob store bool snapshot_on_join = false; // request full snapshot from first seed on join std::uint32_t pump_interval_ms = 10; // background pump loop interval (0 = no pump thread) diff --git a/inc/cvc/state_cluster_shard.h b/inc/cvc/state_cluster_shard.h index 68dab81..178f0ce 100644 --- a/inc/cvc/state_cluster_shard.h +++ b/inc/cvc/state_cluster_shard.h @@ -14,10 +14,10 @@ #include #include #include +#include #include #include #include -#include #include #include #include diff --git a/inc/cvc/state_hash_partition.h b/inc/cvc/state_hash_partition.h index 8aad896..42ecc18 100644 --- a/inc/cvc/state_hash_partition.h +++ b/inc/cvc/state_hash_partition.h @@ -63,8 +63,8 @@ class state_hash_partition { std::uint64_t slice = total / node_ids.size(); std::uint32_t lo = 0; for (std::size_t i = 0; i < node_ids.size(); ++i) { - std::uint32_t hi = (i + 1 == node_ids.size()) ? UINT32_MAX - : static_cast(lo + slice - 1); + std::uint32_t hi = + (i + 1 == node_ids.size()) ? UINT32_MAX : static_cast(lo + slice - 1); _ranges.push_back({node_ids[i], lo, hi + 1}); lo = hi + 1; } diff --git a/inc/cvc/state_hybrid_time.h b/inc/cvc/state_hybrid_time.h index 784f9e6..330868f 100644 --- a/inc/cvc/state_hybrid_time.h +++ b/inc/cvc/state_hybrid_time.h @@ -109,10 +109,9 @@ class hybrid_clock { private: static std::uint64_t wall_ms() noexcept { - return static_cast( - std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count()); + return static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); } mutable std::mutex _mu; diff --git a/inc/cvc/state_transport_ipc.h b/inc/cvc/state_transport_ipc.h index 002dc37..dbb0ab5 100644 --- a/inc/cvc/state_transport_ipc.h +++ b/inc/cvc/state_transport_ipc.h @@ -19,9 +19,9 @@ #include #include #include -#include #include #include +#include #include namespace CVC_NAMESPACE { diff --git a/proto/state_transport.proto b/proto/state_transport.proto index b5d1baf..048bfe2 100644 --- a/proto/state_transport.proto +++ b/proto/state_transport.proto @@ -74,22 +74,22 @@ message Message { // Chunk fetch request: ask a peer for a blob chunk by digest. message ChunkRequest { - string digest = 1; // SHA-256 hex digest - uint64 request_id = 2; // caller-assigned; echoed in response + string digest = 1; // SHA-256 hex digest + uint64 request_id = 2; // caller-assigned; echoed in response } // Chunk fetch response: carries the blob chunk bytes. message ChunkResponse { - string digest = 1; // echoed from request - uint64 request_id = 2; // echoed from request - bool found = 3; // false if the peer does not have the chunk - bytes data = 4; // raw chunk bytes (empty if !found) + string digest = 1; // echoed from request + uint64 request_id = 2; // echoed from request + bool found = 3; // false if the peer does not have the chunk + bytes data = 4; // raw chunk bytes (empty if !found) } // Snapshot request: ask a peer for the current state tree. message SnapshotRequest { string cluster_id = 1; - string path_prefix = 2; // "" = entire tree + string path_prefix = 2; // "" = entire tree uint64 request_id = 3; } @@ -108,7 +108,7 @@ message SnapshotEntry { // Snapshot response: sent in response to SnapshotRequest. message SnapshotResponse { uint64 request_id = 1; - bool final = 2; // true on the last chunk + bool final = 2; // true on the last chunk repeated SnapshotEntry entries = 3; } @@ -117,7 +117,7 @@ message SnapshotResponse { message Heartbeat { string node_id = 1; string cluster_id = 2; - uint64 timestamp_ns = 3; // sender's wall-clock ns (observability) + uint64 timestamp_ns = 3; // sender's wall-clock ns (observability) } // Single bidirectional frame envelope. Both client and server write diff --git a/src/cvc/CMakeLists.txt b/src/cvc/CMakeLists.txt index 4d8b3f6..30f36ee 100644 --- a/src/cvc/CMakeLists.txt +++ b/src/cvc/CMakeLists.txt @@ -836,7 +836,7 @@ endif() # zstd compression (production compressor for distributed state). find_package(PkgConfig QUIET) if(PkgConfig_FOUND) - pkg_check_modules(ZSTD QUIET libzstd) + pkg_check_modules(ZSTD QUIET IMPORTED_TARGET libzstd) endif() if(NOT ZSTD_FOUND) find_library(ZSTD_LIBRARIES NAMES zstd) @@ -846,8 +846,12 @@ if(NOT ZSTD_FOUND) endif() endif() if(ZSTD_FOUND) - target_include_directories(cvc PRIVATE ${ZSTD_INCLUDE_DIRS}) - target_link_libraries(cvc PRIVATE ${ZSTD_LIBRARIES}) + if(TARGET PkgConfig::ZSTD) + target_link_libraries(cvc PRIVATE PkgConfig::ZSTD) + else() + target_include_directories(cvc PRIVATE ${ZSTD_INCLUDE_DIRS}) + target_link_libraries(cvc PRIVATE ${ZSTD_LIBRARIES}) + endif() else() message(WARNING "libzstd not found; zstd compression codec will not link") endif() diff --git a/src/cvc/distributed_state_session.cpp b/src/cvc/distributed_state_session.cpp index b5f146c..ddeb1ea 100644 --- a/src/cvc/distributed_state_session.cpp +++ b/src/cvc/distributed_state_session.cpp @@ -142,8 +142,7 @@ distributed_state_session::join(app &ctx, const distributed_state_config &config // 9. Create hydrator (blob fetch + codec decode). session->_hydrator = std::make_unique( - *session->_blob_store, session->_shard->codecs(), - &state_compression_registry::shared()); + *session->_blob_store, session->_shard->codecs(), &state_compression_registry::shared()); session->_hydrator->set_transport(session->_transport.get()); // 10. Attach shard (start observing state changes). diff --git a/src/cvc/state_brick_manifest.cpp b/src/cvc/state_brick_manifest.cpp index 944b943..90dcb7e 100644 --- a/src/cvc/state_brick_manifest.cpp +++ b/src/cvc/state_brick_manifest.cpp @@ -205,8 +205,7 @@ state_brick_manifest::bricks_in_region(std::uint64_t lo_x, std::uint64_t lo_y, s return result; } -std::vector -state_brick_manifest::bricks_in_frustum(const plane planes[6]) const { +std::vector state_brick_manifest::bricks_in_frustum(const plane planes[6]) const { std::vector result; for (std::size_t i = 0; i < extents.size(); ++i) { const brick_extent &ext = extents[i]; diff --git a/src/cvc/state_cluster_shard.cpp b/src/cvc/state_cluster_shard.cpp index 4c21a69..c2825d7 100644 --- a/src/cvc/state_cluster_shard.cpp +++ b/src/cvc/state_cluster_shard.cpp @@ -148,8 +148,7 @@ state_cluster_shard::recent_conflicts(std::size_t max_entries) const { out.reserve(n); // Walk backwards from the write cursor to get most-recent-first. for (std::size_t i = 0; i < n; ++i) { - std::size_t idx = - (_conflict_ring_pos + _conflict_ring.size() - 1 - i) % _conflict_ring.size(); + std::size_t idx = (_conflict_ring_pos + _conflict_ring.size() - 1 - i) % _conflict_ring.size(); out.push_back(_conflict_ring[idx]); } return out; @@ -310,8 +309,7 @@ state_cluster_shard::ingest_result state_cluster_shard::ingest_remote(const stat e.winner_sequence = it->second.sequence; e.loser_node_id = m.origin_node_id; e.loser_sequence = m.sequence; - _conflict_ring_pos = - (_conflict_ring_pos + 1) % kMaxConflictRing; + _conflict_ring_pos = (_conflict_ring_pos + 1) % kMaxConflictRing; } } } @@ -369,8 +367,7 @@ std::vector state_cluster_shard::drain_local(std::size_t max_cou if (m.hlc_time == 0) m.hlc_time = _clock.now().packed(); // Offload large values to the blob store when a threshold is set. - if (_blob_store && _max_inline_payload_bytes > 0 && - m.op == state_mutation_op::set_value && + if (_blob_store && _max_inline_payload_bytes > 0 && m.op == state_mutation_op::set_value && m.payload.kind == state_payload_kind::none && m.string_value.size() > _max_inline_payload_bytes) { std::vector bytes(m.string_value.begin(), m.string_value.end()); @@ -539,9 +536,7 @@ void state_cluster_shard::set_blob_store(state_blob_store *store) noexcept { _blob_store = store; } -state_blob_store *state_cluster_shard::blob_store() const noexcept { - return _blob_store; -} +state_blob_store *state_cluster_shard::blob_store() const noexcept { return _blob_store; } void state_cluster_shard::set_max_inline_payload_bytes(std::uint32_t bytes) noexcept { _max_inline_payload_bytes = bytes; diff --git a/src/cvc/state_compression_registry.cpp b/src/cvc/state_compression_registry.cpp index a805b06..4b99acf 100644 --- a/src/cvc/state_compression_registry.cpp +++ b/src/cvc/state_compression_registry.cpp @@ -64,8 +64,7 @@ state_zstd_compression_codec::encode(const std::vector &in) const return {}; std::size_t bound = ZSTD_compressBound(in.size()); std::vector out(bound); - std::size_t result = - ZSTD_compress(out.data(), out.size(), in.data(), in.size(), _level); + std::size_t result = ZSTD_compress(out.data(), out.size(), in.data(), in.size(), _level); if (ZSTD_isError(result)) return in; // fall back to uncompressed on error out.resize(result); diff --git a/src/cvc/state_transport_grpc.cpp b/src/cvc/state_transport_grpc.cpp index 971880c..063a36a 100644 --- a/src/cvc/state_transport_grpc.cpp +++ b/src/cvc/state_transport_grpc.cpp @@ -260,8 +260,8 @@ class StateTransportServiceImpl final : public pb::StateTransport::Service { _owner->on_inbound_message(decode_message(in.message())); _owner->increment_recv_messages(); } else if (in.has_chunk_request()) { - _owner->on_inbound_chunk_request( - conn.get(), in.chunk_request().digest(), in.chunk_request().request_id()); + _owner->on_inbound_chunk_request(conn.get(), in.chunk_request().digest(), + in.chunk_request().request_id()); } else if (in.has_chunk_response()) { const auto &cr = in.chunk_response(); std::string data_str = cr.data(); @@ -939,8 +939,9 @@ void state_transport_grpc::on_inbound_snapshot_request(connection *conn, } } -void state_transport_grpc::on_inbound_snapshot_response( - std::uint64_t request_id, const std::vector &entries, bool final) { +void state_transport_grpc::on_inbound_snapshot_response(std::uint64_t request_id, + const std::vector &entries, + bool final) { { std::lock_guard lk(_snap_waiters_mu); auto it = _snap_waiters.find(request_id); @@ -957,7 +958,9 @@ void state_transport_grpc::on_inbound_snapshot_response( void state_transport_grpc::on_inbound_heartbeat(const std::string &node_id, const std::string & /*cluster_id*/) { - _peers.note_seen(node_id); + auto now = std::chrono::steady_clock::now().time_since_epoch(); + _peers.note_seen(node_id, static_cast( + std::chrono::duration_cast(now).count())); } bool state_transport_grpc::request_snapshot(const std::string &cluster_id, @@ -996,8 +999,7 @@ bool state_transport_grpc::request_snapshot(const std::string &cluster_id, bool result = false; if (sent > 0) { std::unique_lock lk(_snap_waiters_mu); - _snap_waiters_cv.wait_for(lk, std::chrono::seconds(30), - [&]() { return waiter->done; }); + _snap_waiters_cv.wait_for(lk, std::chrono::seconds(30), [&]() { return waiter->done; }); if (waiter->done) { if (on_entries) on_entries(waiter->entries, true); @@ -1025,10 +1027,9 @@ void state_transport_grpc::heartbeat_loop() { hb->set_node_id(_node_id); hb->set_cluster_id(_cluster_id); hb->set_timestamp_ns( - static_cast( - std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count())); + static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count())); std::vector> conns; { diff --git a/src/cvc/state_transport_ipc.cpp b/src/cvc/state_transport_ipc.cpp index f751b94..f46c485 100644 --- a/src/cvc/state_transport_ipc.cpp +++ b/src/cvc/state_transport_ipc.cpp @@ -30,7 +30,7 @@ namespace { constexpr std::uint32_t kMagic = 0x43564354u; // 'CVCT' constexpr std::uint16_t kVersion = 1; -constexpr std::uint16_t kMaxVersion = 1; // highest version we understand +constexpr std::uint16_t kMaxVersion = 1; // highest version we understand constexpr std::uint16_t kMsgHello = 1; constexpr std::uint16_t kMsgMutation = 2; constexpr std::uint16_t kMsgOob = 3; @@ -601,8 +601,7 @@ void state_transport_ipc::reader_loop(std::shared_ptr conn) { } // Patch entry count. for (int i = 0; i < 4; ++i) - rsp[count_offset + i] = - static_cast((entry_count >> (8 * i)) & 0xFF); + rsp[count_offset + i] = static_cast((entry_count >> (8 * i)) & 0xFF); std::lock_guard wlk(conn->write_mu); write_frame_locked(*conn, kMsgSnapRsp, rsp); @@ -941,8 +940,7 @@ bool state_transport_ipc::fetch_chunk(const std::string &digest, chunk_callback if (sent > 0) { // Wait for the first response (up to 10 seconds). std::unique_lock lk(_chunk_waiters_mu); - _chunk_waiters_cv.wait_for(lk, std::chrono::seconds(10), - [&]() { return waiter->done; }); + _chunk_waiters_cv.wait_for(lk, std::chrono::seconds(10), [&]() { return waiter->done; }); if (waiter->done && waiter->found) { if (on_chunk) on_chunk(digest, waiter->data); @@ -992,8 +990,7 @@ bool state_transport_ipc::request_snapshot(const std::string &cluster_id, bool result = false; if (sent > 0) { std::unique_lock lk(_snap_waiters_mu); - _snap_waiters_cv.wait_for(lk, std::chrono::seconds(10), - [&]() { return waiter->done; }); + _snap_waiters_cv.wait_for(lk, std::chrono::seconds(10), [&]() { return waiter->done; }); if (waiter->done) { if (on_entries) on_entries(waiter->entries, /*final=*/true); diff --git a/src/cvc/tests/state_brick_frustum_test.cpp b/src/cvc/tests/state_brick_frustum_test.cpp index 3382a22..3c7ec22 100644 --- a/src/cvc/tests/state_brick_frustum_test.cpp +++ b/src/cvc/tests/state_brick_frustum_test.cpp @@ -41,12 +41,12 @@ TEST_F(StateBrickFrustumTest, AllPlanesPassingReturnsAll) { // Planes that encompass everything: each plane's positive half-space // includes the origin and extends far. Use a giant box frustum. state_brick_manifest::plane planes[6] = { - {1, 0, 0, 100}, // x >= -100 - {-1, 0, 0, 100}, // -x >= -100 i.e. x <= 100 - {0, 1, 0, 100}, // y >= -100 - {0, -1, 0, 100}, // y <= 100 - {0, 0, 1, 100}, // z >= -100 - {0, 0, -1, 100}, // z <= 100 + {1, 0, 0, 100}, // x >= -100 + {-1, 0, 0, 100}, // -x >= -100 i.e. x <= 100 + {0, 1, 0, 100}, // y >= -100 + {0, -1, 0, 100}, // y <= 100 + {0, 0, 1, 100}, // z >= -100 + {0, 0, -1, 100}, // z <= 100 }; auto result = manifest.bricks_in_frustum(planes); EXPECT_EQ(result.size(), 9u); @@ -55,12 +55,12 @@ TEST_F(StateBrickFrustumTest, AllPlanesPassingReturnsAll) { TEST_F(StateBrickFrustumTest, TightFrustumSelectsSubset) { // Planes that only include the first brick (0..10, 0..10, 0..10). state_brick_manifest::plane planes[6] = { - {1, 0, 0, 0}, // x >= 0 - {-1, 0, 0, 10}, // x <= 10 - {0, 1, 0, 0}, // y >= 0 - {0, -1, 0, 10}, // y <= 10 - {0, 0, 1, 0}, // z >= 0 - {0, 0, -1, 10}, // z <= 10 + {1, 0, 0, 0}, // x >= 0 + {-1, 0, 0, 10}, // x <= 10 + {0, 1, 0, 0}, // y >= 0 + {0, -1, 0, 10}, // y <= 10 + {0, 0, 1, 0}, // z >= 0 + {0, 0, -1, 10}, // z <= 10 }; auto result = manifest.bricks_in_frustum(planes); // At least the first brick should be included. @@ -73,12 +73,12 @@ TEST_F(StateBrickFrustumTest, TightFrustumSelectsSubset) { TEST_F(StateBrickFrustumTest, FrustumFarAwayReturnsEmpty) { // Planes that are far from any brick (everything at x > 1000). state_brick_manifest::plane planes[6] = { - {1, 0, 0, -1000}, // x >= 1000 - {-1, 0, 0, 2000}, // x <= 2000 - {0, 1, 0, -1000}, // y >= 1000 - {0, -1, 0, 2000}, // y <= 2000 - {0, 0, 1, -1000}, // z >= 1000 - {0, 0, -1, 2000}, // z <= 2000 + {1, 0, 0, -1000}, // x >= 1000 + {-1, 0, 0, 2000}, // x <= 2000 + {0, 1, 0, -1000}, // y >= 1000 + {0, -1, 0, 2000}, // y <= 2000 + {0, 0, 1, -1000}, // z >= 1000 + {0, 0, -1, 2000}, // z <= 2000 }; auto result = manifest.bricks_in_frustum(planes); EXPECT_EQ(result.size(), 0u); @@ -87,8 +87,8 @@ TEST_F(StateBrickFrustumTest, FrustumFarAwayReturnsEmpty) { TEST_F(StateBrickFrustumTest, EmptyManifestReturnsEmpty) { state_brick_manifest empty; state_brick_manifest::plane planes[6] = { - {1, 0, 0, 100}, {-1, 0, 0, 100}, {0, 1, 0, 100}, - {0, -1, 0, 100}, {0, 0, 1, 100}, {0, 0, -1, 100}, + {1, 0, 0, 100}, {-1, 0, 0, 100}, {0, 1, 0, 100}, + {0, -1, 0, 100}, {0, 0, 1, 100}, {0, 0, -1, 100}, }; auto result = empty.bricks_in_frustum(planes); EXPECT_EQ(result.size(), 0u); diff --git a/src/cvc/tests/state_conflict_ring_test.cpp b/src/cvc/tests/state_conflict_ring_test.cpp index 579fb8b..72d14d7 100644 --- a/src/cvc/tests/state_conflict_ring_test.cpp +++ b/src/cvc/tests/state_conflict_ring_test.cpp @@ -44,8 +44,8 @@ class StateConflictRingTest : public ::testing::Test { transport.unregister_shard(&shard_b); } - state_mutation make_mutation(const std::string &node, std::uint64_t seq, - const std::string &path, const std::string &value) { + state_mutation make_mutation(const std::string &node, std::uint64_t seq, const std::string &path, + const std::string &value) { state_mutation m; m.cluster_id = "cluster"; m.origin_node_id = node; @@ -90,12 +90,12 @@ TEST_F(StateConflictRingTest, ConflictsRingWrapsAround) { // Use alphabetically lower node ids to guarantee loss. for (int i = 0; i < 200; ++i) { // Winner: node_C > node_B lexicographically. - auto ma = make_mutation("node_C", static_cast(i + 1), - "path." + std::to_string(i), "c"); + auto ma = + make_mutation("node_C", static_cast(i + 1), "path." + std::to_string(i), "c"); shard_a.ingest_remote(ma); // Loser: node_B < node_C, same path, so should_replace returns false. - auto mc = make_mutation("node_B", static_cast(i + 1), - "path." + std::to_string(i), "b"); + auto mc = + make_mutation("node_B", static_cast(i + 1), "path." + std::to_string(i), "b"); shard_a.ingest_remote(mc); } @@ -108,12 +108,10 @@ TEST_F(StateConflictRingTest, ConflictsRingWrapsAround) { TEST_F(StateConflictRingTest, RecentConflictsMaxEntries) { for (int i = 0; i < 10; ++i) { // Winner first (higher node id). - auto ma = make_mutation("node_C", static_cast(i + 1), - "p." + std::to_string(i), "c"); + auto ma = make_mutation("node_C", static_cast(i + 1), "p." + std::to_string(i), "c"); shard_a.ingest_remote(ma); // Loser: node_B < node_C for the same path. - auto mc = make_mutation("node_B", static_cast(i + 1), - "p." + std::to_string(i), "b"); + auto mc = make_mutation("node_B", static_cast(i + 1), "p." + std::to_string(i), "b"); shard_a.ingest_remote(mc); } diff --git a/src/cvc/tests/state_hash_partition_test.cpp b/src/cvc/tests/state_hash_partition_test.cpp index a650c72..75d2020 100644 --- a/src/cvc/tests/state_hash_partition_test.cpp +++ b/src/cvc/tests/state_hash_partition_test.cpp @@ -36,8 +36,7 @@ TEST_F(StateHashPartitionTest, AssignAndLookup) { for (int i = 0; i < 50; ++i) { std::string path = "test.path." + std::to_string(i); std::string owner = part.owner_of(path); - EXPECT_TRUE(owner == "node-1" || owner == "node-2") - << "path=" << path << " owner=" << owner; + EXPECT_TRUE(owner == "node-1" || owner == "node-2") << "path=" << path << " owner=" << owner; } } diff --git a/src/cvc/tests/state_reconnect_resilience_test.cpp b/src/cvc/tests/state_reconnect_resilience_test.cpp index 8d584ae..8670756 100644 --- a/src/cvc/tests/state_reconnect_resilience_test.cpp +++ b/src/cvc/tests/state_reconnect_resilience_test.cpp @@ -31,8 +31,8 @@ std::string make_socket_path(const std::string &label) { auto pid = static_cast(::getpid()); auto now = std::chrono::steady_clock::now().time_since_epoch().count(); auto dir = std::filesystem::temp_directory_path(); - return (dir / ("cvc_res_" + std::to_string(pid) + "_" + std::to_string(now) + "_" + label + - ".sock")) + return (dir / + ("cvc_res_" + std::to_string(pid) + "_" + std::to_string(now) + "_" + label + ".sock")) .string(); } diff --git a/src/cvc/tests/state_session_status_test.cpp b/src/cvc/tests/state_session_status_test.cpp index a17c15b..ea44f66 100644 --- a/src/cvc/tests/state_session_status_test.cpp +++ b/src/cvc/tests/state_session_status_test.cpp @@ -77,8 +77,7 @@ TEST_F(StateSessionStatusTest, SessionStopIsIdempotent) { TEST_F(StateSessionStatusTest, WaitForDataTimeoutReturns) { auto session = make_session(); // Wait for a non-existent path — should return after timeout. - auto result = session->wait_for_data("nonexistent.path", - std::chrono::milliseconds(50)); + auto result = session->wait_for_data("nonexistent.path", std::chrono::milliseconds(50)); // We just verify it returns without hanging. The status value // depends on whether the hydrator has seen this path. (void)result; diff --git a/src/cvc/tests/state_snapshot_protocol_test.cpp b/src/cvc/tests/state_snapshot_protocol_test.cpp index 9dcde82..b77ff3b 100644 --- a/src/cvc/tests/state_snapshot_protocol_test.cpp +++ b/src/cvc/tests/state_snapshot_protocol_test.cpp @@ -54,8 +54,10 @@ TEST_F(StateSnapshotProtocolTest, SnapshotFromLocalShard) { bool found_b = false, found_c = false; for (const auto &e : snap) { - if (e.path == "a.b" && e.string_value == "hello") found_b = true; - if (e.path == "a.c" && e.string_value == "world") found_c = true; + if (e.path == "a.b" && e.string_value == "hello") + found_b = true; + if (e.path == "a.c" && e.string_value == "world") + found_c = true; } EXPECT_TRUE(found_b) << "a.b not in snapshot"; EXPECT_TRUE(found_c) << "a.c not in snapshot"; @@ -74,8 +76,10 @@ TEST_F(StateSnapshotProtocolTest, SnapshotWithPrefixFilter) { auto snap = shard.snapshot("foo"); bool found_foo = false, found_bar = false; for (const auto &e : snap) { - if (e.path.find("foo") == 0) found_foo = true; - if (e.path.find("bar") == 0) found_bar = true; + if (e.path.find("foo") == 0) + found_foo = true; + if (e.path.find("bar") == 0) + found_bar = true; } EXPECT_TRUE(found_foo); EXPECT_FALSE(found_bar); @@ -118,8 +122,7 @@ TEST_F(StateSnapshotProtocolTest, IpcSnapshotRequestResponse) { // Request snapshot. std::vector entries; bool got_snapshot = client.request_snapshot( - "clust", "", - [&](const std::vector &e, bool /*final*/) { + "clust", "", [&](const std::vector &e, bool /*final*/) { entries = e; }); @@ -128,8 +131,10 @@ TEST_F(StateSnapshotProtocolTest, IpcSnapshotRequestResponse) { bool found_x = false, found_y = false; for (const auto &e : entries) { - if (e.path == "data.x" && e.string_value == "42") found_x = true; - if (e.path == "data.y" && e.string_value == "99") found_y = true; + if (e.path == "data.x" && e.string_value == "42") + found_x = true; + if (e.path == "data.y" && e.string_value == "99") + found_y = true; } EXPECT_TRUE(found_x); EXPECT_TRUE(found_y); diff --git a/src/cvc/tests/state_zstd_round_trip_test.cpp b/src/cvc/tests/state_zstd_round_trip_test.cpp index d18ef7c..e49758a 100644 --- a/src/cvc/tests/state_zstd_round_trip_test.cpp +++ b/src/cvc/tests/state_zstd_round_trip_test.cpp @@ -25,7 +25,8 @@ TEST_F(StateZstdRoundTripTest, ZstdIsRegistered) { auto ids = registry.ids(); bool has_zstd = false; for (const auto &id : ids) - if (id == "zstd") has_zstd = true; + if (id == "zstd") + has_zstd = true; EXPECT_TRUE(has_zstd) << "zstd codec not found in registry"; } @@ -84,6 +85,7 @@ TEST_F(StateZstdRoundTripTest, SharedRegistryHasZstd) { auto ids = shared.ids(); bool has_zstd = false; for (const auto &id : ids) - if (id == "zstd") has_zstd = true; + if (id == "zstd") + has_zstd = true; EXPECT_TRUE(has_zstd); }