Skip to content

Gossmap: compaction support #8869

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rustyrussell wants to merge 20 commits into
base: master
Choose a base branch
from
Open

Gossmap: compaction support #8869

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
7edd8ce
devtools: enhance dump-gossipstore to show some details of messages.
rustyrussell Feb 4, 2026
c9581e9
pytest: fix bogus test_gossip_store_compact_noappend test.
rustyrussell Feb 4, 2026
d1e4c64
devtools/gossmap-compress: generate better scids.
rustyrussell Feb 4, 2026
eef10c9
gossmap: reduce load times by 20%
rustyrussell Feb 4, 2026
c7ab481
gossip_store: add UUID entry at front of the store.
rustyrussell Feb 4, 2026
6f135a5
common/gossmap: use the UUID record on reopen.
rustyrussell Feb 4, 2026
403460f
gossipd: write uuid record on startup.
rustyrussell Feb 4, 2026
e1ca104
common: move gossip_store_wire.csv into common/ from gossipd/
rustyrussell Feb 4, 2026
fee4284
gossmap: add callback for gossipd to see dying messages.
rustyrussell Feb 4, 2026
c9bc7f2
gossmap: keep stats on live/deleted records.
rustyrussell Feb 4, 2026
b7c3857
gossipd: use gossmap to load the dying entries.
rustyrussell Feb 5, 2026
d2a5530
gossipd: don't gather dying channels during compaction.
rustyrussell Feb 5, 2026
c8154c7
gossipd: don't compact on startup.
rustyrussell Feb 5, 2026
de487ec
devtools/gossmap-compress: create latest gossip_store version
rustyrussell Feb 5, 2026
d6056ef
gossipd: put the last_writes array inside struct gossip_store.
rustyrussell Feb 5, 2026
5983087
common: expose gossip_store "header and type" single-read struct.
rustyrussell Feb 5, 2026
763f851
gossipd: lightningd/lightning_gossip_compactd
rustyrussell Feb 5, 2026
9d01bea
gossipd: code to invoke compactd and reopen store.
rustyrussell Feb 5, 2026
3e669d0
gossipd: compact when gossip store is 80% deleted records.
rustyrussell Feb 5, 2026
10176dc
gossipd: dev-compact-gossip-store to manually invoke compaction.
rustyrussell Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -468,6 +468,7 @@ PKGLIBEXEC_PROGRAMS = \
lightningd/lightning_connectd \
lightningd/lightning_dualopend \
lightningd/lightning_gossipd \
lightningd/lightning_gossip_compactd \
lightningd/lightning_hsmd \
lightningd/lightning_onchaind \
lightningd/lightning_openingd \
Expand All @@ -489,9 +490,14 @@ mkdocs.yml: $(MANPAGES:=.md)
# Every single object file.
ALL_OBJS := $(ALL_C_SOURCES:.c=.o)

WIREGEN_FILES := $(filter %printgen.h %printgen.c %wiregen.h %wiregen.c, $(ALL_C_HEADERS) $(ALL_C_SOURCES))

# Always make wiregen files before any object file
$(ALL_OBJS): $(WIREGEN_FILES)

# We always regen wiregen and printgen files, since SHA256STAMP protects against
# spurious rebuilds.
$(filter %printgen.h %printgen.c %wiregen.h %wiregen.c, $(ALL_C_HEADERS) $(ALL_C_SOURCES)): $(FORCE)
$(WIREGEN_FILES): $(FORCE)

ifneq ($(TEST_GROUP_COUNT),)
PYTEST_OPTS += --test-group=$(TEST_GROUP) --test-group-count=$(TEST_GROUP_COUNT)
Expand Down
6 changes: 2 additions & 4 deletions common/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ COMMON_SRC_NOGEN := \
common/wire_error.c


COMMON_SRC_GEN := common/status_wiregen.c common/peer_status_wiregen.c common/scb_wiregen.c
COMMON_SRC_GEN := common/status_wiregen.c common/peer_status_wiregen.c common/scb_wiregen.c common/gossip_store_wiregen.c

COMMON_HEADERS_NOGEN := $(COMMON_SRC_NOGEN:.c=.h) \
common/closing_fee.h \
Expand All @@ -124,7 +124,7 @@ COMMON_HEADERS_NOGEN := $(COMMON_SRC_NOGEN:.c=.h) \
common/jsonrpc_errors.h \
common/overflows.h

COMMON_HEADERS_GEN := common/htlc_state_names_gen.h common/status_wiregen.h common/peer_status_wiregen.h common/scb_wiregen.h
COMMON_HEADERS_GEN := common/htlc_state_names_gen.h common/status_wiregen.h common/peer_status_wiregen.h common/scb_wiregen.h common/gossip_store_wiregen.h

COMMON_HEADERS := $(COMMON_HEADERS_GEN) $(COMMON_HEADERS_NOGEN)
COMMON_SRC := $(COMMON_SRC_NOGEN) $(COMMON_SRC_GEN)
Expand Down Expand Up @@ -157,8 +157,6 @@ ALL_C_SOURCES += $(COMMON_SRC)
common/htlc_state_names_gen.h: common/htlc_state.h ccan/ccan/cdump/tools/cdump-enumstr
ccan/ccan/cdump/tools/cdump-enumstr common/htlc_state.h > $@

common/gossip_store.o: gossipd/gossip_store_wiregen.h

check-source-bolt: $(COMMON_SRC_NOGEN:%=bolt-check/%) $(COMMON_HEADERS:%=bolt-check/%)
check-whitespace: $(COMMON_SRC_NOGEN:%=check-whitespace/%) $(COMMON_HEADERS:%=check-whitespace/%)

Expand Down
16 changes: 4 additions & 12 deletions common/gossip_store.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,27 +1,19 @@
#include "config.h"
#include <common/gossip_store.h>
#include <gossipd/gossip_store_wiregen.h>
#include <common/gossip_store_wiregen.h>
#include <unistd.h>

/* We cheat and read first two bytes of message too. */
struct hdr_and_type {
struct gossip_hdr hdr;
be16 type;
};
/* Beware padding! */
#define HDR_AND_TYPE_SIZE (sizeof(struct gossip_hdr) + sizeof(u16))

bool gossip_store_readhdr(int gossip_store_fd, size_t off,
size_t *len,
u32 *timestamp,
u16 *flags,
u16 *type)
{
struct hdr_and_type buf;
struct gossip_hdr_and_type buf;
int r;

r = pread(gossip_store_fd, &buf, HDR_AND_TYPE_SIZE, off);
if (r != HDR_AND_TYPE_SIZE)
r = pread(gossip_store_fd, &buf, GOSSIP_HDR_AND_TYPE_SIZE, off);
if (r != GOSSIP_HDR_AND_TYPE_SIZE)
return false;
if (!(buf.hdr.flags & CPU_TO_BE16(GOSSIP_STORE_COMPLETED_BIT)))
return false;
Expand Down
17 changes: 11 additions & 6 deletions common/gossip_store.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ struct gossip_rcvd_filter;
/* First byte of file is the version.
*
* Top three bits mean incompatible change.
* As of this writing, major == 0, minor == 15.
* As of this writing, major == 0, minor == 16.
*/
#define GOSSIP_STORE_MAJOR_VERSION_MASK 0xE0
#define GOSSIP_STORE_MINOR_VERSION_MASK 0x1F
Expand Down Expand Up @@ -50,6 +50,14 @@ struct gossip_hdr {
beint32_t timestamp; /* timestamp of msg. */
};

/* Useful to read gossip_hdr and type of msg. */
struct gossip_hdr_and_type {
struct gossip_hdr hdr;
be16 type;
};
/* Beware padding! */
#define GOSSIP_HDR_AND_TYPE_SIZE (sizeof(struct gossip_hdr) + sizeof(u16))

/**
* Direct store accessor: read gossip msg hdr from store.
* @gossip_store_fd: the readable file descriptor
Expand All @@ -59,11 +67,8 @@ struct gossip_hdr {
* @flags (out): if non-NULL, set to the flags.
* @type (out): if non-NULL, set to the msg type.
*
* Returns false if there are no more gossip msgs. If you
* want to read the message, use gossip_store_next, if you
* want to skip, simply add sizeof(gossip_hdr) + *len to *off.
* Note: it's possible that entire record isn't there yet,
* so gossip_store_next can fail.
* Returns false if there are no more gossip msgs, or message
* is incomplete. To iterate, simply add sizeof(gossip_hdr) + *len to off.
*/
bool gossip_store_readhdr(int gossip_store_fd, size_t off,
size_t *len,
Expand Down
4 changes: 4 additions & 0 deletions gossipd/gossip_store_wire.csv → common/gossip_store_wire.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,11 @@ msgdata,gossip_store_delete_chan,scid,short_channel_id,

msgtype,gossip_store_ended,4105
msgdata,gossip_store_ended,equivalent_offset,u64,
msgdata,gossip_store_ended,uuid,u8,32

msgtype,gossip_store_chan_dying,4106
msgdata,gossip_store_chan_dying,scid,short_channel_id,
msgdata,gossip_store_chan_dying,blockheight,u32,

msgtype,gossip_store_uuid,4107
msgdata,gossip_store_uuid,uuid,u8,32
Loading
Loading