common/gossip_store: clean up header.

It's actually two separate u16 fields, so actually treat it as
such!

Cleans up zombie handling code a bit too.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

common/gossip_store.c

@@ -118,21 +118,22 @@ u8 *gossip_store_next(const tal_t *ctx,
 
 	while (!msg) {
 		struct gossip_hdr hdr;
-		u32 msglen, checksum, timestamp;
+		u16 msglen, flags;
+		u32 checksum, timestamp;
 		bool push, ratelimited;
 		int type, r;
 
 		r = pread(*gossip_store_fd, &hdr, sizeof(hdr), *off);
 		if (r != sizeof(hdr))
 			return NULL;
 
-		msglen = be32_to_cpu(hdr.len);
-		push = (msglen & GOSSIP_STORE_LEN_PUSH_BIT);
-		ratelimited = (msglen & GOSSIP_STORE_LEN_RATELIMIT_BIT);
-		msglen &= GOSSIP_STORE_LEN_MASK;
+		msglen = be16_to_cpu(hdr.len);
+		flags = be16_to_cpu(hdr.flags);
+		push = (flags & GOSSIP_STORE_PUSH_BIT);
+		ratelimited = (flags & GOSSIP_STORE_RATELIMIT_BIT);
 
 		/* Skip any deleted entries. */
-		if (be32_to_cpu(hdr.len) & GOSSIP_STORE_LEN_DELETED_BIT) {
+		if (flags & GOSSIP_STORE_DELETED_BIT) {
 			*off += r + msglen;
 			continue;
 		}
@@ -146,14 +147,6 @@ u8 *gossip_store_next(const tal_t *ctx,
 			continue;
 		}
 
-		/* Messages can be up to 64k, but we also have internal ones:
-		 * 128k is plenty. */
-		if (msglen > 128 * 1024)
-			status_failed(STATUS_FAIL_INTERNAL_ERROR,
-				      "gossip_store: oversize msg len %u at"
-				      " offset %zu (was at %zu)",
-				      msglen, *off, initial_off);
-
 		checksum = be32_to_cpu(hdr.crc);
 		msg = tal_arr(ctx, u8, msglen);
 		r = pread(*gossip_store_fd, msg, msglen, *off + r);
@@ -201,7 +194,7 @@ size_t find_gossip_store_end(int gossip_store_fd, size_t off)
 	while ((r = pread(gossip_store_fd, &buf,
 			 sizeof(buf.hdr) + sizeof(buf.type), off))
 	       == sizeof(buf.hdr) + sizeof(buf.type)) {
-		u32 msglen = be32_to_cpu(buf.hdr.len) & GOSSIP_STORE_LEN_MASK;
+		u16 msglen = be16_to_cpu(buf.hdr.len);
 
 		/* Don't swallow end marker! */
 		if (buf.type == CPU_TO_BE16(WIRE_GOSSIP_STORE_ENDED))
@@ -227,7 +220,8 @@ size_t find_gossip_store_by_timestamp(int gossip_store_fd,
 	while ((r = pread(gossip_store_fd, &buf,
 			  sizeof(buf.hdr) + sizeof(buf.type), off))
 	       == sizeof(buf.hdr) + sizeof(buf.type)) {
-		u32 msglen = be32_to_cpu(buf.hdr.len) & GOSSIP_STORE_LEN_MASK;
+		u16 msglen = be16_to_cpu(buf.hdr.len);
+		u16 flags = be16_to_cpu(buf.hdr.flags);
 		u16 type = be16_to_cpu(buf.type);
 
 		/* Don't swallow end marker!  Reset, as they will call
@@ -236,7 +230,7 @@ size_t find_gossip_store_by_timestamp(int gossip_store_fd,
 			return 1;
 
 		/* Only to-be-broadcast types have valid timestamps! */
-		if (!(be32_to_cpu(buf.hdr.len) & GOSSIP_STORE_LEN_DELETED_BIT)
+		if (!(flags & GOSSIP_STORE_DELETED_BIT)
 		    && public_msg_type(type)
 		    && be32_to_cpu(buf.hdr.timestamp) >= timestamp) {
 			break;

common/gossip_store.h

@@ -25,39 +25,32 @@ struct gossip_rcvd_filter;
 #define GOSSIP_STORE_MINOR_VERSION(verbyte) ((verbyte) & GOSSIP_STORE_MINOR_VERSION_MASK)
 
 /**
- * Bit of length we use to mark a deleted record.
+ * Bit of flags we use to mark a deleted record.
  */
-#define GOSSIP_STORE_LEN_DELETED_BIT 0x80000000U
+#define GOSSIP_STORE_DELETED_BIT 0x8000U
 
 /**
- * Bit of length we use to mark an important record.
+ * Bit of flags we use to mark an important record.
  */
-#define GOSSIP_STORE_LEN_PUSH_BIT 0x40000000U
+#define GOSSIP_STORE_PUSH_BIT 0x4000U
 
 /**
- * Bit of length used to define a rate-limited record (do not rebroadcast)
+ * Bit of flags used to define a rate-limited record (do not rebroadcast)
  */
-#define GOSSIP_STORE_LEN_RATELIMIT_BIT 0x20000000U
+#define GOSSIP_STORE_RATELIMIT_BIT 0x2000U
 
 /**
- * Bit used to mark a channel announcement as inactive (needs channel updates.)
+ * Bit of flags used to mark a channel announcement as inactive (needs channel updates.)
  */
-#define GOSSIP_STORE_LEN_ZOMBIE_BIT 0x10000000U
+#define GOSSIP_STORE_ZOMBIE_BIT 0x1000U
 
-/**
- * Full flags mask
- */
-#define GOSSIP_STORE_FLAGS_MASK 0xFFFF0000U
-
-/* Mask for extracting just the length part of len field */
-#define GOSSIP_STORE_LEN_MASK \
-	(~(GOSSIP_STORE_FLAGS_MASK))
 
 /**
  * gossip_hdr -- On-disk format header.
  */
 struct gossip_hdr {
-	beint32_t len; /* Length of message after header. */
+	beint16_t flags; /* Length of message after header. */
+	beint16_t len; /* Length of message after header. */
 	beint32_t crc; /* crc of message of timestamp, after header. */
 	beint32_t timestamp; /* timestamp of msg. */
 };

common/gossmap.c

@@ -611,16 +611,16 @@ static bool map_catchup(struct gossmap *map, size_t *num_rejected)
 	     map->map_end += reclen) {
 		struct gossip_hdr ghdr;
 		size_t off;
-		u16 type;
+		u16 type, flags;
 
 		map_copy(map, map->map_end, &ghdr, sizeof(ghdr));
-		reclen = (be32_to_cpu(ghdr.len) & GOSSIP_STORE_LEN_MASK)
-			+ sizeof(ghdr);
+		reclen = be16_to_cpu(ghdr.len) + sizeof(ghdr);
 
-		if (be32_to_cpu(ghdr.len) & GOSSIP_STORE_LEN_DELETED_BIT)
+		flags = be16_to_cpu(ghdr.flags);
+		if (flags & GOSSIP_STORE_DELETED_BIT)
 			continue;
 
-		if (be32_to_cpu(ghdr.len) & GOSSIP_STORE_LEN_ZOMBIE_BIT)
+		if (flags & GOSSIP_STORE_ZOMBIE_BIT)
 			continue;
 
 		/* Partial write, this can happen. */
@@ -1014,7 +1014,7 @@ bool gossmap_chan_get_capacity(const struct gossmap *map,
 	/* Skip over this record to next; expect a gossip_store_channel_amount */
 	off = c->cann_off - sizeof(ghdr);
 	map_copy(map, off, &ghdr, sizeof(ghdr));
-	off += sizeof(ghdr) + (be32_to_cpu(ghdr.len) & GOSSIP_STORE_LEN_MASK);
+	off += sizeof(ghdr) + be16_to_cpu(ghdr.len);
 
 	/* Partial write, this can happen. */
 	if (off + sizeof(ghdr) + 2 > map->map_size)
@@ -1128,7 +1128,7 @@ u8 *gossmap_chan_get_announce(const tal_t *ctx,
 			      const struct gossmap *map,
 			      const struct gossmap_chan *c)
 {
-	u32 len;
+	u16 len;
 	u8 *msg;
 	u32 pre_off;
 
@@ -1137,7 +1137,8 @@ u8 *gossmap_chan_get_announce(const tal_t *ctx,
 		pre_off = 2 + 8 + 2 + sizeof(struct gossip_hdr);
 	else
 		pre_off = sizeof(struct gossip_hdr);
-	len = (map_be32(map, c->cann_off - pre_off) & GOSSIP_STORE_LEN_MASK);
+	len = map_be16(map, c->cann_off - pre_off
+		       + offsetof(struct gossip_hdr, len));
 
 	msg = tal_arr(ctx, u8, len);
 	map_copy(map, c->cann_off, msg, len);
@@ -1149,14 +1150,14 @@ u8 *gossmap_node_get_announce(const tal_t *ctx,
 			      const struct gossmap *map,
 			      const struct gossmap_node *n)
 {
-	u32 len;
+	u16 len;
 	u8 *msg;
 
 	if (n->nann_off == 0)
 		return NULL;
 
-	len = (map_be32(map, n->nann_off - sizeof(struct gossip_hdr))
-	       & GOSSIP_STORE_LEN_MASK);
+	len = map_be16(map, n->nann_off - sizeof(struct gossip_hdr)
+		       + offsetof(struct gossip_hdr, len));
 	msg = tal_arr(ctx, u8, len);
 
 	map_copy(map, n->nann_off, msg, len);

common/test/run-route-specific.c

@@ -51,7 +51,8 @@ static void write_to_store(int store_fd, const u8 *msg)
 {
 	struct gossip_hdr hdr;
 
-	hdr.len = cpu_to_be32(tal_count(msg));
+	hdr.flags = cpu_to_be16(0);
+	hdr.len = cpu_to_be16(tal_count(msg));
 	/* We don't actually check these! */
 	hdr.crc = 0;
 	hdr.timestamp = 0;

common/test/run-route.c

@@ -44,7 +44,8 @@ static void write_to_store(int store_fd, const u8 *msg)
 {
 	struct gossip_hdr hdr;
 
-	hdr.len = cpu_to_be32(tal_count(msg));
+	hdr.flags = cpu_to_be16(0);
+	hdr.len = cpu_to_be16(tal_count(msg));
 	/* We don't actually check these! */
 	hdr.crc = 0;
 	hdr.timestamp = 0;

devtools/dump-gossipstore.c

@@ -65,17 +65,17 @@ int main(int argc, char *argv[])
 	while (read(fd, &hdr, sizeof(hdr)) == sizeof(hdr)) {
 		struct amount_sat sat;
 		struct short_channel_id scid;
-		u32 msglen = be32_to_cpu(hdr.len);
+		u16 flags = be16_to_cpu(hdr.flags);
+		u16 msglen = be16_to_cpu(hdr.len);
 		u8 *msg, *inner;
 		bool deleted, push, ratelimit, zombie;
 		u32 blockheight;
 
-		deleted = (msglen & GOSSIP_STORE_LEN_DELETED_BIT);
-		push = (msglen & GOSSIP_STORE_LEN_PUSH_BIT);
-		ratelimit = (msglen & GOSSIP_STORE_LEN_RATELIMIT_BIT);
-		zombie = (msglen & GOSSIP_STORE_LEN_ZOMBIE_BIT);
+		deleted = (flags & GOSSIP_STORE_DELETED_BIT);
+		push = (flags & GOSSIP_STORE_PUSH_BIT);
+		ratelimit = (flags & GOSSIP_STORE_RATELIMIT_BIT);
+		zombie = (msglen & GOSSIP_STORE_ZOMBIE_BIT);
 
-		msglen &= GOSSIP_STORE_LEN_MASK;
 		msg = tal_arr(NULL, u8, msglen);
 		if (read(fd, msg, msglen) != msglen)
 			errx(1, "%zu: Truncated file?", off);