From 71cae7875f215a17d9ca035fe4bc5421c3323407 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 2 May 2020 20:10:16 +0930 Subject: [PATCH 1/9] tools/generate-wire.py: don't define empty enums. For bolt 13, we have no message types, just a TLV. Signed-off-by: Rusty Russell --- tools/generate-wire.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/generate-wire.py b/tools/generate-wire.py index 1e2552dbade4..e7849141091d 100755 --- a/tools/generate-wire.py +++ b/tools/generate-wire.py @@ -500,10 +500,11 @@ def post_process(self): def write(self, options, output): template = self.find_template(options) enum_sets = [] - enum_sets.append({ - 'name': options.enum_name, - 'set': self.messages.values(), - }) + if len(self.messages.values()) != 0: + enum_sets.append({ + 'name': options.enum_name, + 'set': self.messages.values(), + }) stuff = {} stuff['top_comments'] = self.top_comments stuff['options'] = options From db97e0c6988c6cd5b19d05b7810a2fa279ae2229 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 2 May 2020 20:11:16 +0930 Subject: [PATCH 2/9] tools/generate-wire.py: add --include argument for putting #includes in spec-generated files. We need this for bolt13. Signed-off-by: Rusty Russell --- tools/generate-wire.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/generate-wire.py b/tools/generate-wire.py index e7849141091d..a666da17cc75 100755 --- a/tools/generate-wire.py +++ b/tools/generate-wire.py @@ -535,6 +535,9 @@ def main(options, args=None, output=sys.stdout, lines=None): # Create a new 'master' that serves as the coordinator for the file generation master = Master() + for i in options.include: + master.add_include('#include <{}>'.format(i)) + try: while True: ln, line = next(genline) @@ -677,6 +680,7 @@ def main(options, args=None, output=sys.stdout, lines=None): action="store_true", default=False) parser.add_argument("--page", choices=['header', 'impl'], help="page to print") parser.add_argument('--expose-tlv-type', action='append', default=[]) + parser.add_argument('--include', action='append', default=[]) parser.add_argument('header_filename', help='The filename of the header') parser.add_argument('enum_name', help='The name of the enum to produce') parser.add_argument("files", help='Files to read in (or stdin)', nargs=REMAINDER) From e803cad794671685cce65a209ec203db27a04c67 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 2 May 2020 20:12:16 +0930 Subject: [PATCH 3/9] tools/generate-wire.py: don't prettify headers. The formatting makes it harder for update-mocks, eg: /* Generated stub for fmt_wireaddr_without_port */ char *fmt_wireaddr_without_port(const tal_t *ctx UNNEEDED, const struct wireaddr *a UNNEEDED) { fprintf(stderr, "fmt_wireaddr_without_port called!\n"); abort(); } /* Could not find declaration for fromwire_onionmsg_path */ /* Generated stub for json_add_member */ Signed-off-by: Rusty Russell --- tools/gen/header_template | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/gen/header_template b/tools/gen/header_template index 6175dcd5ae2a..adb5600b95b3 100644 --- a/tools/gen/header_template +++ b/tools/gen/header_template @@ -125,8 +125,7 @@ extern const struct tlv_record_type tlvs_${tlv.name}[]; % endfor void towire_${subtype.name}(u8 **p, const ${subtype.type_name()} *${subtype.name}); % if subtype.is_varsize(): -${subtype.type_name()} * -fromwire_${subtype.name}(const tal_t *ctx, const u8 **cursor, size_t *plen); +${subtype.type_name()} *fromwire_${subtype.name}(const tal_t *ctx, const u8 **cursor, size_t *plen); % else: void fromwire_${subtype.name}(const u8 **cursor, size_t *plen, ${subtype.type_name()} *${subtype.name}); % endif From 35feac24970e52be262928d90377804c1b6191db Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 2 May 2020 20:13:16 +0930 Subject: [PATCH 4/9] Makefile: COMPAT_V082. Signed-off-by: Rusty Russell --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 65d26b79b44e..95e6f1eb3461 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ endif ifeq ($(COMPAT),1) # We support compatibility with pre-0.6. -COMPAT_CFLAGS=-DCOMPAT_V052=1 -DCOMPAT_V060=1 -DCOMPAT_V061=1 -DCOMPAT_V062=1 -DCOMPAT_V070=1 -DCOMPAT_V072=1 -DCOMPAT_V073=1 -DCOMPAT_V080=1 -DCOMPAT_V081=1 +COMPAT_CFLAGS=-DCOMPAT_V052=1 -DCOMPAT_V060=1 -DCOMPAT_V061=1 -DCOMPAT_V062=1 -DCOMPAT_V070=1 -DCOMPAT_V072=1 -DCOMPAT_V073=1 -DCOMPAT_V080=1 -DCOMPAT_V081=1 -DCOMPAT_V082=1 endif # Timeout shortly before the 600 second travis silence timeout From 246af4bfc58b25597906b8d654b95b22d7a3e2bc Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 May 2020 09:47:49 +0930 Subject: [PATCH 5/9] channeld: tell gossipd what the features are for our local channels. This msg is stored in the gossip_store, so it means a version bump. Signed-off-by: Rusty Russell --- channeld/channeld.c | 6 +++++- common/gossip_store.h | 2 +- gossipd/gossip_peerd_wire.csv | 2 ++ gossipd/routing.c | 5 +++-- gossipd/test/run-bench-find_route.c | 2 +- gossipd/test/run-find_route-specific.c | 2 +- gossipd/test/run-find_route.c | 2 +- gossipd/test/run-overlong.c | 2 +- gossipd/test/run-txout_failure.c | 2 +- tests/test_gossip.py | 13 +++++++------ 10 files changed, 23 insertions(+), 15 deletions(-) diff --git a/channeld/channeld.c b/channeld/channeld.c index 24b037fd5be7..758bc4bec531 100644 --- a/channeld/channeld.c +++ b/channeld/channeld.c @@ -349,12 +349,16 @@ static const u8 *get_local_channel_update(const tal_t *ctx, struct peer *peer) static void make_channel_local_active(struct peer *peer) { u8 *msg; + const u8 *annfeatures = get_agreed_channelfeatures(tmpctx, + peer->our_features, + peer->their_features); /* Tell gossipd about local channel. */ msg = towire_gossipd_local_add_channel(NULL, &peer->short_channel_ids[LOCAL], &peer->node_ids[REMOTE], - peer->channel->funding); + peer->channel->funding, + annfeatures); wire_sync_write(peer->pps->gossip_fd, take(msg)); /* Tell gossipd and the other side what parameters we expect should diff --git a/common/gossip_store.h b/common/gossip_store.h index 4b351cd6c17d..c64137be3946 100644 --- a/common/gossip_store.h +++ b/common/gossip_store.h @@ -10,7 +10,7 @@ struct per_peer_state; /** * gossip_store -- On-disk storage related information */ -#define GOSSIP_STORE_VERSION 7 +#define GOSSIP_STORE_VERSION 8 /** * Bit of length we use to mark a deleted record. diff --git a/gossipd/gossip_peerd_wire.csv b/gossipd/gossip_peerd_wire.csv index 5e0fa4c7d807..c1028b06a733 100644 --- a/gossipd/gossip_peerd_wire.csv +++ b/gossipd/gossip_peerd_wire.csv @@ -16,6 +16,8 @@ msgtype,gossipd_local_add_channel,3503 msgdata,gossipd_local_add_channel,short_channel_id,short_channel_id, msgdata,gossipd_local_add_channel,remote_node_id,node_id, msgdata,gossipd_local_add_channel,satoshis,amount_sat, +msgdata,gossipd_local_add_channel,flen,u16, +msgdata,gossipd_local_add_channel,features,u8,flen # Send this channel_update. msgtype,gossipd_local_channel_update,3504 diff --git a/gossipd/routing.c b/gossipd/routing.c index 0e600e736568..ecbab127d1fa 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -2904,9 +2904,10 @@ bool handle_local_add_channel(struct routing_state *rstate, struct node_id remote_node_id; struct amount_sat sat; struct chan *chan; + u8 *features; - if (!fromwire_gossipd_local_add_channel(msg, &scid, &remote_node_id, - &sat)) { + if (!fromwire_gossipd_local_add_channel(msg, msg, &scid, &remote_node_id, + &sat, &features)) { status_peer_broken(peer ? &peer->id : NULL, "Unable to parse local_add_channel message: %s", tal_hex(msg, msg)); diff --git a/gossipd/test/run-bench-find_route.c b/gossipd/test/run-bench-find_route.c index 761439bfa915..87d7572139e3 100644 --- a/gossipd/test/run-bench-find_route.c +++ b/gossipd/test/run-bench-find_route.c @@ -44,7 +44,7 @@ bool fromwire_gossip_store_channel_amount(const void *p UNNEEDED, struct amount_ bool fromwire_gossip_store_private_update(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **update UNNEEDED) { fprintf(stderr, "fromwire_gossip_store_private_update called!\n"); abort(); } /* Generated stub for fromwire_gossipd_local_add_channel */ -bool fromwire_gossipd_local_add_channel(const void *p UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, struct node_id *remote_node_id UNNEEDED, struct amount_sat *satoshis UNNEEDED) +bool fromwire_gossipd_local_add_channel(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, struct node_id *remote_node_id UNNEEDED, struct amount_sat *satoshis UNNEEDED, u8 **features UNNEEDED) { fprintf(stderr, "fromwire_gossipd_local_add_channel called!\n"); abort(); } /* Generated stub for fromwire_wireaddr */ bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED) diff --git a/gossipd/test/run-find_route-specific.c b/gossipd/test/run-find_route-specific.c index 32d9b3725d9a..954a5647a1c8 100644 --- a/gossipd/test/run-find_route-specific.c +++ b/gossipd/test/run-find_route-specific.c @@ -31,7 +31,7 @@ bool fromwire_gossip_store_channel_amount(const void *p UNNEEDED, struct amount_ bool fromwire_gossip_store_private_update(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **update UNNEEDED) { fprintf(stderr, "fromwire_gossip_store_private_update called!\n"); abort(); } /* Generated stub for fromwire_gossipd_local_add_channel */ -bool fromwire_gossipd_local_add_channel(const void *p UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, struct node_id *remote_node_id UNNEEDED, struct amount_sat *satoshis UNNEEDED) +bool fromwire_gossipd_local_add_channel(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, struct node_id *remote_node_id UNNEEDED, struct amount_sat *satoshis UNNEEDED, u8 **features UNNEEDED) { fprintf(stderr, "fromwire_gossipd_local_add_channel called!\n"); abort(); } /* Generated stub for fromwire_wireaddr */ bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED) diff --git a/gossipd/test/run-find_route.c b/gossipd/test/run-find_route.c index ea6be04c5942..811bed55e321 100644 --- a/gossipd/test/run-find_route.c +++ b/gossipd/test/run-find_route.c @@ -31,7 +31,7 @@ bool fromwire_gossip_store_channel_amount(const void *p UNNEEDED, struct amount_ bool fromwire_gossip_store_private_update(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **update UNNEEDED) { fprintf(stderr, "fromwire_gossip_store_private_update called!\n"); abort(); } /* Generated stub for fromwire_gossipd_local_add_channel */ -bool fromwire_gossipd_local_add_channel(const void *p UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, struct node_id *remote_node_id UNNEEDED, struct amount_sat *satoshis UNNEEDED) +bool fromwire_gossipd_local_add_channel(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, struct node_id *remote_node_id UNNEEDED, struct amount_sat *satoshis UNNEEDED, u8 **features UNNEEDED) { fprintf(stderr, "fromwire_gossipd_local_add_channel called!\n"); abort(); } /* Generated stub for fromwire_wireaddr */ bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED) diff --git a/gossipd/test/run-overlong.c b/gossipd/test/run-overlong.c index f9e37d3537f8..e730d9f41897 100644 --- a/gossipd/test/run-overlong.c +++ b/gossipd/test/run-overlong.c @@ -31,7 +31,7 @@ bool fromwire_gossip_store_channel_amount(const void *p UNNEEDED, struct amount_ bool fromwire_gossip_store_private_update(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **update UNNEEDED) { fprintf(stderr, "fromwire_gossip_store_private_update called!\n"); abort(); } /* Generated stub for fromwire_gossipd_local_add_channel */ -bool fromwire_gossipd_local_add_channel(const void *p UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, struct node_id *remote_node_id UNNEEDED, struct amount_sat *satoshis UNNEEDED) +bool fromwire_gossipd_local_add_channel(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, struct node_id *remote_node_id UNNEEDED, struct amount_sat *satoshis UNNEEDED, u8 **features UNNEEDED) { fprintf(stderr, "fromwire_gossipd_local_add_channel called!\n"); abort(); } /* Generated stub for fromwire_wireaddr */ bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED) diff --git a/gossipd/test/run-txout_failure.c b/gossipd/test/run-txout_failure.c index f92843c4db80..8ff9712c9ab1 100644 --- a/gossipd/test/run-txout_failure.c +++ b/gossipd/test/run-txout_failure.c @@ -13,7 +13,7 @@ bool cupdate_different(struct gossip_store *gs UNNEEDED, char *fmt_wireaddr_without_port(const tal_t *ctx UNNEEDED, const struct wireaddr *a UNNEEDED) { fprintf(stderr, "fmt_wireaddr_without_port called!\n"); abort(); } /* Generated stub for fromwire_gossipd_local_add_channel */ -bool fromwire_gossipd_local_add_channel(const void *p UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, struct node_id *remote_node_id UNNEEDED, struct amount_sat *satoshis UNNEEDED) +bool fromwire_gossipd_local_add_channel(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, struct node_id *remote_node_id UNNEEDED, struct amount_sat *satoshis UNNEEDED, u8 **features UNNEEDED) { fprintf(stderr, "fromwire_gossipd_local_add_channel called!\n"); abort(); } /* Generated stub for fromwire_wireaddr */ bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED) diff --git a/tests/test_gossip.py b/tests/test_gossip.py index ad809729ce78..2a22026345fc 100644 --- a/tests/test_gossip.py +++ b/tests/test_gossip.py @@ -920,7 +920,7 @@ def test_gossip_store_load(node_factory): """Make sure we can read canned gossip store""" l1 = node_factory.get_node(start=False) with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f: - f.write(bytearray.fromhex("07" # GOSSIP_STORE_VERSION + f.write(bytearray.fromhex("08" # GOSSIP_STORE_VERSION "000001b0" # len "fea676e8" # csum "5b8d9b44" # timestamp @@ -952,7 +952,7 @@ def test_gossip_store_load_announce_before_update(node_factory): """Make sure we can read canned gossip store with node_announce before update. This happens when a channel_update gets replaced, leaving node_announce before it""" l1 = node_factory.get_node(start=False) with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f: - f.write(bytearray.fromhex("07" # GOSSIP_STORE_VERSION + f.write(bytearray.fromhex("08" # GOSSIP_STORE_VERSION "000001b0" # len "fea676e8" # csum "5b8d9b44" # timestamp @@ -995,7 +995,7 @@ def test_gossip_store_load_amount_truncated(node_factory): """Make sure we can read canned gossip store with truncated amount""" l1 = node_factory.get_node(start=False, allow_broken_log=True) with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f: - f.write(bytearray.fromhex("07" # GOSSIP_STORE_VERSION + f.write(bytearray.fromhex("08" # GOSSIP_STORE_VERSION "000001b0" # len "fea676e8" # csum "5b8d9b44" # timestamp @@ -1419,7 +1419,7 @@ def test_gossip_store_load_no_channel_update(node_factory): # A channel announcement with no channel_update. with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f: - f.write(bytearray.fromhex("07" # GOSSIP_STORE_VERSION + f.write(bytearray.fromhex("08" # GOSSIP_STORE_VERSION "000001b0" # len "fea676e8" # csum "5b8d9b44" # timestamp @@ -1446,7 +1446,7 @@ def test_gossip_store_load_no_channel_update(node_factory): l1.rpc.call('dev-compact-gossip-store') with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), "rb") as f: - assert bytearray(f.read()) == bytearray.fromhex("07") + assert bytearray(f.read()) == bytearray.fromhex("08") @unittest.skipIf(not DEVELOPER, "gossip without DEVELOPER=1 is slow") @@ -1456,7 +1456,8 @@ def test_gossip_store_compact_on_load(node_factory, bitcoind): l2.restart() wait_for(lambda: l2.daemon.is_in_log(r'gossip_store_compact_offline: [5-8] deleted, 9 copied')) - wait_for(lambda: l2.daemon.is_in_log(r'gossip_store: Read 1/4/2/0 cannounce/cupdate/nannounce/cdelete from store \(0 deleted\) in 1460 bytes')) + + wait_for(lambda: l2.daemon.is_in_log(r'gossip_store: Read 1/4/2/0 cannounce/cupdate/nannounce/cdelete from store \(0 deleted\) in [0-9]* bytes')) def test_gossip_announce_invalid_block(node_factory, bitcoind): From 3f56e89192ecf7b83d76f506e0d0f1b7bfa06482 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 May 2020 09:47:54 +0930 Subject: [PATCH 6/9] pytest: test gossip_store upgrade from version 7 to version 8. The previous patch changed the gossip_store, but in a trivial way. The next patch will implement upgrading, so this is the test. Signed-off-by: Rusty Russell --- tests/test_gossip.py | 70 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/tests/test_gossip.py b/tests/test_gossip.py index 2a22026345fc..2788527236e0 100644 --- a/tests/test_gossip.py +++ b/tests/test_gossip.py @@ -2,7 +2,7 @@ from ephemeral_port_reserve import reserve from fixtures import * # noqa: F401,F403 from fixtures import TEST_NETWORK -from pyln.client import RpcError +from pyln.client import RpcError, Millisatoshi from utils import ( wait_for, TIMEOUT, only_one, sync_blockheight, expected_node_features ) @@ -1657,3 +1657,71 @@ def test_torport_onions(node_factory): assert l1.daemon.is_in_log('45321,127.0.0.1:{}'.format(l1.port)) assert l2.daemon.is_in_log('x2y4zvh4fn5q3eouuh7nxnc7zeawrqoutljrup2xjtiyxgx3emgkemad.onion:45321,127.0.0.1:{}'.format(l2.port)) + + +@pytest.mark.xfail(strict=True) +def test_gossip_store_upgrade_v7_v8(node_factory): + """Version 8 added feature bits to local channel announcements""" + l1 = node_factory.get_node(start=False) + + # A channel announcement with no channel_update. + with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f: + f.write(bytearray.fromhex("07000000428ce4d2d8000000000daf00" + "00670000010001022d223620a359a47f" + "f7f7ac447c85c46c923da53389221a00" + "54c11c1e3ca31d5900000000000f4240" + "000d8000000000000000000000000000" + "00008e3af3badf000000001006008a01" + "02005a9911d425effd461f803a380f05" + "e72d3332eb6e9a7c6c58405ae61eacde" + "4e2da18240ffb3d5c595f85e4f78b594" + "c59e4d01c0470edd4f5afe645026515e" + "fe06226e46111a0b59caaf126043eb5b" + "bf28c34f3a5e332a1fc7b2b73cf18891" + "0f00006700000100015eaa5eb0010100" + "06000000000000000000000001000000" + "0a000000003b0233800000008e074a6e" + "0f000000001006008a0102463de636b2" + "f46ccd6c23259787fc39dc4fdb983510" + "1651879325b18cf1bb26330127e51ce8" + "7a111b05ef92fe00a9a089979dc49178" + "200f49139a541e7078cdc506226e4611" + "1a0b59caaf126043eb5bbf28c34f3a5e" + "332a1fc7b2b73cf188910f0000670000" + "0100015eaa5eb0010000060000000000" + "000000000000010000000a000000003b" + "023380")) + + l1.start() + + assert l1.rpc.listchannels()['channels'] == [ + {'source': '022d223620a359a47ff7f7ac447c85c46c923da53389221a0054c11c1e3ca31d59', + 'destination': '0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518', + 'short_channel_id': '103x1x1', + 'public': False, + 'satoshis': 1000000, + 'amount_msat': Millisatoshi(1000000000), + 'message_flags': 1, + 'channel_flags': 0, + 'active': False, + 'last_update': 1588223664, + 'base_fee_millisatoshi': 1, + 'fee_per_millionth': 10, + 'delay': 6, + 'htlc_minimum_msat': Millisatoshi(0), + 'htlc_maximum_msat': Millisatoshi(990000000)}, + {'source': '0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518', + 'destination': '022d223620a359a47ff7f7ac447c85c46c923da53389221a0054c11c1e3ca31d59', + 'short_channel_id': '103x1x1', + 'public': False, + 'satoshis': 1000000, + 'amount_msat': Millisatoshi(1000000000), + 'message_flags': 1, + 'channel_flags': 1, + 'active': False, + 'last_update': 1588223664, + 'base_fee_millisatoshi': 1, + 'fee_per_millionth': 10, + 'delay': 6, + 'htlc_minimum_msat': Millisatoshi(0), + 'htlc_maximum_msat': Millisatoshi(990000000)}] From 3f3a6d5c2e625c8f434a720b86d34b3006984ecd Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 May 2020 09:48:25 +0930 Subject: [PATCH 7/9] gossipd: upgrade v7 gossip_store to v8. Signed-off-by: Rusty Russell --- gossipd/gossip_store.c | 49 +++++++++++++++++++++++++++++++++++++++--- tests/test_gossip.py | 4 ++-- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/gossipd/gossip_store.c b/gossipd/gossip_store.c index 278c8436e817..67531a958c6a 100644 --- a/gossipd/gossip_store.c +++ b/gossipd/gossip_store.c @@ -116,6 +116,36 @@ static bool append_msg(int fd, const u8 *msg, u32 timestamp, return true; } +#ifdef COMPAT_V082 +/* The upgrade from version 7 is trivial */ +static bool can_upgrade(u8 oldversion) +{ + return oldversion == 7; +} + +static bool upgrade_field(u8 oldversion, u8 **msg) +{ + assert(can_upgrade(oldversion)); + + /* We only need to upgrade this */ + if (fromwire_peektype(*msg) == WIRE_GOSSIPD_LOCAL_ADD_CHANNEL) { + /* Append two 0 bytes, for (empty) feature bits */ + tal_resizez(msg, tal_bytelen(*msg) + 2); + } + return true; +} +#else +static bool can_upgrade(u8 oldversion) +{ + return false; +} + +static bool upgrade_field(u8 oldversion, u8 **msg) +{ + abort(); +} +#endif /* !COMPAT_V082 */ + /* Read gossip store entries, copy non-deleted ones. This code is written * as simply and robustly as possible! */ static u32 gossip_store_compact_offline(void) @@ -123,7 +153,7 @@ static u32 gossip_store_compact_offline(void) size_t count = 0, deleted = 0; int old_fd, new_fd; struct gossip_hdr hdr; - u8 version; + u8 oldversion, version = GOSSIP_STORE_VERSION; struct stat st; old_fd = open(GOSSIP_STORE_FILENAME, O_RDONLY); @@ -143,8 +173,8 @@ static u32 gossip_store_compact_offline(void) goto close_old; } - if (!read_all(old_fd, &version, sizeof(version)) - || version != GOSSIP_STORE_VERSION) { + if (!read_all(old_fd, &oldversion, sizeof(oldversion)) + || (oldversion != version && !can_upgrade(oldversion))) { status_broken("gossip_store_compact: bad version"); goto close_and_delete; } @@ -175,6 +205,19 @@ static u32 gossip_store_compact_offline(void) continue; } + if (oldversion != version) { + if (!upgrade_field(oldversion, &msg)) { + tal_free(msg); + goto close_and_delete; + } + + /* Recalc msglen and header */ + msglen = tal_bytelen(msg); + hdr.len = cpu_to_be32(msglen); + hdr.crc = cpu_to_be32(crc32c(be32_to_cpu(hdr.timestamp), + msg, msglen)); + } + if (!write_all(new_fd, &hdr, sizeof(hdr)) || !write_all(new_fd, msg, msglen)) { status_broken("gossip_store_compact_offline: writing msg len %zu to new store: %s", diff --git a/tests/test_gossip.py b/tests/test_gossip.py index 2788527236e0..8e7187f9b418 100644 --- a/tests/test_gossip.py +++ b/tests/test_gossip.py @@ -4,7 +4,7 @@ from fixtures import TEST_NETWORK from pyln.client import RpcError, Millisatoshi from utils import ( - wait_for, TIMEOUT, only_one, sync_blockheight, expected_node_features + wait_for, TIMEOUT, only_one, sync_blockheight, expected_node_features, COMPAT ) import json @@ -1659,7 +1659,7 @@ def test_torport_onions(node_factory): assert l2.daemon.is_in_log('x2y4zvh4fn5q3eouuh7nxnc7zeawrqoutljrup2xjtiyxgx3emgkemad.onion:45321,127.0.0.1:{}'.format(l2.port)) -@pytest.mark.xfail(strict=True) +@unittest.skipIf(not COMPAT, "needs COMPAT to convert obsolete gossip_store") def test_gossip_store_upgrade_v7_v8(node_factory): """Version 8 added feature bits to local channel announcements""" l1 = node_factory.get_node(start=False) From f0895ed4fe8b651a770afad852cf4f9c670ead2b Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 May 2020 09:48:34 +0930 Subject: [PATCH 8/9] gossipd: keep a flag to indicate that we have features in channel_announcement. This saves us keeping it in memory (so far, no channels have features), but lets us optimize that case so we don't need to hit the disk for most of the channels in listchannels. Signed-off-by: Rusty Russell --- gossipd/routing.c | 13 ++++++++++--- gossipd/routing.h | 7 ++++++- gossipd/test/run-bench-find_route.c | 2 +- gossipd/test/run-find_route-specific.c | 2 +- gossipd/test/run-find_route.c | 2 +- gossipd/test/run-overlong.c | 4 ++-- 6 files changed, 21 insertions(+), 9 deletions(-) diff --git a/gossipd/routing.c b/gossipd/routing.c index ecbab127d1fa..c571f0ad366b 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -93,6 +93,8 @@ HTABLE_DEFINE_TYPE(struct pending_node_announce, pending_node_announce_keyof, struct unupdated_channel { /* The channel_announcement message */ const u8 *channel_announce; + /* The feature bitmap within it */ + const u8 *features; /* The short_channel_id */ struct short_channel_id scid; /* The ids of the nodes */ @@ -571,7 +573,8 @@ struct chan *new_chan(struct routing_state *rstate, const struct short_channel_id *scid, const struct node_id *id1, const struct node_id *id2, - struct amount_sat satoshis) + struct amount_sat satoshis, + const u8 *features) { struct chan *chan = tal(rstate, struct chan); int n1idx = node_id_idx(id1, id2); @@ -606,6 +609,8 @@ struct chan *new_chan(struct routing_state *rstate, init_half_chan(rstate, chan, n1idx); init_half_chan(rstate, chan, !n1idx); + /* Stash hint here about whether we have features */ + chan->half[0].any_features = tal_bytelen(features) != 0; uintmap_add(&rstate->chanmap, scid->u64, chan); /* Initialize shadow structure if it's local */ @@ -1651,6 +1656,7 @@ bool routing_add_channel_announcement(struct routing_state *rstate, uc = tal(rstate, struct unupdated_channel); uc->channel_announce = tal_dup_talarr(uc, u8, msg); + uc->features = tal_steal(uc, features); uc->added = gossip_time_now(rstate); uc->index = index; uc->sat = sat; @@ -2098,7 +2104,7 @@ bool routing_add_channel_update(struct routing_state *rstate, if (uc) { assert(!chan); chan = new_chan(rstate, &short_channel_id, - &uc->id[0], &uc->id[1], sat); + &uc->id[0], &uc->id[1], sat, uc->features); } /* Discard older updates */ @@ -2926,7 +2932,8 @@ bool handle_local_add_channel(struct routing_state *rstate, type_to_string(tmpctx, struct short_channel_id, &scid)); /* Create new (unannounced) channel */ - chan = new_chan(rstate, &scid, &rstate->local_id, &remote_node_id, sat); + chan = new_chan(rstate, &scid, &rstate->local_id, &remote_node_id, sat, + features); if (!index) index = gossip_store_add(rstate->gs, msg, 0, false, NULL); chan->bcast.index = index; diff --git a/gossipd/routing.h b/gossipd/routing.h index 33e461891b56..3e3eb1b4efe0 100644 --- a/gossipd/routing.h +++ b/gossipd/routing.h @@ -41,6 +41,10 @@ struct half_chan { /* Token bucket */ u8 tokens; + /* Feature cache for parent chan: squeezed in here where it would + * otherwise simply be padding. */ + u8 any_features; + /* Minimum and maximum number of msatoshi in an HTLC */ struct amount_msat htlc_minimum, htlc_maximum; }; @@ -361,7 +365,8 @@ struct chan *new_chan(struct routing_state *rstate, const struct short_channel_id *scid, const struct node_id *id1, const struct node_id *id2, - struct amount_sat sat); + struct amount_sat sat, + const u8 *features); /* Handlers for incoming messages */ diff --git a/gossipd/test/run-bench-find_route.c b/gossipd/test/run-bench-find_route.c index 87d7572139e3..89c26b483e92 100644 --- a/gossipd/test/run-bench-find_route.c +++ b/gossipd/test/run-bench-find_route.c @@ -149,7 +149,7 @@ static void add_connection(struct routing_state *rstate, chan = get_channel(rstate, &scid); if (!chan) { chan = new_chan(rstate, &scid, &nodes[from], &nodes[to], - AMOUNT_SAT(1000000)); + AMOUNT_SAT(1000000), NULL); } c = &chan->half[idx]; diff --git a/gossipd/test/run-find_route-specific.c b/gossipd/test/run-find_route-specific.c index 954a5647a1c8..3eaf9dd8095f 100644 --- a/gossipd/test/run-find_route-specific.c +++ b/gossipd/test/run-find_route-specific.c @@ -134,7 +134,7 @@ get_or_make_connection(struct routing_state *rstate, abort(); chan = get_channel(rstate, &scid); if (!chan) - chan = new_chan(rstate, &scid, from_id, to_id, satoshis); + chan = new_chan(rstate, &scid, from_id, to_id, satoshis, NULL); /* Make sure it's seen as initialized (index non-zero). */ chan->half[idx].bcast.index = 1; diff --git a/gossipd/test/run-find_route.c b/gossipd/test/run-find_route.c index 811bed55e321..513e42df3411 100644 --- a/gossipd/test/run-find_route.c +++ b/gossipd/test/run-find_route.c @@ -142,7 +142,7 @@ static void add_connection(struct routing_state *rstate, chan = get_channel(rstate, &scid); if (!chan) - chan = new_chan(rstate, &scid, from, to, satoshis); + chan = new_chan(rstate, &scid, from, to, satoshis, NULL); c = &chan->half[node_id_idx(from, to)]; /* Make sure it's seen as initialized (index non-zero). */ diff --git a/gossipd/test/run-overlong.c b/gossipd/test/run-overlong.c index e730d9f41897..262f70d9ce96 100644 --- a/gossipd/test/run-overlong.c +++ b/gossipd/test/run-overlong.c @@ -163,7 +163,7 @@ int main(void) if (!mk_short_channel_id(&scid, i, i-1, 0)) abort(); chan = new_chan(rstate, &scid, &ids[i], &ids[i-1], - AMOUNT_SAT(1000000)); + AMOUNT_SAT(1000000), NULL); hc = &chan->half[node_id_idx(&ids[i-1], &ids[i])]; hc->bcast.index = 1; @@ -183,7 +183,7 @@ int main(void) if (!mk_short_channel_id(&scid, i, 1, 0)) abort(); chan = new_chan(rstate, &scid, &ids[i], &ids[1], - AMOUNT_SAT(1000000)); + AMOUNT_SAT(1000000), NULL); hc = &chan->half[node_id_idx(&ids[1], &ids[i])]; hc->bcast.index = 1; hc->base_fee = 1 << i; From fc11b90ac22357b6f518b5e1468d27cbd2c347d5 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 4 May 2020 09:49:32 +0930 Subject: [PATCH 9/9] gossipd: return channel_announcement features for listchannels. Signed-off-by: Rusty Russell Changelog-Added: JSON API: `listchannels` now shows channel `features`. --- gossipd/gossipd.c | 36 ++++++++++++++++++++++++++++++++++++ lightningd/gossip_control.c | 1 + lightningd/gossip_msg.c | 4 ++++ lightningd/gossip_msg.h | 1 + tests/test_gossip.py | 7 +++++-- tests/utils.py | 9 +++++++++ 6 files changed, 56 insertions(+), 2 deletions(-) diff --git a/gossipd/gossipd.c b/gossipd/gossipd.c index c2d77373ad91..257c25a5d33f 100644 --- a/gossipd/gossipd.c +++ b/gossipd/gossipd.c @@ -966,6 +966,41 @@ static struct gossip_halfchannel_entry *hc_entry(const tal_t *ctx, return e; } +/*~ We don't keep channel features in memory; they're rarely used. So we + * remember if it exists, and load it off disk when needed. */ +static u8 *get_channel_features(const tal_t *ctx, + struct gossip_store *gs, + const struct chan *chan) +{ + secp256k1_ecdsa_signature sig; + u8 *features; + struct bitcoin_blkid chain_hash; + struct short_channel_id short_channel_id; + struct node_id node_id; + struct pubkey bitcoin_key; + struct amount_sat sats; + const u8 *ann; + + /* This is where we stash a flag to indicate it exists. */ + if (!chan->half[0].any_features) + return NULL; + + /* Could be a channel_announcement, could be a local_add_channel */ + ann = gossip_store_get(tmpctx, gs, chan->bcast.index); + if (!fromwire_channel_announcement(ctx, ann, &sig, &sig, &sig, &sig, + &features, &chain_hash, + &short_channel_id, + &node_id, &node_id, + &bitcoin_key, &bitcoin_key) + && !fromwire_gossipd_local_add_channel(ctx, ann, &short_channel_id, + &node_id, &sats, &features)) + status_failed(STATUS_FAIL_INTERNAL_ERROR, + "bad channel_announcement / local_add_channel at %u: %s", + chan->bcast.index, tal_hex(tmpctx, ann)); + + return features; +} + /*~ Marshal (possibly) both channel directions into entries. */ static void append_channel(struct routing_state *rstate, const struct gossip_getchannels_entry ***entries, @@ -980,6 +1015,7 @@ static void append_channel(struct routing_state *rstate, e->local_disabled = is_chan_local_disabled(rstate, chan); e->public = is_chan_public(chan); e->short_channel_id = chan->scid; + e->features = get_channel_features(e, rstate->gs, chan); if (!srcfilter || node_id_eq(&e->node[0], srcfilter)) e->e[0] = hc_entry(*entries, chan, 0); else diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index 946552336f18..4ed1b4bf5193 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -482,6 +482,7 @@ static void json_add_halfchan(struct json_stream *response, json_add_num(response, "delay", he->delay); json_add_amount_msat_only(response, "htlc_minimum_msat", he->min); json_add_amount_msat_only(response, "htlc_maximum_msat", he->max); + json_add_hex_talarr(response, "features", e->features); json_object_end(response); } diff --git a/lightningd/gossip_msg.c b/lightningd/gossip_msg.c index 49bfb8c74b6a..834236dfc9ae 100644 --- a/lightningd/gossip_msg.c +++ b/lightningd/gossip_msg.c @@ -143,6 +143,8 @@ fromwire_gossip_getchannels_entry(const tal_t *ctx, fromwire_short_channel_id(pptr, max, &entry->short_channel_id); entry->public = fromwire_bool(pptr, max); entry->local_disabled = fromwire_bool(pptr, max); + entry->features = fromwire_tal_arrn(entry, + pptr, max, fromwire_u16(pptr, max)); if (fromwire_bool(pptr, max)) { entry->e[0] = tal(entry, struct gossip_halfchannel_entry); @@ -180,6 +182,8 @@ void towire_gossip_getchannels_entry(u8 **pptr, towire_short_channel_id(pptr, &entry->short_channel_id); towire_bool(pptr, entry->public); towire_bool(pptr, entry->local_disabled); + towire_u16(pptr, tal_bytelen(entry->features)); + towire_u8_array(pptr, entry->features, tal_bytelen(entry->features)); if (entry->e[0]) { towire_bool(pptr, true); towire_gossip_halfchannel_entry(pptr, entry->e[0]); diff --git a/lightningd/gossip_msg.h b/lightningd/gossip_msg.h index 1693b9748a5b..5f405619212a 100644 --- a/lightningd/gossip_msg.h +++ b/lightningd/gossip_msg.h @@ -33,6 +33,7 @@ struct gossip_getchannels_entry { bool local_disabled; /* NULL if we haven't received an update */ struct gossip_halfchannel_entry *e[2]; + u8 *features; }; struct gossip_getnodes_entry * diff --git a/tests/test_gossip.py b/tests/test_gossip.py index 8e7187f9b418..fee9d9f16bc8 100644 --- a/tests/test_gossip.py +++ b/tests/test_gossip.py @@ -1709,7 +1709,9 @@ def test_gossip_store_upgrade_v7_v8(node_factory): 'fee_per_millionth': 10, 'delay': 6, 'htlc_minimum_msat': Millisatoshi(0), - 'htlc_maximum_msat': Millisatoshi(990000000)}, + 'htlc_maximum_msat': Millisatoshi(990000000), + # This store was created on an experimental branch (OPT_ONION_MESSAGES) + 'features': '80000000000000000000000000'}, {'source': '0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518', 'destination': '022d223620a359a47ff7f7ac447c85c46c923da53389221a0054c11c1e3ca31d59', 'short_channel_id': '103x1x1', @@ -1724,4 +1726,5 @@ def test_gossip_store_upgrade_v7_v8(node_factory): 'fee_per_millionth': 10, 'delay': 6, 'htlc_minimum_msat': Millisatoshi(0), - 'htlc_maximum_msat': Millisatoshi(990000000)}] + 'htlc_maximum_msat': Millisatoshi(990000000), + 'features': '80000000000000000000000000'}] diff --git a/tests/utils.py b/tests/utils.py index 814f05ded8d0..80daf2c8729b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -18,3 +18,12 @@ def expected_node_features(): """Return the expected node features hexstring for this configuration""" # features 1, 3, 7, 9, 11, 13, 15, 17 and 55 (0x8000000002aaa2). return "8000000002aaa2" + + +def expected_channel_features(): + """Return the expected channel features hexstring for this configuration""" + # experimental OPT_ONION_MESSAGES + if EXPERIMENTAL_FEATURES: + return '80000000000000000000000000' + else: + return ''