update with upstream

Makefile

@@ -35,7 +35,7 @@ endif
 
 ifeq ($(COMPAT),1)
 # We support compatibility with pre-0.6.
-COMPAT_CFLAGS=-DCOMPAT_V052=1
+COMPAT_CFLAGS=-DCOMPAT_V052=1 -DCOMPAT_V060=1
 endif
 
 PYTEST_OPTS := -v -x
@@ -216,7 +216,7 @@ check:
 
 pytest: $(ALL_PROGRAMS)
 ifndef PYTEST
-	@echo "py.test is required to run the integration tests, please install using 'pip3 install -r tests/requirements.txt'"
+	@echo "py.test is required to run the integration tests, please install using 'pip3 install -r tests/requirements.txt', and rerun 'configure'."
 	exit 1
 else
 # Explicitly hand DEVELOPER and VALGRIND so you can override on make cmd line.
@@ -307,7 +307,7 @@ ncc: external/libwally-core/src/libwallycore.la
 
 # Ignore test/ directories.
 TAGS: FORCE
-	$(RM) TAGS; find * -name test -type d -prune -o -name '*.[ch]' -print | xargs etags --append
+	$(RM) TAGS; find * -name test -type d -prune -o -name '*.[ch]' -print -o -name '*.py' -print | xargs etags --append
 FORCE::
 
 ccan/ccan/cdump/tools/cdump-enumstr: ccan/ccan/cdump/tools/cdump-enumstr.o $(CDUMP_OBJS) $(CCAN_OBJS)

channeld/Makefile

@@ -20,7 +20,7 @@ LIGHTNINGD_CHANNEL_HEADERS_NOGEN :=			\
 
 LIGHTNINGD_CHANNEL_HEADERS := $(LIGHTNINGD_CHANNEL_HEADERS_GEN) $(LIGHTNINGD_CHANNEL_HEADERS_NOGEN)
 
-LIGHTNINGD_CHANNEL_SRC := channeld/channel.c	\
+LIGHTNINGD_CHANNEL_SRC := channeld/channeld.c	\
 	channeld/commit_tx.c			\
 	channeld/full_channel.c		\
 	channeld/gen_channel_wire.c

channeld/channel.c → channeld/channeld.c

@@ -26,7 +26,6 @@
 #include <channeld/full_channel.h>
 #include <channeld/gen_channel_wire.h>
 #include <common/crypto_sync.h>
-#include <common/derive_basepoints.h>
 #include <common/dev_disconnect.h>
 #include <common/htlc_tx.h>
 #include <common/key_derive.h>

closingd/Makefile

@@ -16,7 +16,7 @@ LIGHTNINGD_CLOSING_HEADERS_NOGEN :=
 
 LIGHTNINGD_CLOSING_HEADERS := $(LIGHTNINGD_CLOSING_HEADERS_GEN) $(LIGHTNINGD_CLOSING_HEADERS_NOGEN)
 
-LIGHTNINGD_CLOSING_SRC := closingd/closing.c	\
+LIGHTNINGD_CLOSING_SRC := closingd/closingd.c	\
 	$(LIGHTNINGD_CLOSING_HEADERS:.h=.c)
 LIGHTNINGD_CLOSING_OBJS := $(LIGHTNINGD_CLOSING_SRC:.c=.o)
 

common/daemon.c

@@ -7,7 +7,6 @@
 #include <common/status.h>
 #include <common/utils.h>
 #include <common/version.h>
-#include <poll.h>
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -66,7 +65,7 @@ static void crashlog_activate(void)
 }
 #endif
 
-static int daemon_poll(struct pollfd *fds, nfds_t nfds, int timeout)
+int daemon_poll(struct pollfd *fds, nfds_t nfds, int timeout)
 {
 	const char *t;
 

common/daemon.h

@@ -1,12 +1,16 @@
 #ifndef LIGHTNING_COMMON_DAEMON_H
 #define LIGHTNING_COMMON_DAEMON_H
 #include "config.h"
+#include <poll.h>
 
 /* Common setup for all daemons */
 void daemon_setup(const char *argv0,
 		  void (*backtrace_print)(const char *fmt, ...),
 		  void (*backtrace_exit)(void));
 
+/* Exposed for lightningd's use. */
+int daemon_poll(struct pollfd *fds, nfds_t nfds, int timeout);
+
 /* Shutdown for a valgrind-clean exit (frees everything) */
 void daemon_shutdown(void);
 

common/json.c

@@ -119,6 +119,17 @@ bool json_tok_bitcoin_amount(const char *buffer, const jsmntok_t *tok,
 	return true;
 }
 
+bool json_tok_is_num(const char *buffer, const jsmntok_t *tok)
+{
+	if (tok->type != JSMN_PRIMITIVE)
+		return false;
+
+	for (int i = tok->start; i < tok->end; i++)
+		if (!cisdigit(buffer[i]))
+			return false;
+	return true;
+}
+
 bool json_tok_is_null(const char *buffer, const jsmntok_t *tok)
 {
 	if (tok->type != JSMN_PRIMITIVE)

common/json.h

@@ -39,6 +39,9 @@ bool json_to_double(const char *buffer, const jsmntok_t *tok, double *num);
 bool json_tok_bitcoin_amount(const char *buffer, const jsmntok_t *tok,
 			     uint64_t *satoshi);
 
+/* Is this a number? [0..9]+ */
+bool json_tok_is_num(const char *buffer, const jsmntok_t *tok);
+
 /* Is this the null primitive? */
 bool json_tok_is_null(const char *buffer, const jsmntok_t *tok);
 

common/json_escaped.c

@@ -13,9 +13,9 @@ struct json_escaped *json_escaped_string_(const tal_t *ctx,
 	return esc;
 }
 
-struct json_escaped *json_tok_escaped_string(const tal_t *ctx,
-					     const char *buffer,
-					     const jsmntok_t *tok)
+struct json_escaped *json_to_escaped_string(const tal_t *ctx,
+					    const char *buffer,
+					    const jsmntok_t *tok)
 {
 	if (tok->type != JSMN_STRING)
 		return NULL;

common/json_escaped.h

@@ -17,9 +17,9 @@ struct json_escaped *json_partial_escape(const tal_t *ctx,
 					 const char *str TAKES);
 
 /* Extract a JSON-escaped string. */
-struct json_escaped *json_tok_escaped_string(const tal_t *ctx,
-					     const char *buffer,
-					     const jsmntok_t *tok);
+struct json_escaped *json_to_escaped_string(const tal_t *ctx,
+					    const char *buffer,
+					    const jsmntok_t *tok);
 
 /* Are two escaped json strings identical? */
 bool json_escaped_eq(const struct json_escaped *a,

common/subdaemon.c

@@ -1,3 +1,4 @@
+#include <ccan/tal/str/str.h>
 #include <common/dev_disconnect.h>
 #include <common/status.h>
 #include <common/subdaemon.h>
@@ -24,11 +25,6 @@ static void status_backtrace_exit(void)
 	status_failed(STATUS_FAIL_INTERNAL_ERROR, "FATAL SIGNAL");
 }
 
-#if DEVELOPER
-extern volatile bool debugger_connected;
-volatile bool debugger_connected;
-#endif
-
 void subdaemon_setup(int argc, char *argv[])
 {
 	if (argc == 2 && streq(argv[1], "--version")) {
@@ -45,10 +41,11 @@ void subdaemon_setup(int argc, char *argv[])
 	/* From debugger, set debugger_spin to 0. */
 	for (int i = 1; i < argc; i++) {
 		if (streq(argv[i], "--debugger")) {
-			fprintf(stderr, "gdb -ex 'attach %u' -ex 'p debugger_connected=1' %s\n",
-				getpid(), argv[0]);
-			while (!debugger_connected)
-				usleep(10000);
+			char *cmd = tal_fmt(NULL, "${DEBUG_TERM:-gnome-terminal --} gdb -ex 'attach %u' %s &", getpid(), argv[0]);
+			fprintf(stderr, "Running %s\n", cmd);
+			system(cmd);
+			/* Continue in the debugger. */
+			kill(getpid(), SIGSTOP);
 		}
 		if (strstarts(argv[i], "--dev-disconnect=")) {
 			dev_disconnect_init(atoi(argv[i]

configure

@@ -71,6 +71,17 @@ find_pytest()
 	    return
 	fi
     done
+
+    PYTHON_BINS="python python3"
+    for p in $PYTHON_BINS; do
+        if [ "$(which $p)" != "" ] ; then
+            $p --version 2>&1 | grep -q "Python 3." || continue
+            if $p -c "import pytest" ; then
+                echo "$p -m pytest"
+                return
+            fi
+        fi
+    done
 }
 
 PYTEST=${PYTEST:-`find_pytest`}

connectd/Makefile

@@ -14,7 +14,7 @@ LIGHTNINGD_CONNECT_CONTROL_OBJS := $(LIGHTNINGD_CONNECT_CONTROL_SRC:.c=.o)
 # connectd needs these:
 LIGHTNINGD_CONNECT_HEADERS := connectd/gen_connect_wire.h \
 	connectd/gen_connect_gossip_wire.h		\
-	connectd/connect.h				\
+	connectd/connectd.h				\
 	connectd/handshake.h				\
 	connectd/netaddress.h				\
 	connectd/tor_autoservice.h			\

connectd/connect.c → connectd/connectd.c

@@ -30,7 +30,7 @@
 #include <common/version.h>
 #include <common/wire_error.h>
 #include <common/wireaddr.h>
-#include <connectd/connect.h>
+#include <connectd/connectd.h>
 #include <connectd/gen_connect_gossip_wire.h>
 #include <connectd/gen_connect_wire.h>
 #include <connectd/handshake.h>

gossipd/gossip.h → connectd/connectd.h

@@ -1,10 +1,10 @@
-#ifndef LIGHTNING_GOSSIPD_GOSSIP_H
-#define LIGHTNING_GOSSIPD_GOSSIP_H
+#ifndef LIGHTNING_CONNECTD_CONNECTD_H
+#define LIGHTNING_CONNECTD_CONNECTD_H
 #include "config.h"
 
 struct io_conn;
 struct reaching;
 
 struct io_plan *connection_out(struct io_conn *conn, struct reaching *reach);
 
-#endif /* LIGHTNING_GOSSIPD_GOSSIP_H */
+#endif /* LIGHTNING_CONNECTD_CONNECTD_H */

connectd/tor.c

@@ -4,7 +4,7 @@
 #include <common/status.h>
 #include <common/utils.h>
 #include <common/wireaddr.h>
-#include <connectd/connect.h>
+#include <connectd/connectd.h>
 #include <connectd/tor.h>
 #include <netdb.h>
 #include <netinet/in.h>

devtools/.gitignore

@@ -1,3 +1,4 @@
 bolt11-cli
 decodemsg
 onion
+dump-gossipstore

devtools/Makefile

@@ -1,6 +1,6 @@
 DEVTOOLS_SRC := devtools/gen_print_wire.c devtools/gen_print_onion_wire.c devtools/print_wire.c
 DEVTOOLS_OBJS := $(DEVTOOLS_SRC:.c=.o)
-DEVTOOLS_TOOL_SRC := devtools/bolt11-cli.c devtools/decodemsg.c devtools/onion.c
+DEVTOOLS_TOOL_SRC := devtools/bolt11-cli.c devtools/decodemsg.c devtools/onion.c devtools/dump-gossipstore.c
 DEVTOOLS_TOOL_OBJS := $(DEVTOOLS_TOOL_SRC:.c=.o)
 
 DEVTOOLS_COMMON_OBJS :=				\
@@ -15,7 +15,7 @@ DEVTOOLS_COMMON_OBJS :=				\
 	common/version.o			\
 	common/wireaddr.o
 
-devtools-all: devtools/bolt11-cli devtools/decodemsg devtools/onion
+devtools-all: devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/dump-gossipstore
 
 devtools/gen_print_wire.h: $(WIRE_GEN) wire/gen_peer_wire_csv
 	$(WIRE_GEN) --bolt --printwire --header $@ wire_type < wire/gen_peer_wire_csv > $@
@@ -33,6 +33,9 @@ devtools/bolt11-cli: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCA
 
 devtools/decodemsg: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/decodemsg.o
 
+devtools/dump-gossipstore: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/dump-gossipstore.o gossipd/gen_gossip_store.o
+
+devtools/dump-gossipstore.o: gossipd/gen_gossip_store.h
 devtools/onion.c: ccan/config.h
 
 devtools/onion: $(DEVTOOLS_OBJS) $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/onion.o common/sphinx.o
@@ -42,7 +45,7 @@ devtools/gen_print_wire.o: devtools/gen_print_wire.h wire/gen_peer_wire.h devtoo
 devtools/gen_print_onion_wire.o: devtools/gen_print_onion_wire.h devtools/print_wire.h
 
 # Make sure these depend on everything.
-ALL_PROGRAMS += devtools/bolt11-cli devtools/decodemsg devtools/onion
+ALL_PROGRAMS += devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/dump-gossipstore
 ALL_OBJS += $(DEVTOOLS_OBJS) $(DEVTOOLS_TOOL_OBJS)
 
 check-source: $(DEVTOOLS_SRC:%=check-src-include-order/%) $(DEVTOOLS_TOOLS_SRC:%=check-src-include-order/%)

devtools/dump-gossipstore.c

@@ -0,0 +1,82 @@
+#include <ccan/crc/crc.h>
+#include <ccan/err/err.h>
+#include <common/type_to_string.h>
+#include <common/utils.h>
+#include <fcntl.h>
+#include <gossipd/gen_gossip_store.h>
+#include <gossipd/gossip_store.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+int main(int argc, char *argv[])
+{
+	int fd;
+	u8 version;
+	beint32_t belen, becsum;
+
+	setup_locale();
+
+	if (argc > 2)
+		errx(1, "Need the filename of a gossip store, or stdin");
+
+	if (argc == 2) {
+		fd = open(argv[1], O_RDONLY);
+		if (fd < 0)
+			err(1, "Opening %s", argv[1]);
+	} else
+		fd = STDIN_FILENO;
+
+	if (read(fd, &version, sizeof(version)) != sizeof(version))
+		errx(1, "Empty file");
+
+	if (version != GOSSIP_STORE_VERSION)
+		warnx("UNSUPPORTED GOSSIP VERSION %u (expected %u)",
+		      version, GOSSIP_STORE_VERSION);
+
+	printf("GOSSIP VERSION %u\n", version);
+
+	while (read(fd, &belen, sizeof(belen)) == sizeof(belen) &&
+	       read(fd, &becsum, sizeof(becsum)) == sizeof(becsum)) {
+		u64 satoshis;
+		struct short_channel_id scid;
+		u8 *gossip_msg;
+		u32 msglen = be32_to_cpu(belen);
+		u8 *msg = tal_arr(NULL, u8, msglen);
+
+		if (read(fd, msg, msglen) != msglen)
+			errx(1, "Truncated file?");
+
+		if (be32_to_cpu(becsum) != crc32c(0, msg, msglen))
+			warnx("Checksum verification failed");
+
+		if (fromwire_gossip_store_channel_announcement(msg, msg,
+							       &gossip_msg,
+							       &satoshis)) {
+			printf("channel_announce for %"PRIu64" satoshis: %s\n",
+			       satoshis, tal_hex(msg, gossip_msg));
+		} else if (fromwire_gossip_store_channel_update(msg, msg,
+								&gossip_msg)) {
+			printf("channel_update: %s\n",
+			       tal_hex(msg, gossip_msg));
+		} else if (fromwire_gossip_store_node_announcement(msg, msg,
+								   &gossip_msg)) {
+			printf("node_announcement: %s\n",
+			       tal_hex(msg, gossip_msg));
+		} else if (fromwire_gossip_store_channel_delete(msg, &scid)) {
+			printf("channel_delete: %s\n",
+			       type_to_string(msg, struct short_channel_id,
+					      &scid));
+		} else if (fromwire_gossip_store_local_add_channel(
+			       msg, msg, &gossip_msg)) {
+			printf("local_add_channel: %s\n",
+			       tal_hex(msg, gossip_msg));
+		} else {
+			warnx("Unknown message %u", fromwire_peektype(msg));
+		}
+		tal_free(msg);
+	}
+	return 0;
+}