.github/workflows/bsd.yml

@@ -63,7 +63,6 @@ jobs:
             blinker \
             flake8 \
             mako \
-            mrkd \
             pytest-sentry \
             pytest-test-groups==1.0.3 \
             pytest-custom-exit-code==0.3.0 \

.github/workflows/ci.yaml

@@ -99,7 +99,7 @@ jobs:
 
   proto-test:
     name: Protocol Test Config
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 300
     needs: [smoke-test]
     strategy:
@@ -122,7 +122,7 @@ jobs:
           -e VALGRIND=${{ matrix.valgrind }} \
           -e DEVELOPER=1 \
           -e EXPERIMENTAL_FEATURES=1 \
-          -e COMPA=0 \
+          -e COMPAT=0 \
           -e PYTEST_PAR=2 \
           -e PYTEST_OPTS="--timeout=300" \
           -e TEST_CMD="make check-protos" \

.gitmodules

@@ -16,4 +16,9 @@
 	url = https://github.com/valyala/gheap
 [submodule "external/lnprototest"]
 	path = external/lnprototest
-	url = https://github.com/rustyrussell/lnprototest.git
+	url = https://github.com/niftynei/lnprototest.git
+	branch = nifty/ripemd160-fallback
+[submodule "external/lowdown"]
+	path = external/lowdown
+	url = https://github.com/kristapsdz/lowdown.git
+	ignore = dirty

CHANGELOG.md

@@ -8,6 +8,44 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 TODO: Insert version codename, and username of the contributor that named the release.
 -->
 
+## [0.12.1] - 2022-09-13: Web-8 init (dot one)
+
+Point release with some bugfixes and patches.
+
+### Added
+
+None
+
+### Changed
+
+None
+
+### Deprecated
+
+None
+
+### Removed
+
+- build: `mrkd` and `mistune` not required to build project
+
+### Fixed
+
+- lnprototest: builds for lnprototest tests now use 22.04 LTS, which fixes a problem with loading `mako`. ([#5583])
+- Plugins: topology plugin could crash when it sees duplicate private channel announcements ([#5593])
+- connectd: proper `gossip_store` operation may resolve some previous gossip propagation issues and connectd crashes ([#5591])
+- connectd: Fixed a condition for newly created channels that could trigger a need for reconnect. ([#5601])
+- `peer_control`: getinfo showing unannounced addresses. ([#5584])
+- `peer_control`: getinfo shows the correct port on discovered IPs ([#5585])
+
+
+[#5583]: https://github.com/ElementsProject/lightning/pull/5583
+[#5584]: https://github.com/ElementsProject/lightning/pull/5584
+[#5585]: https://github.com/ElementsProject/lightning/pull/5585
+[#5593]: https://github.com/ElementsProject/lightning/pull/5593
+[#5591]: https://github.com/ElementsProject/lightning/pull/5591
+[#5601]: https://github.com/ElementsProject/lightning/pull/5601
+
+
 ## [0.12.0] - 2022-08-23: Web-8 init
 
 This release named by @adi2011.
@@ -1863,6 +1901,7 @@ There predate the BOLT specifications, and are only of vague historic interest:
 6. [0.5.1] - 2016-10-21
 7. [0.5.2] - 2016-11-21: "Bitcoin Savings & Trust Daily Interest II"
 
+[0.12.1]: https://github.com/ElementsProject/lightning/releases/tag/v0.12.1
 [0.12.0]: https://github.com/ElementsProject/lightning/releases/tag/v0.12.0
 [0.11.2]: https://github.com/ElementsProject/lightning/releases/tag/v0.11.2
 [0.11.1]: https://github.com/ElementsProject/lightning/releases/tag/v0.11.1

Makefile

@@ -317,19 +317,13 @@ endif
 ifeq ($(SUPPRESS_GENERATION),1)
 SHA256STAMP_CHANGED = false
 SHA256STAMP = exit 1
-SHA256STAMP_CHANGED_ALL = false
-SHA256STAMP_ALL = exit 1
 else
 # Git doesn't maintain timestamps, so we only regen if sources actually changed:
 # We place the SHA inside some generated files so we can tell if they need updating.
 # Usage: $(call SHA256STAMP_CHANGED)
-SHA256STAMP_CHANGED = [ x"`sed -n 's/.*SHA256STAMP:\([a-f0-9]*\).*/\1/p' $@ 2>/dev/null`" != x"`cat $(sort $(filter-out FORCE,$<)) | $(SHA256SUM) | cut -c1-64`" ]
+SHA256STAMP_CHANGED = [ x"`sed -n 's/.*SHA256STAMP:\([a-f0-9]*\).*/\1/p' $@ 2>/dev/null`" != x"`cat $(sort $(filter-out FORCE,$^)) | $(SHA256SUM) | cut -c1-64`" ]
 # Usage: $(call SHA256STAMP,commentprefix,commentpostfix)
-SHA256STAMP = echo "$(1) SHA256STAMP:"`cat $(sort $(filter-out FORCE,$<)) | $(SHA256SUM) | cut -c1-64`"$(2)" >> $@
-
-SHA256STAMP_CHANGED_ALL = [ x"`sed -n 's/.*SHA256STAMP:\([a-f0-9]*\).*/\1/p' $@ 2>/dev/null`" != x"`cat $(sort $(filter-out FORCE,$^)) | $(SHA256SUM) | cut -c1-64`" ]
-# Usage: $(call SHA256STAMP,commentprefix,commentpostfix)
-SHA256STAMP_ALL = echo "$(1) SHA256STAMP:"`cat $(sort $(filter-out FORCE,$^)) | $(SHA256SUM) | cut -c1-64`"$(2)" >> $@
+SHA256STAMP = echo "$(1) SHA256STAMP:"`cat $(sort $(filter-out FORCE,$^)) | $(SHA256SUM) | cut -c1-64`"$(2)" >> $@
 endif
 
 # generate-wire.py --page [header|impl] hdrfilename wirename < csv > file

channeld/channeld.c

@@ -955,6 +955,10 @@ static void send_shutdown_complete(struct peer *peer)
 	wire_sync_write(MASTER_FD,
 			take(towire_channeld_shutdown_complete(NULL)));
 	per_peer_state_fdpass_send(MASTER_FD, peer->pps);
+
+	/* Give master a chance to pass the fd along */
+	sleep(1);
+
 	close(MASTER_FD);
 }
 

common/gossip_store.c

@@ -115,6 +115,7 @@ u8 *gossip_store_next(const tal_t *ctx,
 		      size_t *off, size_t *end)
 {
 	u8 *msg = NULL;
+	size_t initial_off = *off;
 
 	while (!msg) {
 		struct gossip_hdr hdr;
@@ -146,6 +147,14 @@ u8 *gossip_store_next(const tal_t *ctx,
 			continue;
 		}
 
+		/* Messages can be up to 64k, but we also have internal ones:
+		 * 128k is plenty. */
+		if (msglen > 128 * 1024)
+			status_failed(STATUS_FAIL_INTERNAL_ERROR,
+				      "gossip_store: oversize msg len %u at"
+				      " offset %zu (was at %zu)",
+				      msglen, *off, initial_off);
+
 		checksum = be32_to_cpu(hdr.crc);
 		msg = tal_arr(ctx, u8, msglen);
 		r = pread(*gossip_store_fd, msg, msglen, *off + r);
@@ -155,8 +164,8 @@ u8 *gossip_store_next(const tal_t *ctx,
 		if (checksum != crc32c(be32_to_cpu(hdr.timestamp), msg, msglen))
 			status_failed(STATUS_FAIL_INTERNAL_ERROR,
 				      "gossip_store: bad checksum at offset %zu"
-				      ": %s",
-				      *off, tal_hex(tmpctx, msg));
+				      "(was at %zu): %s",
+				      *off, initial_off, tal_hex(tmpctx, msg));
 
 		/* Definitely processing it now */
 		*off += sizeof(hdr) + msglen;
@@ -190,8 +199,8 @@ size_t find_gossip_store_end(int gossip_store_fd, size_t off)
 	} buf;
 	int r;
 
-	while ((r = read(gossip_store_fd, &buf,
-			 sizeof(buf.hdr) + sizeof(buf.type)))
+	while ((r = pread(gossip_store_fd, &buf,
+			 sizeof(buf.hdr) + sizeof(buf.type), off))
 	       == sizeof(buf.hdr) + sizeof(buf.type)) {
 		u32 msglen = be32_to_cpu(buf.hdr.len) & GOSSIP_STORE_LEN_MASK;
 
@@ -200,7 +209,6 @@ size_t find_gossip_store_end(int gossip_store_fd, size_t off)
 			break;
 
 		off += sizeof(buf.hdr) + msglen;
-		lseek(gossip_store_fd, off, SEEK_SET);
 	}
 	return off;
 }

common/gossmap.c

@@ -414,11 +414,8 @@ static struct gossmap_chan *add_channel(struct gossmap *map,
 	 * that's the only time we get duplicates */
 	scid.u64 = map_be64(map, cannounce_off + plus_scid_off);
 	chan = gossmap_find_chan(map, &scid);
-	if (chan) {
-		assert(chan->private);
-		assert(!private);
+	if (chan)
 		gossmap_remove_chan(map, chan);
-	}
 
 	/* We carefully map pointers to indexes, since new_node can move them! */
 	n[0] = gossmap_find_node(map, &node_id[0]);

configure

@@ -102,7 +102,7 @@ check_command()
     shift 1
 
     echo -n "checking for $name... "
-    if "$@" >/dev/null 2>&1; then
+    if "$@" >/dev/null 2>&1 </dev/null; then
 	echo 'found'
 	return 0
     fi
@@ -417,6 +417,12 @@ else
     HAVE_PYTHON3_MAKO=0
 fi
 
+if check_command 'lowdown' lowdown; then
+    HAVE_LOWDOWN=1
+else
+    HAVE_LOWDOWN=0
+fi
+
 if echo | check_command sha256sum sha256sum; then
     SHA256SUM=sha256sum
 elif echo | check_command "shasum -a 256" shasum -a 256; then
@@ -455,6 +461,7 @@ add_var ASAN "$ASAN"
 add_var UBSAN "$UBSAN"
 add_var TEST_NETWORK "$TEST_NETWORK"
 add_var HAVE_PYTHON3_MAKO "$HAVE_PYTHON3_MAKO"
+add_var HAVE_LOWDOWN "$HAVE_LOWDOWN"
 add_var SHA256SUM "$SHA256SUM"
 add_var FUZZING "$FUZZING"
 add_var RUST "$RUST"

contrib/docker/Dockerfile.alpine

@@ -15,7 +15,6 @@ COPY . lightning
 RUN cd lightning && \
     git submodule update --init --recursive && \
     ./configure && \
-    pip3 install mrkd mistune==0.8.4 && \
     make -j$(nproc) && \
     make install
 

contrib/docker/Dockerfile.builder

@@ -65,5 +65,4 @@ RUN pip3 install --upgrade pip && \
 	tqdm==4.26.0 \
 	pytest-test-groups==1.0.3 \
 	flake8==3.5.0 \
-	pytest-rerunfailures==3.1 \
-	mrkd==0.1.6
+	pytest-rerunfailures==3.1

contrib/docker/Dockerfile.builder.fedora

@@ -32,4 +32,4 @@ RUN wget https://bitcoin.org/bin/bitcoin-core-$BITCOIN_VERSION/bitcoin-$BITCOIN_
     rm -rf bitcoin.tar.gz bitcoin-$BITCOIN_VERSION
 
 RUN python3 -m pip install --upgrade pip && \
-    python3 -m pip install python-bitcoinlib==0.10.2 pytest==3.0.5 setuptools==36.6.0 pytest-test-groups==1.0.3 flake8==3.5.0 pytest-rerunfailures==3.1 ephemeral-port-reserve==1.1.0 mrkd==0.1.6
+    python3 -m pip install python-bitcoinlib==0.10.2 pytest==3.0.5 setuptools==36.6.0 pytest-test-groups==1.0.3 flake8==3.5.0 pytest-rerunfailures==3.1 ephemeral-port-reserve==1.1.0

contrib/docker/Dockerfile.ubuntu

@@ -1,10 +1,8 @@
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 LABEL mantainer="Vincenzo Palazzo vincenzopalazzodev@gmail.com"
 
 WORKDIR /work
 
-COPY . .
-
 ENV DEBIAN_FRONTEND=noninteractive
 ENV LANGUAGE=en_US.UTF-8
 ENV LANG=en_US.UTF-8
@@ -23,7 +21,7 @@ RUN locale-gen en_US.UTF-8 && dpkg-reconfigure --frontend noninteractive tzdata
 
 COPY . .
 
-# install package for pytho cryptography lib
+# install package for python cryptography lib
 # https://cryptography.io/en/latest/installation/#debian-ubuntu
 
 RUN apt-get -qq update && \

contrib/docker/linuxarm32v7.Dockerfile

@@ -92,7 +92,6 @@ RUN git clone --recursive /tmp/lightning . && \
 
 ARG DEVELOPER=0
 ENV PYTHON_VERSION=3
-RUN pip3 install mrkd
 RUN ./configure --prefix=/tmp/lightning_install --enable-static && make -j3 DEVELOPER=${DEVELOPER} && make install
 
 FROM arm32v7/debian:buster-slim as final

contrib/docker/linuxarm64v8.Dockerfile

@@ -91,7 +91,6 @@ RUN git clone --recursive /tmp/lightning . && \
 
 ARG DEVELOPER=0
 ENV PYTHON_VERSION=3
-RUN pip3 install mrkd
 RUN ./configure --prefix=/tmp/lightning_install --enable-static && make -j3 DEVELOPER=${DEVELOPER} && make install
 
 FROM arm64v8/debian:buster-slim as final

contrib/docker/scripts/setup.sh

@@ -6,8 +6,6 @@ export ELEMENTS_VERSION=0.18.1.8
 export RUST_VERSION=nightly
 export TZ="Europe/London"
 
-sudo useradd -ms /bin/bash tester
-
 sudo apt-get update -qq
 
 sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
@@ -54,9 +52,6 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
      zlib1g-dev
 
 
-echo "tester ALL=(root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tester
-sudo chmod 0440 /etc/sudoers.d/tester
-
 (
     cd /tmp/ || exit 1
     wget https://storage.googleapis.com/c-lightning-tests/bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2

contrib/pyln-client/pyln/client/__init__.py

@@ -2,7 +2,7 @@
 from .plugin import Plugin, monkey_patch, RpcException
 from .gossmap import Gossmap, GossmapNode, GossmapChannel, GossmapNodeId
 
-__version__ = "0.12.0"
+__version__ = "0.12.1"
 
 __all__ = [
     "LightningRpc",

contrib/pyln-client/pyln/client/plugin.py

@@ -72,7 +72,7 @@ def __init__(self, message: str, code: int = -32600):
 class Request(dict):
     """A request object that wraps params and allows async return
     """
-    def __init__(self, plugin: 'Plugin', req_id: Optional[int], method: str,
+    def __init__(self, plugin: 'Plugin', req_id: Optional[str], method: str,
                  params: Any, background: bool = False):
         self.method = method
         self.params = params
@@ -700,13 +700,9 @@ def notify_progress(self, request: Request,
         request.progress(progress, progress_total, stage, stage_total)
 
     def _parse_request(self, jsrequest: Dict[str, JSONType]) -> Request:
-        i = jsrequest.get('id', None)
-        if not isinstance(i, int) and i is not None:
-            raise ValueError('Non-integer request id "{i}"'.format(i=i))
-
         request = Request(
             plugin=self,
-            req_id=i,
+            req_id=jsrequest.get('id', None),
             method=str(jsrequest['method']),
             params=jsrequest['params'],
             background=False,

contrib/pyln-proto/pyln/proto/__init__.py

@@ -4,7 +4,7 @@
 from .onion import OnionPayload, TlvPayload, LegacyOnionPayload
 from .wire import LightningConnection, LightningServerSocket
 
-__version__ = "0.12.0"
+__version__ = "0.12.1"
 
 __all__ = [
     "Invoice",

contrib/pyln-testing/pyln/testing/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.12.0"
+__version__ = "0.12.1"
 
 __all__ = [
     "__version__",

contrib/pyln-testing/pyln/testing/utils.py

@@ -1435,8 +1435,8 @@ def get_nodes(self, num_nodes, opts=None):
         return [j.result() for j in jobs]
 
     def get_node(self, node_id=None, options=None, dbfile=None,
-                 feerates=(15000, 11000, 7500, 3750), start=True,
-                 wait_for_bitcoind_sync=True, may_fail=False,
+                 bkpr_dbfile=None, feerates=(15000, 11000, 7500, 3750),
+                 start=True, wait_for_bitcoind_sync=True, may_fail=False,
                  expect_fail=False, cleandir=True, **kwargs):
         self.throttler.wait()
         node_id = self.get_node_id() if not node_id else node_id
@@ -1470,6 +1470,12 @@ def get_node(self, node_id=None, options=None, dbfile=None,
             with lzma.open(os.path.join('tests/data', dbfile), 'rb') as f:
                 out.write(f.read())
 
+        if bkpr_dbfile:
+            out = open(os.path.join(node.daemon.lightning_dir, TEST_NETWORK,
+                                    'accounts.sqlite3'), 'xb')
+            with lzma.open(os.path.join('tests/data', bkpr_dbfile), 'rb') as f:
+                out.write(f.read())
+
         if start:
             try:
                 node.start(wait_for_bitcoind_sync)