Merge pull request #16 from dflatemaster/master

[WIP] grs use wally sha256 and warp to v.0.7.1

.gitignore

@@ -38,6 +38,6 @@ tools/headerversions
 contrib/pylightning/build/
 contrib/pylightning/dist/
 contrib/pylightning/pylightning.egg-info/
-
+devtools/create-gossipstore
 contrib/compose/groestlcoin_datadir/
 contrib/compose/clightning_groestlcoin_datadir/

.gitmodules

@@ -1,9 +1,6 @@
 [submodule "daemon/jsmn"]
 	path = external/jsmn
 	url = https://github.com/zserge/jsmn
-[submodule "bitcoin/libbase58"]
-	path = external/libbase58
-	url = https://github.com/bitcoin/libbase58.git
 [submodule "libsodium"]
 	path = external/libsodium
 	url = https://github.com/jedisct1/libsodium.git

.travis.yml

@@ -17,7 +17,7 @@ addons:
       - gcc-4.8
 
 env:
-  - ARCH=64 SOURCE_CHECK_ONLY=true CDEBUGFLAGS="-std=gnu11 -g -fstack-protector -O3 -flto" LDFLAGS="-O3 -flto"
+  - ARCH=64 SOURCE_CHECK_ONLY=true COPTFLAGS="-O3"
 #  - VALGRIND=0 ARCH=32 DEVELOPER=1 COMPILER=gcc TEST_GROUP=1 TEST_GROUP_COUNT=2 SOURCE_CHECK_ONLY=false
 #  - VALGRIND=0 ARCH=32 DEVELOPER=1 COMPILER=gcc TEST_GROUP=2 TEST_GROUP_COUNT=2 SOURCE_CHECK_ONLY=false
   - VALGRIND=0 ARCH=64 DEVELOPER=1 COMPILER=gcc SOURCE_CHECK_ONLY=false

.travis/build.sh

@@ -8,6 +8,12 @@ export DEVELOPER=${DEVELOPER:-1}
 export SOURCE_CHECK_ONLY=${SOURCE_CHECK_ONLY:-"false"}
 export COMPAT=${COMPAT:-1}
 export PATH=$CWD/dependencies/bin:"$HOME"/.local/bin:"$PATH"
+export TIMEOUT=180
+export PYTEST_PAR=2
+# If we're not in developer mode, tests spend a lot of time waiting for gossip!
+if [ "$DEVELOPER" = 0 ]; then
+    PYTEST_PAR=4
+fi
 
 mkdir -p dependencies/bin || true
 
@@ -40,5 +46,8 @@ if [ "$SOURCE_CHECK_ONLY" == "false" ]; then
     echo -en 'travis_fold:end:script.3\\r'
 else
     git clone https://github.com/lightningnetwork/lightning-rfc.git
+    echo -en 'travis_fold:start:script.2\\r'
+    make -j3 > /dev/null
+    echo -en 'travis_fold:end:script.2\\r'
     make check-source BOLTDIR=lightning-rfc
 fi

CHANGELOG.md

@@ -4,6 +4,71 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.7.1] - 2019-06-29: "The Unfailing Twitter Consensus Algorithm"
+
+This release named by (C-Lightning Core Team member) Lisa Neigut @niftynei.
+
+### Added
+
+- Protocol: we now enforce `option_upfront_shutdown_script` if a peer negotiates it.
+- JSON API: New command `setchannelfee` sets channel specific routing fees.
+- JSON API: new withdraw methods `txprepare`, `txsend` and `txdiscard`.
+- JSON API: add three new RPC commands: `fundchannel_start`, `fundchannel_complete` and `fundchannel_cancel`. Allows a user to initiate and complete a channel open using funds that are in a external wallet.
+- Plugin: new hooks `db_write` for intercepting database writes, `invoice_payment` for intercepting invoices before they're paid, `openchannel` for intercepting channel opens, and `htlc_accepted` to decide whether to resolve, reject or continue an incoming or forwarded payment..
+- Plugin: new notification `warning` to report any `LOG_UNUSUAL`/`LOG_BROKEN` level event.
+- Plugin: Added a default plugin directory : `lightning_dir/plugins`. Each plugin directory it contains will be added to lightningd on startup.
+- Plugin: the `connected` hook can now send an `error_message` to the rejected peer.
+- JSON API: `newaddr` outputs `bech32` or `p2sh-segwit`, or both with new `all` parameter (#2390)
+- JSON API: `listpeers` status now shows how many confirmations until channel is open (#2405)
+- Config: Adds parameter `min-capacity-sat` to reject tiny channels.
+- JSON API: `listforwards` now includes the time an HTLC was received and when it was resolved. Both are expressed as UNIX timestamps to facilitate parsing (Issue [#2491](https://github.com/ElementsProject/lightning/issues/2491), PR [#2528](https://github.com/ElementsProject/lightning/pull/2528))
+- JSON API: `listforwards` now includes the local_failed forwards with failcode (Issue [#2435](https://github.com/ElementsProject/lightning/issues/2435), PR [#2524](https://github.com/ElementsProject/lightning/pull/2524))
+- DB: Store the signatures of channel announcement sent from remote peer into DB, and init channel with signatures from DB directly when reenable the channel.
+(Issue [#2409](https://github.com/ElementsProject/lightning/issues/2409))
+- JSON API: `listchannels` has new fields `htlc_minimum_msat` and `htlc_maximum_msat`.
+
+### Changed
+
+- Gossip: we no longer compact the `gossip_store` file dynamically, due to lingering bugs.  Restart if it gets too large.
+- Protocol: no longer ask for entire gossip flood from peers, unless we're missing gossip.
+- JSON API: `invoice` expiry defaults to 7 days, and can have s/m/h/d/w suffixes.
+- Config: Increased default amount for minimal channel capacity from 1k sat to 10k sat.
+- JSON API: A new parameter is added to `fundchannel`, which now accepts an utxo array to use to fund the channel.
+- Build: Non-developer builds are now done with "-Og" optimization.
+- JSON API: `pay` will no longer return failure until it is no longer retrying; previously it could "timeout" but still make the payment.
+- JSON API: the command objects that `help` outputs now contain a new string field : `category` (can be "bitcoin", "channels", "network", "payment", "plugins", "utility", "developer" for native commands, or any other new category set by a plugin).
+- Plugin: a plugin can now set the category of a newly created RPC command. This possibility has been added to libplugin.c and pylightning.
+- lightning-cli: the human readable help is now more human and more readable : commands are sorted alphabetically and ordered by categories.
+
+### Deprecated
+
+Note: You should always set `allow-deprecated-apis=false` to test for
+changes.
+
+- JSON API: `newaddr` output field `address`: use `bech32` or `p2sh-segwit` instead.
+
+### Removed
+
+- JSON RPC: `global_features` and `local_features` fields and `listchannels`' `flags` field.  (Deprecated since 0.6.2).
+- pylightning: Remove RPC support for c-lightning before 0.6.3.
+
+### Fixed
+
+- Protocol: reconnection during closing negotiation now supports
+  `option_data_loss_protect` properly.
+- `--bind-addr=<path>` fixed for nodes using local sockets (eg. testing).
+- Unannounced local channels were forgotten for routing on restart until reconnection occurred.
+- lightning-cli: arguments containing `"` now succeed, rather than causing JSON errors.
+- Protocol: handle lnd sending more messages before `reestablish`; don't fail channel, and handle older lnd's spurious empty commitments.
+- Fixed `fundchannel` crash when we have many UTXOs and we skip unconfirmed ones.
+- lightningd: fixed occasional hang on `connect` when peer had sent error.
+- JSON RPC: `decodeinvoice` and `pay` now handle unknown invoice fields properly.
+- JSON API: `waitsendpay` (PAY_STOPPED_RETRYING) error handler now returns valid JSON
+- protocol: don't send multiple identical feerate changes if we want the feerate higher than we can afford.
+- JSON API: `stop` now only returns once lightningd has released all resources.
+
+### Security
+
 ## [0.7.0] - 2019-02-28: "Actually an Altcoin"
 
 This release named by Mark Beckwith @wythe.
@@ -79,6 +144,9 @@ provide appropriate suffixes for JSON input fields.
 - You can no longer make giant unpayable "wumbo" invoices.
 - CLTV of total route now correctly evaluated when finding best route.
 - `riskfactor` arguments to `pay` and `getroute` now have an effect.
+- Fixed the version of bip32 private_key to BIP32_VER_MAIN_PRIVATE: we used
+  BIP32_VER_MAIN_PRIVATE for bitcoin/litecoin mainnet, and BIP32_VER_TEST_PRIVATE
+  for others. (PR #2436)
 
 ### Security
 
@@ -307,12 +375,13 @@ There predate the BOLT specifications, and are only of vague historic interest:
 1. [0.1] - 2015-08-08: "MtGox's Cold Wallet" (named by Rusty Russell)
 2. [0.2] - 2016-01-22: "Butterfly Labs' Timely Delivery" (named by Anthony Towns)
 3. [0.3] - 2016-05-25: "Nakamoto's Genesis Coins" (named by Braydon Fuller)
-4. [0.4] - 2016-08-19: "Wright's Cryptographic Proof" (named by Chrstian Decker)
+4. [0.4] - 2016-08-19: "Wright's Cryptographic Proof" (named by Christian Decker)
 5. [0.5] - 2016-10-19: "Bitcoin Savings & Trust Daily Interest" (named by Glenn Willen)
 6. [0.5.1] - 2016-10-21
 7. [0.5.2] - 2016-11-21: "Bitcoin Savings & Trust Daily Interest II"
 
-[0.7.0]: https://github.com/ElementsProject/lightning/compare/v0.6.3...HEAD
+[0.7.1]: https://github.com/ElementsProject/lightning/releases/tag/v0.7.1
+[0.7.0]: https://github.com/ElementsProject/lightning/releases/tag/v0.7.0
 [0.6.3]: https://github.com/ElementsProject/lightning/releases/tag/v0.6.3
 [0.6.2]: https://github.com/ElementsProject/lightning/releases/tag/v0.6.2
 [0.6.1]: https://github.com/ElementsProject/lightning/releases/tag/v0.6.1

Dockerfile

@@ -37,7 +37,7 @@ RUN git clone --recursive /tmp/lightning . && \
     git checkout $(git --work-tree=/tmp/lightning --git-dir=/tmp/lightning/.git rev-parse HEAD)
 
 ARG DEVELOPER=0
-RUN ./configure && make -j3 DEVELOPER=${DEVELOPER} && cp lightningd/lightning* cli/lightning-cli /usr/bin/
+RUN ./configure --prefix=/tmp/lightning_install && make -j3 DEVELOPER=${DEVELOPER} && make install
 
 FROM debian:stretch-slim
 
@@ -52,12 +52,9 @@ RUN mkdir $LIGHTNINGD_DATA && \
     touch $LIGHTNINGD_DATA/config
 
 VOLUME [ "/root/.lightning" ]
-
-COPY --from=builder /opt/lightningd/cli/lightning-cli /usr/bin
-COPY --from=builder /opt/lightningd/lightningd/lightning* /usr/bin/
-COPY --from=builder /opt/lightningd/plugins/pay /usr/libexec/c-lightning/plugins/
+COPY --from=builder /tmp/lightning_install/ /usr/local/
 COPY --from=builder /opt/groestlcoin/bin /usr/bin
 COPY tools/docker-entrypoint.sh entrypoint.sh
 
 EXPOSE 9735 9835
-ENTRYPOINT  [ "./entrypoint.sh" ]
+ENTRYPOINT  [ "/sbin/tini", "-g", "--", "./entrypoint.sh" ]

Makefile

@@ -15,12 +15,12 @@ CCANDIR := ccan
 
 # Where we keep the BOLT RFCs
 BOLTDIR := ../lightning-rfc/
-BOLTVERSION := 3fef98d10695462edecc63cba05e4a96374f4664
+BOLTVERSION := da71867c840abe057d7a68f358dd9493e0e79110
 
 -include config.vars
 
 ifneq ($(VALGRIND),0)
-VG=valgrind -q --error-exitcode=7
+VG=VALGRIND=1 valgrind -q --error-exitcode=7
 VG_TEST_ARGS = --track-origins=yes --leak-check=full --show-reachable=yes --errors-for-leak-kinds=all
 endif
 
@@ -31,7 +31,7 @@ SANITIZER_FLAGS=
 endif
 
 ifeq ($(DEVELOPER),1)
-DEV_CFLAGS=-DCCAN_TAKE_DEBUG=1 -DCCAN_TAL_DEBUG=1
+DEV_CFLAGS=-DCCAN_TAKE_DEBUG=1 -DCCAN_TAL_DEBUG=1 -DCCAN_JSON_OUT_DEBUG=1
 else
 DEV_CFLAGS=
 endif
@@ -47,7 +47,7 @@ endif
 
 ifeq ($(COMPAT),1)
 # We support compatibility with pre-0.6.
-COMPAT_CFLAGS=-DCOMPAT_V052=1 -DCOMPAT_V060=1 -DCOMPAT_V061=1 -DCOMPAT_V062=1
+COMPAT_CFLAGS=-DCOMPAT_V052=1 -DCOMPAT_V060=1 -DCOMPAT_V061=1 -DCOMPAT_V062=1 -DCOMPAT_V070=1
 endif
 
 # Timeout shortly before the 600 second travis silence timeout
@@ -63,7 +63,7 @@ CCAN_OBJS :=					\
 	ccan-bitmap.o				\
 	ccan-bitops.o				\
 	ccan-breakpoint.o			\
-	ccan-crc.o				\
+	ccan-crc32c.o				\
 	ccan-crypto-hmac.o			\
 	ccan-crypto-hkdf.o			\
 	ccan-crypto-ripemd160.o			\
@@ -81,6 +81,8 @@ CCAN_OBJS :=					\
 	ccan-io-fdpass.o			\
 	ccan-isaac.o				\
 	ccan-isaac64.o				\
+	ccan-json_escape.o			\
+	ccan-json_out.o				\
 	ccan-list.o				\
 	ccan-mem.o				\
 	ccan-membuf.o				\
@@ -123,7 +125,7 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/compiler/compiler.h		\
 	$(CCANDIR)/ccan/container_of/container_of.h	\
 	$(CCANDIR)/ccan/cppmagic/cppmagic.h		\
-	$(CCANDIR)/ccan/crc/crc.h			\
+	$(CCANDIR)/ccan/crc32c/crc32c.h			\
 	$(CCANDIR)/ccan/crypto/hkdf_sha256/hkdf_sha256.h \
 	$(CCANDIR)/ccan/crypto/hmac_sha256/hmac_sha256.h \
 	$(CCANDIR)/ccan/crypto/ripemd160/ripemd160.h	\
@@ -144,6 +146,8 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/io/io_plan.h			\
 	$(CCANDIR)/ccan/isaac/isaac.h			\
 	$(CCANDIR)/ccan/isaac/isaac64.h			\
+	$(CCANDIR)/ccan/json_escape/json_escape.h	\
+	$(CCANDIR)/ccan/json_out/json_out.h		\
 	$(CCANDIR)/ccan/likely/likely.h			\
 	$(CCANDIR)/ccan/list/list.h			\
 	$(CCANDIR)/ccan/mem/mem.h			\
@@ -185,14 +189,12 @@ WIRE_GEN := tools/generate-wire.py
 ALL_PROGRAMS =
 
 CPPFLAGS = -DBINTOPKGLIBEXECDIR='"'$(shell sh tools/rel.sh $(bindir) $(pkglibexecdir))'"'
-CWARNFLAGS := -Werror -Wall -Wundef -Wmissing-prototypes -Wmissing-declarations -Wstrict-prototypes -Wold-style-definition
-CDEBUGFLAGS := -std=gnu11 -g -fstack-protector $(SANITIZER_FLAGS)
-CFLAGS = $(CPPFLAGS) $(CWARNFLAGS) $(CDEBUGFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . -I/usr/local/include $(FEATURES) $(COVFLAGS) $(DEV_CFLAGS) -DSHACHAIN_BITS=48 -DJSMN_PARENT_LINKS $(PIE_CFLAGS) $(COMPAT_CFLAGS)
+CFLAGS = $(CPPFLAGS) $(CWARNFLAGS) $(CDEBUGFLAGS) $(COPTFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . -I/usr/local/include $(FEATURES) $(COVFLAGS) $(DEV_CFLAGS) -DSHACHAIN_BITS=48 -DJSMN_PARENT_LINKS $(PIE_CFLAGS) $(COMPAT_CFLAGS)
 
 # We can get configurator to run a different compile cmd to cross-configure.
 CONFIGURATOR_CC := $(CC)
 
-LDFLAGS = $(PIE_LDFLAGS) $(SANITIZER_FLAGS)
+LDFLAGS = $(PIE_LDFLAGS) $(SANITIZER_FLAGS) $(COPTFLAGS)
 ifeq ($(STATIC),1)
 LDLIBS = -L/usr/local/lib -Wl,-dn -lgmp -lsqlite3 -lz -Wl,-dy -lm -lpthread -ldl $(COVFLAGS)
 else
@@ -487,7 +489,7 @@ PKGLIBEXEC_PROGRAMS = \
 	       lightningd/lightning_hsmd \
 	       lightningd/lightning_onchaind \
 	       lightningd/lightning_openingd
-PLUGINS=plugins/pay
+PLUGINS=plugins/pay plugins/autoclean
 
 install-program: installdirs $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS)
 	@$(NORMAL_INSTALL)
@@ -603,7 +605,7 @@ ccan-noerr.o: $(CCANDIR)/ccan/noerr/noerr.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 ccan-str-hex.o: $(CCANDIR)/ccan/str/hex/hex.c
 	$(CC) $(CFLAGS) -c -o $@ $<
-ccan-crc.o: $(CCANDIR)/ccan/crc/crc.c
+ccan-crc32c.o: $(CCANDIR)/ccan/crc32c/crc32c.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 ccan-crypto-hmac.o: $(CCANDIR)/ccan/crypto/hmac_sha256/hmac_sha256.c
 	$(CC) $(CFLAGS) -c -o $@ $<
@@ -659,3 +661,7 @@ ccan-bitmap.o: $(CCANDIR)/ccan/bitmap/bitmap.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 ccan-membuf.o: $(CCANDIR)/ccan/membuf/membuf.c
 	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-json_escape.o: $(CCANDIR)/ccan/json_escape/json_escape.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-json_out.o: $(CCANDIR)/ccan/json_out/json_out.c
+	$(CC) $(CFLAGS) -c -o $@ $<

README.md

@@ -188,7 +188,6 @@ the file, for example:
 ```
 alias=SLEEPYDRAGON
 rgb=008000
-port=9735
 network=testnet
 ```