Merge #249: Add regtest support

9951f10 test: add scenario 'regtest' (Erik Arvstedt)
1f96ca6 electrs test: make service shutdown optional (Erik Arvstedt)
eb42fc8 test: extract test 'joinmarket-yieldgenerator' (Erik Arvstedt)
06b2ec5 joinmarket: add regtest support (Erik Arvstedt)
975b30c joinmarket: don't hardcode bitcoind rpc port (Erik Arvstedt)
031df42 joinmarket: move comment out of config file (Erik Arvstedt)
848c4c6 joinmarket: add variable 'bitcoind' (Erik Arvstedt)
96b08f5 btcpayserver: add regtest support (Erik Arvstedt)
bd2145d btcpayserver: add 'port' option (Erik Arvstedt)
001f8fe btcpayserver: use option bitcoind.rpc.port (Erik Arvstedt)
6f4715a electrs: add regtest support (Erik Arvstedt)
46efd14 lightning-loop: add regtest support (Erik Arvstedt)
75ec85b lnd: add regtest support (Erik Arvstedt)
1935c25 lnd: remove redundant option 'bitcoind-host' (Erik Arvstedt)
b1a8629 lnd: add variable 'bitcoind' (Erik Arvstedt)
937aee0 spark-wallet: add regtest support (Erik Arvstedt)
47d611b spark-wallet: use tor rate provider only when enforceTor (Erik Arvstedt)
127b186 spark-wallet: simplify start script (Erik Arvstedt)
0f32f3c clightning: add regtest support (Erik Arvstedt)
c24ac5d clightning: remove redundant option 'bitcoin-rpcconnect' (Erik Arvstedt)
abd32cd clightning: enable config file read access for group (Erik Arvstedt)
ddadaed clightning: always use bind-addr in config (Erik Arvstedt)
9e928e2 bitcoind: add regtest support (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK 9951f10

Tree-SHA512: 42e2d95755a16b59044e400bc4c9d891bfc22eb73b920fdcf29e607f7df88de599bec99677cf49be7c275c0113a2224a45b1f47f40c029878421eae1a44f3254

modules/bitcoind.nix

@@ -11,7 +11,10 @@ let
     # We're already logging via journald
     nodebuglogfile=1
 
-    ${optionalString cfg.testnet "testnet=1"}
+    ${optionalString cfg.regtest ''
+      regtest=1
+      [regtest]
+    ''}
     ${optionalString (cfg.dbCache != null) "dbcache=${toString cfg.dbCache}"}
     prune=${toString cfg.prune}
     ${optionalString (cfg.sysperms != null) "sysperms=${if cfg.sysperms then "1" else "0"}"}
@@ -159,10 +162,18 @@ in {
           Allow JSON-RPC connections from specified source.
         '';
       };
-      testnet = mkOption {
+      regtest = mkOption {
         type = types.bool;
         default = false;
-        description = "Whether to use the test chain.";
+        description = "Enable regtest mode.";
+      };
+      network = mkOption {
+        readOnly = true;
+        default = if cfg.regtest then "regtest" else "mainnet";
+      };
+      makeNetworkName = mkOption {
+        readOnly = true;
+        default = mainnet: regtest: if cfg.regtest then regtest else mainnet;
       };
       port = mkOption {
         type = types.nullOr types.port;

modules/btcpayserver.nix

@@ -34,6 +34,11 @@ in {
         default = "127.0.0.1";
         description = "The address on which to bind.";
       };
+      port = mkOption {
+        type = types.port;
+        default = 24444;
+        description = "Port on which to bind.";
+      };
       enable = mkOption {
         # This option is only used by netns-isolation
         internal = true;
@@ -70,6 +75,11 @@ in {
         default = "127.0.0.1";
         description = "The address on which to bind.";
       };
+      port = mkOption {
+        type = types.port;
+        default = 23000;
+        description = "Port on which to bind.";
+      };
       lightningBackend = mkOption {
         type = types.nullOr (types.enum [ "clightning" "lnd" ]);
         default = null;
@@ -104,11 +114,12 @@ in {
 
     systemd.services.nbxplorer = let
       configFile = builtins.toFile "config" ''
-        network=mainnet
+        network=${config.services.bitcoind.network}
         btcrpcuser=${cfg.bitcoind.rpc.users.btcpayserver.name}
-        btcrpcurl=http://${builtins.elemAt config.services.bitcoind.rpcbind 0}:8332
+        btcrpcurl=http://${builtins.elemAt config.services.bitcoind.rpcbind 0}:${toString cfg.bitcoind.rpc.port}
         btcnodeendpoint=${config.services.bitcoind.bind}:8333
         bind=${cfg.nbxplorer.bind}
+        port=${toString cfg.nbxplorer.port}
       '';
     in {
       description = "Run nbxplorer";
@@ -138,12 +149,13 @@ in {
 
     systemd.services.btcpayserver = let
       configFile = builtins.toFile "config" (''
-        network=mainnet
+        network=${config.services.bitcoind.network}
         postgres=User ID=${cfg.btcpayserver.user};Host=/run/postgresql;Database=btcpaydb
         socksendpoint=${cfg.tor.client.socksListenAddress}
-        btcexplorerurl=http://${cfg.nbxplorer.bind}:24444/
-        btcexplorercookiefile=${cfg.nbxplorer.dataDir}/Main/.cookie
+        btcexplorerurl=http://${cfg.nbxplorer.bind}:${toString cfg.nbxplorer.port}/
+        btcexplorercookiefile=${cfg.nbxplorer.dataDir}/${config.services.bitcoind.makeNetworkName "Main" "RegTest"}/.cookie
         bind=${cfg.btcpayserver.bind}
+        port=${toString cfg.btcpayserver.port}
       '' + optionalString (cfg.btcpayserver.lightningBackend == "clightning") ''
         btclightning=type=clightning;server=unix:///${cfg.clightning.dataDir}/bitcoin/lightning-rpc
       '');

modules/clightning.nix

@@ -6,13 +6,15 @@ let
   cfg = config.services.clightning;
   inherit (config) nix-bitcoin-services;
   onion-chef-service = (if cfg.announce-tor then [ "onion-chef.service" ] else []);
+  network = config.services.bitcoind.makeNetworkName "bitcoin" "regtest";
   configFile = pkgs.writeText "config" ''
-    network=bitcoin
+    network=${network}
     bitcoin-datadir=${config.services.bitcoind.dataDir}
     ${optionalString (cfg.proxy != null) "proxy=${cfg.proxy}"}
     always-use-proxy=${if cfg.always-use-proxy then "true" else "false"}
-    ${optionalString (cfg.bind-addr != null) "bind-addr=${cfg.bind-addr}:${toString cfg.bindport}"}
-    ${optionalString (cfg.bitcoin-rpcconnect != null) "bitcoin-rpcconnect=${cfg.bitcoin-rpcconnect}"}
+    bind-addr=${cfg.bind-addr}:${toString cfg.bindport}
+    bitcoin-rpcconnect=${builtins.elemAt config.services.bitcoind.rpcbind 0}
+    bitcoin-rpcport=${toString config.services.bitcoind.rpc.port}
     bitcoin-rpcuser=${config.services.bitcoind.rpc.users.public.name}
     rpc-file-mode=0660
     ${cfg.extraConfig}
@@ -61,16 +63,16 @@ in {
       default = false;
       description = "Announce clightning Tor Hidden Service";
     };
-    bitcoin-rpcconnect = mkOption {
-      type = types.nullOr types.str;
-      default = null;
-      description = "The bitcoind RPC host to connect to.";
-    };
     dataDir = mkOption {
       type = types.path;
       default = "/var/lib/clightning";
       description = "The data directory for clightning.";
     };
+    networkDir = mkOption {
+      readOnly = true;
+      default = "${cfg.dataDir}/${network}";
+      description = "The network data directory.";
+    };
     extraConfig = mkOption {
       type = types.lines;
       default = "";
@@ -122,8 +124,8 @@ in {
         cp ${configFile} ${cfg.dataDir}/config
         chown -R '${cfg.user}:${cfg.group}' '${cfg.dataDir}'
         # The RPC socket has to be removed otherwise we might have stale sockets
-        rm -f ${cfg.dataDir}/bitcoin/lightning-rpc
-        chmod 600 ${cfg.dataDir}/config
+        rm -f ${cfg.networkDir}/lightning-rpc
+        chmod 640 ${cfg.dataDir}/config
         echo "bitcoin-rpcpassword=$(cat ${config.nix-bitcoin.secretsDir}/bitcoin-rpcpassword-public)" >> '${cfg.dataDir}/config'
         ${optionalString cfg.announce-tor "echo announce-addr=$(cat /var/lib/onion-chef/clightning/clightning) >> '${cfg.dataDir}/config'"}
         '';
@@ -139,11 +141,11 @@ in {
         );
       # Wait until the rpc socket appears
       postStart = ''
-        while [[ ! -e ${cfg.dataDir}/bitcoin/lightning-rpc ]]; do
+        while [[ ! -e ${cfg.networkDir}/lightning-rpc ]]; do
             sleep 0.1
         done
         # Needed to enable lightning-cli for users with group 'clightning'
-        chmod g+x ${cfg.dataDir}/bitcoin
+        chmod g+x ${cfg.networkDir}
       '';
     };
   };

modules/electrs.nix

@@ -90,6 +90,7 @@ in {
             else
               "--jsonrpc-import --index-batch-size=10"
           } \
+          --network=${bitcoind.makeNetworkName "bitcoin" "regtest"} \
           --db-dir='${cfg.dataDir}' \
           --daemon-dir='${bitcoind.dataDir}' \
           --electrum-rpc-addr=${cfg.address}:${toString cfg.port} \

modules/joinmarket.nix

@@ -7,9 +7,10 @@ let
   inherit (config) nix-bitcoin-services;
   secretsDir = config.nix-bitcoin.secretsDir;
 
+  inherit (config.services) bitcoind;
   torAddress = builtins.head (builtins.split ":" config.services.tor.client.socksListenAddress);
+  # Based on https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/master/jmclient/jmclient/configure.py
   configFile = builtins.toFile "config" ''
-    # Based on https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/master/jmclient/jmclient/configure.py
     [DAEMON]
     no_daemon = 0
     daemon_port = 27183
@@ -18,10 +19,10 @@ let
 
     [BLOCKCHAIN]
     blockchain_source = bitcoin-rpc
-    network = mainnet
-    rpc_host = ${builtins.elemAt config.services.bitcoind.rpcbind 0}
-    rpc_port = 8332
-    rpc_user = ${config.services.bitcoind.rpc.users.privileged.name}
+    network = ${bitcoind.network}
+    rpc_host = ${builtins.elemAt bitcoind.rpcbind 0}
+    rpc_port = ${toString bitcoind.rpc.port}
+    rpc_user = ${bitcoind.rpc.users.privileged.name}
     @@RPC_PASSWORD@@
 
     [MESSAGING:server1]
@@ -155,7 +156,8 @@ in {
              "s|@@RPC_PASSWORD@@|rpc_password = $(cat ${secretsDir}/bitcoin-rpcpassword-privileged)|" \
              '${cfg.dataDir}/joinmarket.cfg'
         '';
-        ExecStartPost = nix-bitcoin-services.privileged ''
+        # Generating wallets (jmclient/wallet.py) is only supported for mainnet or testnet
+        ExecStartPost = mkIf (bitcoind.network == "mainnet") (nix-bitcoin-services.privileged ''
           walletname=wallet.jmdat
           pw=$(cat "${secretsDir}"/jm-wallet-password)
           mnemonic=${secretsDir}/jm-wallet-seed
@@ -170,7 +172,7 @@ in {
             recoveryseed=$(echo "$out" | grep 'recovery_seed')
             echo "$recoveryseed" | cut -d ':' -f2 > $mnemonic
           fi
-        '';
+        '');
         ExecStart = "${pkgs.nix-bitcoin.joinmarket}/bin/joinmarketd";
         WorkingDirectory = "${cfg.dataDir}"; # The service creates 'commitmentlist' in the working dir
         User = "${cfg.user}";

modules/lightning-loop.nix

@@ -8,12 +8,13 @@ let
   secretsDir = config.nix-bitcoin.secretsDir;
   configFile = builtins.toFile "loop.conf" ''
     datadir=${cfg.dataDir}
+    network=${config.services.bitcoind.network}
     logdir=${cfg.dataDir}/logs
     tlscertpath=${secretsDir}/loop-cert
     tlskeypath=${secretsDir}/loop-key
 
     lnd.host=${builtins.elemAt config.services.lnd.rpclisten 0}:${toString config.services.lnd.rpcPort}
-    lnd.macaroondir=${config.services.lnd.dataDir}/chain/bitcoin/mainnet
+    lnd.macaroondir=${config.services.lnd.networkDir}
     lnd.tlspath=${secretsDir}/lnd-cert
 
     ${optionalString (cfg.proxy != null) "server.proxy=${cfg.proxy}"}

modules/lnd.nix

@@ -5,30 +5,33 @@ with lib;
 let
   cfg = config.services.lnd;
   inherit (config) nix-bitcoin-services;
-  onion-chef-service = (if cfg.announce-tor then [ "onion-chef.service" ] else []);
   secretsDir = config.nix-bitcoin.secretsDir;
-  mainnetDir = "${cfg.dataDir}/chain/bitcoin/mainnet";
+
+  bitcoind = config.services.bitcoind;
+  bitcoindRpcAddress = builtins.elemAt bitcoind.rpcbind 0;
+  onion-chef-service = (if cfg.announce-tor then [ "onion-chef.service" ] else []);
+  networkDir = "${cfg.dataDir}/chain/bitcoin/${bitcoind.network}";
   configFile = pkgs.writeText "lnd.conf" ''
     datadir=${cfg.dataDir}
     logdir=${cfg.dataDir}/logs
-    bitcoin.mainnet=1
     tlscertpath=${secretsDir}/lnd-cert
     tlskeypath=${secretsDir}/lnd-key
 
     listen=${toString cfg.listen}:${toString cfg.listenPort}
     ${lib.concatMapStrings (rpclisten: "rpclisten=${rpclisten}:${toString cfg.rpcPort}\n") cfg.rpclisten}
     ${lib.concatMapStrings (restlisten: "restlisten=${restlisten}:${toString cfg.restPort}\n") cfg.restlisten}
 
+    bitcoin.${bitcoind.network}=1
     bitcoin.active=1
     bitcoin.node=bitcoind
 
     tor.active=true
     ${optionalString (cfg.tor-socks != null) "tor.socks=${cfg.tor-socks}"}
 
-    bitcoind.rpchost=${cfg.bitcoind-host}
-    bitcoind.rpcuser=${config.services.bitcoind.rpc.users.public.name}
-    bitcoind.zmqpubrawblock=${config.services.bitcoind.zmqpubrawblock}
-    bitcoind.zmqpubrawtx=${config.services.bitcoind.zmqpubrawtx}
+    bitcoind.rpchost=${bitcoindRpcAddress}:${toString bitcoind.rpc.port}
+    bitcoind.rpcuser=${bitcoind.rpc.users.public.name}
+    bitcoind.zmqpubrawblock=${bitcoind.zmqpubrawblock}
+    bitcoind.zmqpubrawtx=${bitcoind.zmqpubrawtx}
 
     ${cfg.extraConfig}
   '';
@@ -47,6 +50,11 @@ in {
       default = "/var/lib/lnd";
       description = "The data directory for LND.";
     };
+    networkDir = mkOption {
+      readOnly = true;
+      default = networkDir;
+      description = "The network data directory.";
+    };
     listen = mkOption {
       type = pkgs.nix-bitcoin.lib.ipv4Address;
       default = "localhost";
@@ -81,13 +89,6 @@ in {
       default = 8080;
       description = "Port on which to listen for REST connections.";
     };
-    bitcoind-host = mkOption {
-      type = types.str;
-      default = "127.0.0.1";
-      description = ''
-        The host that your local bitcoind daemon is listening on.
-      '';
-    };
     tor-socks = mkOption {
       type = types.nullOr types.str;
       default = null;
@@ -138,7 +139,7 @@ in {
       # Switch user because lnd makes datadir contents readable by user only
       ''
         ${cfg.cliExec} sudo -u lnd ${cfg.package}/bin/lncli --tlscertpath ${secretsDir}/lnd-cert \
-          --macaroonpath '${cfg.dataDir}/chain/bitcoin/mainnet/admin.macaroon' "$@"
+          --macaroonpath '${networkDir}/admin.macaroon' "$@"
       '';
       description = "Binary to connect with the lnd instance.";
     };
@@ -148,7 +149,7 @@ in {
 
   config = mkIf cfg.enable {
     assertions = [
-      { assertion = config.services.bitcoind.prune == 0;
+      { assertion = bitcoind.prune == 0;
         message = "lnd does not support bitcoind pruning.";
       }
     ];
@@ -160,8 +161,8 @@ in {
     ];
 
     services.bitcoind = {
-      zmqpubrawblock = "tcp://${cfg.bitcoind-host}:28332";
-      zmqpubrawtx = "tcp://${cfg.bitcoind-host}:28333";
+      zmqpubrawblock = "tcp://${bitcoindRpcAddress}:28332";
+      zmqpubrawtx = "tcp://${bitcoindRpcAddress}:28333";
     };
 
     services.onion-chef.access.lnd = if cfg.announce-tor then [ "lnd" ] else [];
@@ -206,7 +207,7 @@ in {
             chown lnd: "$mnemonic"
           ''}"
           "${nix-bitcoin-services.script ''
-            if [[ ! -f ${mainnetDir}/wallet.db ]]; then
+            if [[ ! -f ${networkDir}/wallet.db ]]; then
               echo Create lnd wallet
 
               ${pkgs.curl}/bin/curl -s --output /dev/null --show-error \
@@ -217,14 +218,14 @@ in {
 
               # Guarantees that RPC calls with cfg.cli succeed after the service is started
               echo Wait until wallet is created
-              while [[ ! -f ${mainnetDir}/admin.macaroon ]]; do
+              while [[ ! -f ${networkDir}/admin.macaroon ]]; do
                 sleep 0.1
               done
             else
               echo Unlock lnd wallet
 
               ${pkgs.curl}/bin/curl -s \
-                -H "Grpc-Metadata-macaroon: $(${pkgs.xxd}/bin/xxd -ps -u -c 99999 '${mainnetDir}/admin.macaroon')" \
+                -H "Grpc-Metadata-macaroon: $(${pkgs.xxd}/bin/xxd -ps -u -c 99999 '${networkDir}/admin.macaroon')" \
                 --cacert ${secretsDir}/lnd-cert \
                 -X POST \
                 -d "{\"wallet_password\": \"$(cat ${secretsDir}/lnd-wallet-password | tr -d '\n' | base64 -w0)\"}" \
@@ -244,7 +245,7 @@ in {
               echo "Create custom macaroon ${macaroon}"
               macaroonPath="$RUNTIME_DIRECTORY/${macaroon}.macaroon"
               ${pkgs.curl}/bin/curl -s \
-                -H "Grpc-Metadata-macaroon: $(${pkgs.xxd}/bin/xxd -ps -u -c 99999 '${mainnetDir}/admin.macaroon')" \
+                -H "Grpc-Metadata-macaroon: $(${pkgs.xxd}/bin/xxd -ps -u -c 99999 '${networkDir}/admin.macaroon')" \
                 --cacert ${secretsDir}/lnd-cert \
                 -X POST \
                 -d '{"permissions":[${cfg.macaroons.${macaroon}.permissions}]}' \

modules/netns-isolation.nix

@@ -267,10 +267,7 @@ in {
     };
     systemd.services.bitcoind-import-banlist.serviceConfig.NetworkNamespacePath = "/var/run/netns/nb-bitcoind";
 
-    services.clightning = {
-      bitcoin-rpcconnect = netns.bitcoind.address;
-      bind-addr = netns.clightning.address;
-    };
+    services.clightning.bind-addr = netns.clightning.address;
 
     services.lnd = {
       listen = netns.lnd.address;
@@ -282,7 +279,6 @@ in {
         "${netns.lnd.address}"
         "127.0.0.1"
       ];
-      bitcoind-host = netns.bitcoind.address;
       cliExec = mkCliExec "lnd";
     };