Skip to content

Commit

Permalink
fix: create non-descriptor wallet (#487)
Browse files Browse the repository at this point in the history
* dev(regtest): remove nbxplorer in favor of simpler custom regtest-initializer

* dev(regtest): use jsonrpc 2.0 requests to create and load wallet

* dev(regtest): upgrade bitcoin core from v22 to v23

* fix(regtest): wait-for-bitcoind should wait for at least >=0 blocks

* dev(regtest): use jam-dev-standalone

* dev(regtest): add npm script regtest:build

this command will build the development setup with docker cache
enabled. usually must fast as it will only pull in newer base
images and not download and recompile jm again. this is the
preferred way if you made only a few adaptions to the images
and e.g. not want to fetch the current master of jm.
  • Loading branch information
theborakompanioni committed Sep 9, 2022
1 parent ace3734 commit 0d70415
Show file tree
Hide file tree
Showing 9 changed files with 139 additions and 56 deletions.
60 changes: 26 additions & 34 deletions docker/regtest/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ services:
dockerfile: Dockerfile
restart: unless-stopped
environment:
READY_FILE: /root/.nbxplorer/btc_fully_synched
READY_FILE: /root/.regtest-initializer/btc_fully_synched
ENSURE_WALLET: "true"
REMOVE_LOCK_FILES: "true"
jm_rpc_wallet_file: jm_primary
Expand All @@ -34,10 +34,10 @@ services:
- "28283:28283"
volumes:
- "joinmarket_datadir:/root/.joinmarket"
- "nbxplorer_datadir:/root/.nbxplorer"
- "initializer_datadir:/root/.regtest-initializer"
depends_on:
- bitcoind
- nbxplorer
- bitcoind_regtest_initializer
- irc

joinmarket2:
Expand All @@ -47,7 +47,7 @@ services:
dockerfile: Dockerfile
restart: unless-stopped
environment:
READY_FILE: /root/.nbxplorer/btc_fully_synched
READY_FILE: /root/.regtest-initializer/btc_fully_synched
ENSURE_WALLET: "true"
REMOVE_LOCK_FILES: "true"
APP_USER: joinmarket
Expand Down Expand Up @@ -75,10 +75,10 @@ services:
- "29080:80"
volumes:
- "joinmarket2_datadir:/root/.joinmarket"
- "nbxplorer_datadir:/root/.nbxplorer"
- "initializer_datadir:/root/.regtest-initializer"
depends_on:
- bitcoind
- nbxplorer
- bitcoind_regtest_initializer
- irc

joinmarket_directory_node:
Expand All @@ -99,7 +99,7 @@ services:
irc:
container_name: jm_regtest_irc
restart: unless-stopped
image: ghcr.io/ergochat/ergo:stable
image: ghcr.io/ergochat/ergo:v2.10.0
expose:
- 6667
volumes:
Expand All @@ -108,9 +108,9 @@ services:
bitcoind:
container_name: jm_regtest_bitcoind
restart: unless-stopped
image: btcpayserver/bitcoin:22.0-1
image: btcpayserver/bitcoin:23.0
environment:
BITCOIN_NETWORK: ${NBITCOIN_NETWORK:-regtest}
BITCOIN_NETWORK: regtest
BITCOIN_WALLETDIR: "/walletdata"
BITCOIN_EXTRA_ARGS: |
rpcport=43782
Expand Down Expand Up @@ -138,8 +138,6 @@ services:
rpcauth=joinmarket:260b4c5b1fbd09d75a4aabf90226282f$$76e170af088d43a588992cdd5e7bae2242b03c33aa672cccfd1fb75f9281299e
# rpcauth (user=joinmarket2; password=joinmarket2)
rpcauth=joinmarket2:521bf9f4468529d49c0a41f9c9f8fdbf$$63ae94a73d2aa45e7ee756945d9b1e469f9873ce026b815d676a748f777e0b8d
# rpcauth (user=nbxplorer; password=nbxplorer)
rpcauth=nbxplorer:8953867752ed07ca27bc0b1d7a10fa99$$e46553870d4a2c59753bfe453797be070ab4dbd5fdfc2fecdb7862ea0ae6b127
expose:
- 43782 # RPC
- 39388 # P2P
Expand All @@ -150,28 +148,22 @@ services:
- "bitcoin_datadir:/data"
- "bitcoin_wallet_datadir:/walletdata"

nbxplorer:
container_name: jm_regtest_nbxplorer
restart: unless-stopped
image: nicolasdorier/nbxplorer:2.2.18
environment:
NBXPLORER_NOAUTH: 1
NBXPLORER_CHAINS: "btc"
NBXPLORER_BTCRPCURL: http://bitcoind:43782/
NBXPLORER_BTCRPCUSER: nbxplorer
NBXPLORER_BTCRPCPASSWORD: nbxplorer
NBXPLORER_BTCNODEENDPOINT: bitcoind:39388
NBXPLORER_NETWORK: ${NBITCOIN_NETWORK:-regtest}
NBXPLORER_BIND: 0.0.0.0:32838
NBXPLORER_TRIMEVENTS: 10000
NBXPLORER_SIGNALFILESDIR: /datadir
expose:
- 32838
volumes:
- "nbxplorer_datadir:/datadir"
- "bitcoin_datadir:/root/.bitcoin"
depends_on:
- bitcoind
bitcoind_regtest_initializer:
container_name: jm_regtest_bitcoind_initializer
build:
context: ./dockerfile-deps/bitcoin/regtest-initializer
dockerfile: Dockerfile
restart: on-failure
environment:
READY_FILE: /root/.regtest-initializer/btc_fully_synched
RPC_HOST: bitcoind
RPC_PORT: 43782
RPC_USER: regtest
RPC_PASSWORD: regtest
volumes:
- "initializer_datadir:/root/.regtest-initializer"
depends_on:
- bitcoind

nginx:
container_name: jm_regtest_nginx_test_basepath
Expand All @@ -186,7 +178,7 @@ services:
volumes:
bitcoin_datadir:
bitcoin_wallet_datadir:
nbxplorer_datadir:
initializer_datadir:
joinmarket_datadir:
joinmarket2_datadir:
joinmarket_directory_node_datadir:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
FROM alpine:3.16

# install build dependencies
RUN apk add --no-cache --update curl jq

COPY wait-for-bitcoind.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/wait-for-bitcoind.sh

COPY wait-for-blocks.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/wait-for-blocks.sh

COPY mine-blocks.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/mine-blocks.sh

COPY entrypoint.sh /
RUN chmod +x /entrypoint.sh

ENTRYPOINT [ "/entrypoint.sh" ]
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/bin/sh
set -Eeuo pipefail

export _BTC_USER="${RPC_USER}:${RPC_PASSWORD}"
export _BTC_URL="http://${RPC_HOST}:${RPC_PORT}"

if [ -f "${READY_FILE}" ]; then
echo "Removing $READY_FILE..."
rm -f "${READY_FILE}"
echo "Removed $READY_FILE."
fi

MINE_BLOCKS=101

source /usr/local/bin/wait-for-bitcoind.sh
source /usr/local/bin/mine-blocks.sh "${MINE_BLOCKS}"
source /usr/local/bin/wait-for-blocks.sh "${MINE_BLOCKS}"

if [ "${READY_FILE}" ] && [ ! -f "${READY_FILE}" ]; then
echo "Creating $READY_FILE..."
echo "1" > "${READY_FILE}"
echo "Created $READY_FILE."
fi
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#!/bin/sh
set -Eeuo pipefail

BLOCKS=${1:-101} # default to mine a single block
ADDRESS=${2:-bcrt1qrnz0thqslhxu86th069r9j6y7ldkgs2tzgf5wx} # default to a "random" address

echo "Mining ${BLOCKS} blocks to address ${ADDRESS}..."
payload="{\
\"jsonrpc\":\"1.0\",\
\"id\":\"curl\",\
\"method\":\"generatetoaddress\",\
\"params\":[${BLOCKS},\"${ADDRESS}\"]\
}"
curl --silent --user "${_BTC_USER}" --data-binary "${payload}" "${_BTC_URL}" > /dev/null 2>&1

echo "Successfully mined ${BLOCKS} blocks to address ${ADDRESS}."
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
#!/bin/sh
set -Eeuo pipefail

source /usr/local/bin/wait-for-blocks.sh 0
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#!/bin/sh
set -Eeuo pipefail

BLOCKS=${1:-0} # wait for x amount of blocks

echo "Waiting for bitcoind to report at least ${BLOCKS} blocks..."
payload="{\
\"jsonrpc\":\"1.0\",\
\"id\":\"curl\",\
\"method\":\"getblockchaininfo\",\
\"params\":[]\
}"
until curl --silent --user "${_BTC_USER}" --data-binary "${payload}" "${_BTC_URL}" | jq -e ".result.blocks >= ${BLOCKS}" > /dev/null 2>&1
do
echo -n "."
sleep 1
done
echo "Successfully waited for ${BLOCKS} blocks to be reported."
21 changes: 19 additions & 2 deletions docker/regtest/dockerfile-deps/joinmarket/latest/jam-entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -81,11 +81,28 @@ if [ "${ENSURE_WALLET}" = "true" ]; then
wallet_name="${jmenv['rpc_wallet_file']}"

echo "Creating wallet $wallet_name if missing..."
create_payload="{\"jsonrpc\":\"1.0\",\"id\":\"curl\",\"method\":\"createwallet\",\"params\":[\"${wallet_name}\"]}"
create_payload="{\
\"jsonrpc\":\"2.0\",\
\"id\":\"curl\",\
\"method\":\"createwallet\",\
\"params\":{\
\"wallet_name\":\"${wallet_name}\",\
\"descriptors\":false,\
\"load_on_startup\":true\
}\
}"
curl --silent --user "${btcuser}" --data-binary "${create_payload}" "${btchost}" > /dev/null || true

echo "Loading wallet $wallet_name..."
load_payload="{\"jsonrpc\":\"1.0\",\"id\":\"curl\",\"method\":\"loadwallet\",\"params\":[\"${wallet_name}\"]}"
load_payload="{\
\"jsonrpc\":\"2.0\",\
\"id\":\"curl\",\
\"method\":\"loadwallet\",\
\"params\":{\
\"filename\":\"${wallet_name}\",\
\"load_on_startup\":true\
}\
}"
curl --silent --user "${btcuser}" --data-binary "${load_payload}" "${btchost}" > /dev/null || true
fi

Expand Down
32 changes: 13 additions & 19 deletions docker/regtest/readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ This setup will help you set up a regtest environment quickly.
It starts multiple JoinMarket containers, hence not only API calls but also actual CoinJoin transactions can be tested.
Communication between these containers is done via Tor (if internet connection is available) and IRC (locally running container).

Both containers will have a wallet named `Satoshi.jmdat` with password `test`.
The second container has basic auth enabled (username `joinmarket` and password `joinmarket`).

## Common flow
```sh
# (optional) once in a while rebuild the images
Expand Down Expand Up @@ -71,22 +74,26 @@ npm run regtest:mine

## Images

The [Docker setup](dockerfile-deps/joinmarket/latest/Dockerfile) is an adaption of [joinmarket-webui-standalone](https://github.com/joinmarket-webui/joinmarket-webui-docker/tree/master/standalone) with as little adaptations as possible.
The [Docker setup](dockerfile-deps/joinmarket/latest/Dockerfile) is an adaption of [jam-standalone](https://github.com/joinmarket-webui/jam-docker/tree/master/standalone) with as little adaptations as possible.
It will fetch the latest commit from the [`master` branch of the joinmarket-clientserver repo](https://github.com/JoinMarket-Org/joinmarket-clientserver/tree/master).
Keep in mind: Building from `master` is not always reliable. This tradeoff is made to enable testing new features immediately by just rebuilding the images.

The second JoinMarket container is based on `joinmarket-webui/joinmarket-webui-dev-standalone:master` which exposes an UI on port `29080`
The second JoinMarket container is based on `joinmarket-webui/jam-dev-standalone:master` which exposes an UI on port `29080`
(username `joinmarket` and pass `joinmarket` for Basic Authentication).
This is useful if you want to perform regression tests.

The third JoinMarket container acts as [Directory Node](https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/master/docs/onion-message-channels.md#directory) and exists solely to enable communication between peers.

### Rebuild
### Build
```sh
# building the images
npm run regtest:build
```

In order to incorporate the current contents of `master` branch, simply rebuild the joinmarket images from scratch.
In order to incorporate recent upstream changes (of the `master` branch), simply rebuild the setup from scratch.

```sh
# rebuilding the images with contents of current master branch
# download and recompile the images from scratch (without using docker cache)
npm run regtest:rebuild
```

Expand All @@ -95,7 +102,7 @@ npm run regtest:rebuild
### Debug logs

```sh
# logs and follows content of log file `.joinmarket/logs/jmwalletd_stdout.log` in primary joinmarket container
# logs and follows content of log file in primary joinmarket container
npm run regtest:logs:jmwalletd
```

Expand Down Expand Up @@ -205,19 +212,6 @@ Generating 5 blocks with rewards to bcrt1qs0aqmzxjq96jk8hhmta5jfn339dk4cme074lq3
Successfully generated 5 blocks with rewards to bcrt1qs0aqmzxjq96jk8hhmta5jfn339dk4cme074lq3
```

## Troubleshooting

### Joinmarket won't start in initial run

Solution: Somehow nbxplorer does not notify joinmarket that the chain is fully synced in the initial run.
Just shutdown all containers with `docker-compose down` wait for it to finish and run `docker-compose up` again (`docker-compose restart` did _not_ work sometimes!).
Now you should see joinmarket coming up and see something like the following output:

```log
joinmarket_1 | 2009-01-03 00:02:44,907 INFO success: jmwalletd entered RUNNING state, process has stayed up for > than 1 seconds (startsecs)
joinmarket_1 | 2009-01-03 00:02:44,907 INFO success: ob-watcher entered RUNNING state, process has stayed up for > than 1 seconds (startsecs)
```

## Resources

- [JoinMarket Server (GitHub)](https://github.com/JoinMarket-Org/joinmarket-clientserver)
Expand Down
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@
"start:regtest:secondary": "PORT=3001 JAM_BACKEND=jam-standalone JAM_API_PORT=29080 npm start",
"version": "node scripts/changelog.mjs && git checkout -b \"prepare-v${npm_package_version}-$(date +%s)\" && git add --all && git commit --message \"chore(release): prepare v${npm_package_version}\" && git push --set-upstream origin $(git branch --show-current)",
"postversion": "which gh && gh pr create --title \"chore(release): prepare v${npm_package_version}\" --body \"Prepares the v${npm_package_version} release.\" --assignee @me --label release --repo joinmarket-webui/joinmarket-webui --draft",
"regtest:rebuild": "npm run regtest:clear && docker-compose --env-file docker/regtest/.env.example --file docker/regtest/docker-compose.yml build --pull --no-cache",
"regtest:build": "npm run regtest:clear && docker-compose --env-file docker/regtest/.env.example --file docker/regtest/docker-compose.yml build --pull",
"regtest:rebuild": "npm run regtest:build -- --no-cache",
"regtest:clear": "docker-compose --env-file docker/regtest/.env.example --file docker/regtest/docker-compose.yml down --volumes --remove-orphans",
"regtest:up": "./docker/regtest/prepare-setup.sh && docker-compose --env-file docker/regtest/.env.generated --file docker/regtest/docker-compose.yml up",
"regtest:down": "docker-compose --env-file docker/regtest/.env.example --file docker/regtest/docker-compose.yml down",
Expand Down

0 comments on commit 0d70415

Please sign in to comment.