[ADP-3387] Cleanup wallet in mainnet run instructions and E2E smoke t…

…ests (#4681)

- [x] Add run instructions to start a wallet in mainnet via docker
- [x] Use the run instructions to redefine the smoke test via docker
- [x] Enhance the README to cover those instructions
- [x] Add a nix smoke test for the wallet in mainnet

.buildkite/pipeline.yml

@@ -159,6 +159,24 @@ steps:
         NODE_LOGS_FILE: ./logs/node.log
         WALLET_LOGS_FILE: ./logs/wallet.log
 
+    - label: Mainnet Boot Sync
+      depends_on: linux-nix
+      timeout: 30
+      command: |
+        cd run/mainnet/nix
+        rm -rf logs
+        mkdir -p logs
+        rm -rf databases
+        export SUCCESS_STATUS=syncing
+        ./run.sh sync
+      artifact_paths:
+        - "./run/mainnet/nix/logs/*"
+      agents:
+        system: x86_64-linux
+      env:
+        NODE_LOGS_FILE: ./logs/node.log
+        WALLET_LOGS_FILE: ./logs/wallet.log
+
     - block: Sanchonet Full Sync
       if: build.env("RELEASE_CANDIDATE") == null
       depends_on: linux-nix
@@ -546,18 +564,15 @@ steps:
   - group: Docker Checks
     key: docker-e2e
     steps:
-      - label: Smoke Test docker-compose
-        if: build.env("RELEASE_CANDIDATE") != null
-        key: docker-smoke-test
+      - label: Mainnet Boot Sync
         depends_on:
           - docker-build
-        timeout_in_minutes: 120
-        commands:
-          - ./scripts/buildkite/release/docker-smoke-test.sh
-        artifact_paths:
-          - "./logs/*"
-        env:
-          NODE_STATE_DIR: "${STATE_DIR?}/node/preprod"
+        timeout_in_minutes: 30
+        command: |
+          cd run/mainnet/docker
+          rm -rf databases
+          export SUCCESS_STATUS=syncing
+          ./run.sh sync
         agents:
           system: x86_64-linux
         concurrency: 1
@@ -613,3 +628,28 @@ steps:
           system: x86_64-linux
         concurrency: 1
         concurrency_group: 'docker'
+
+  - group: Links
+    key: links-validity
+    steps:
+      - block: Snapshot Links
+        depends_on: linux-nix
+        if: build.env("RELEASE_CANDIDATE") == null
+        key: snapshot-links
+
+      - label: Check Mainnet Snapshot Link
+        depends_on: snapshot-links
+        command: |
+          cd run/mainnet/docker
+          LINK_TEST=1 ./snapshot.sh
+        agents:
+          system: ${linux}
+
+      - label: Check Preprod Snapshot Link
+        depends_on: snapshot-links
+        command: |
+          cd run/preprod/docker
+          LINK_TEST=1 ./snapshot.sh
+        agents:
+          system: ${linux}
+

README.md

@@ -138,13 +138,45 @@ NixOS users can also use the [NixOS service](https://cardano-foundation.github.i
 
 ### Running on mainnet
 
-**Take care when running on mainnet, as the docker compose will expose the wallet port to the host machine, allowing other applications or users to access wallet funds.**
-
 On `mainnet`, the Minimum System Requirements for a `cardano-node` are high:
 
 - 200GB of disk space (for the history of blocks)
 - 24GB of RAM (for the current UTxO set)
 
+To speed up the synchronization process,
+you can download a snapshot of the `cardano-node` state database,
+but you will have much less security than the full Ouroboros consensus protocol.
+In particular, the snapshot could be created by a malicious block producer and
+contain erroneous transactions that are not consensus on `mainnet`!
+
+```bash
+cd run/mainnet/docker
+./snapshot.sh
+```
+
+The snapshot.sh will try to use `NODE_DB` as the directory to store the snapshot.
+
+Wether you used a snapshot or not, you can now start the wallet with
+
+```bash
+cd run/mainnet/docker
+./run.sh start
+```
+
+Notice that the wallet port is not exposed.
+For security reasons **you should not expose** the wallet port to the internet.
+The cardano-wallet **is not designed to be exposed** to the internet.
+It would be no different from exposing the keys to your bank account to the internet!
+
+You can connect to the wallet by attaching a container to the network and using the internal port.
+
+```bash
+docker run -it --network mainnet_default alpine/curl curl http://cardano-wallet:8090/v2/network/information | jq
+```
+
+Or modify the `docker-compose.yml` to host your client in the `mainnet_default` network.
+
+
 ### Running on preprod
 
 Preprod network is broken for node 9.0.0 at some block in the past.

configs/cardano/mainnet/config.json

@@ -8,15 +8,15 @@
   "LastKnownBlockVersion-Major": 3,
   "LastKnownBlockVersion-Minor": 0,
   "MaxKnownMajorProtocolVersion": 2,
-  "MinNodeVersion": "8.9.2",
+  "MinNodeVersion": "8.12.0",
   "PeerSharing": true,
   "Protocol": "Cardano",
   "RequiresNetworkMagic": "RequiresNoMagic",
   "ShelleyGenesisFile": "shelley-genesis.json",
   "ShelleyGenesisHash": "1a3be38bcbb7911969283716ad7aa550250226b76a61fc51cc9a9a35d9276d81",
   "TargetNumberOfActivePeers": 20,
   "TargetNumberOfEstablishedPeers": 50,
-  "TargetNumberOfKnownPeers": 100,
+  "TargetNumberOfKnownPeers": 150,
   "TargetNumberOfRootPeers": 60,
   "TraceAcceptPolicy": true,
   "TraceBlockFetchClient": false,

configs/cardano/mainnet/download.sh

@@ -2,15 +2,9 @@
 
 set -euo pipefail
 
-curl https://book.play.dev.cardano.org/environments/mainnet/config.json \
-    > config.json
-curl https://book.play.dev.cardano.org/environments/mainnet/conway-genesis.json \
-    > conway-genesis.json
-curl https://book.play.dev.cardano.org/environments/mainnet/topology.json \
-    > topology.json
-curl https://book.play.dev.cardano.org/environments/mainnet/byron-genesis.json \
-    > byron-genesis.json
-curl https://book.play.dev.cardano.org/environments/mainnet/shelley-genesis.json \
-    > shelley-genesis.json
-curl https://book.play.dev.cardano.org/environments/mainnet/alonzo-genesis.json \
-    > alonzo-genesis.json
+curl https://book.play.dev.cardano.org/environments-pre/mainnet/config.json > config.json
+curl https://book.play.dev.cardano.org/environments-pre/mainnet/conway-genesis.json  > conway-genesis.json
+curl https://book.play.dev.cardano.org/environments-pre/mainnet/topology.json  > topology.json
+curl https://book.play.dev.cardano.org/environments-pre/mainnet/byron-genesis.json  > byron-genesis.json
+curl https://book.play.dev.cardano.org/environments-pre/mainnet/shelley-genesis.json  > shelley-genesis.json
+curl https://book.play.dev.cardano.org/environments-pre/mainnet/alonzo-genesis.json  > alonzo-genesis.json

configs/cardano/mainnet/topology.json

@@ -27,5 +27,5 @@
       "advertise": false
     }
   ],
-  "useLedgerAfterSlot": 116812831
+  "useLedgerAfterSlot": 128908821
 }

run/common/docker/docker-compose.yml

@@ -1,4 +1,4 @@
-name: sanchonet
+name: ${NETWORK}
 services:
   cardano-node:
     image: ghcr.io/intersectmbo/cardano-node:${NODE_TAG}
@@ -9,7 +9,7 @@ services:
       - ${NODE_SOCKET_DIR}:/ipc
       - ${NODE_CONFIGS}:/configs
     restart: on-failure
-    user: ${USER_ID}
+    user: ${USER_ID}:${GROUP_ID}
     logging:
       driver: "json-file"
       options:
@@ -40,7 +40,7 @@ services:
         --listen-address 0.0.0.0
         --testnet /configs/byron-genesis.json
 
-    user: ${USER_ID}
+    user: ${USER_ID}:${GROUP_ID}
     restart: on-failure
     logging:
       driver: "json-file"

run/common/docker/run.sh

@@ -43,6 +43,7 @@ if [[ -z "${WALLET_DB-}" ]]; then
     WALLET_DB=$LOCAL_WALLET_DB
     export WALLET_DB
 fi
+rm -rf "${WALLET_DB:?}/*"
 
 # Define a local db if NODE_DB is not set
 if [[ -z "${NODE_DB-}" ]]; then
@@ -51,11 +52,16 @@ if [[ -z "${NODE_DB-}" ]]; then
     NODE_DB=$LOCAL_NODE_DB
     export NODE_DB
 fi
+rm -rf "${NODE_DB:?}/*"
 
 # Get the current user's ID and export it
 USER_ID=$(id -u)
 export USER_ID
 
+# Get the current user's group ID and export it
+GROUP_ID=$(id -g)
+export GROUP_ID
+
 # Define and export the node socket name
 NODE_SOCKET_NAME=node.socket
 export NODE_SOCKET_NAME
@@ -72,9 +78,9 @@ export NODE_CONFIGS
 
 startup() {
     # Pull the latest images
-    docker compose pull -q 2>/dev/null
+    docker compose pull -q
     # Start the service in detached mode
-    docker compose up -d 2>/dev/null
+    docker compose up -d
 }
 
 # Function to clean up the service
@@ -97,20 +103,24 @@ case "$1" in
         start_time=$(date +%s)
 
         # Commands to query service status and node tip time
-        command="curl -s localhost:$WALLET_PORT/v2/network/information | jq -r"
-        query_status="$command .sync_progress.status"
+        command=$(printf "docker run --network %s_default alpine/curl curl -s --max-time 5 http://cardano-wallet:8090/v2/network/information | jq -r" "$NETWORK" )
+        query_status="$command  .sync_progress.status"
         query_time="$command .node_tip.time"
         query_progress="$command .sync_progress.progress.quantity"
 
         # Execute and display the full query result
         trap cleanup ERR INT
+
+        # Define the wanted status and result, can be "syncing" or "ready"
+        SUCCESS_STATUS=${SUCCESS_STATUS:="ready"}
+
         while true; do
             # Check the sync status
             status=$(cat <(bash -c "$query_status")) || echo "failed"
             if [[ $(date +%s) -ge $((start_time + timeout)) ]]; then
                 result="timeout"
                 break
-            elif [[ "$status" == "ready" ]]; then
+            elif [[ "$status" == "$SUCCESS_STATUS" ]]; then
                 result="success"
                 printf "\n"
                 break

run/common/nix/run.sh

@@ -105,13 +105,14 @@ case "$1" in
         query_time="$command .node_tip.time"
         query_progress="$command .sync_progress.progress.quantity"
 
+        SUCCESS_STATUS=${SUCCESS_STATUS:="ready"}
         while true; do
             # Check the sync status
             status=$(cat <(bash -c "$query_status")) || echo "failed"
             if [[ $(date +%s) -ge $((start_time + timeout)) ]]; then
                 result="timeout"
                 break
-            elif [[ "$status" == "ready" ]]; then
+            elif [[ "$status" == "$SUCCESS_STATUS" ]]; then
                 result="success"
                 printf "\n"
                 break

run/common/snapshot.sh

@@ -24,6 +24,11 @@ case "$NETWORK" in
         echo "Snapshot name: $SNAPSHOT_NAME"
         SNAPSHOT_URL="https://downloads.csnapshots.io/testnet/$SNAPSHOT_NAME"
         ;;
+    mainnet)
+        SNAPSHOT_NAME=$(curl -s https://downloads.csnapshots.io/mainnet/mainnet-db-snapshot.json| jq -r .[].file_name )
+        echo "Snapshot name: $SNAPSHOT_NAME"
+        SNAPSHOT_URL="https://downloads.csnapshots.io/mainnet/$SNAPSHOT_NAME"
+        ;;
     *)
         echo "Error: Invalid network $NETWORK"
         exit 1
@@ -32,6 +37,14 @@ esac
 
 echo "Downloading the snapshot..."
 
+if [ -n "${LINK_TEST:-}" ]; then
+    echo "Link test enabled"
+    echo "Snapshot URL: $SNAPSHOT_URL"
+    curl -f -LI "$SNAPSHOT_URL" > /dev/null
+    curl -r 0-1000000 -SL "$SNAPSHOT_URL" > /dev/null
+    exit 0
+fi
+
 curl -SL "$SNAPSHOT_URL" | lz4 -c -d - | tar -x -C "$NODE_DB"
 
 mv -f "$NODE_DB"/db/* "$NODE_DB"/

run/mainnet/docker/.env

@@ -0,0 +1 @@
+NETWORK="mainnet"

run/mainnet/docker/configs

@@ -0,0 +1 @@
+../../../configs/cardano/mainnet

run/mainnet/docker/docker-compose.yml

@@ -0,0 +1,49 @@
+name: mainnet
+services:
+  cardano-node:
+    image: ghcr.io/intersectmbo/cardano-node:9.0.0
+    environment:
+      CARDANO_NODE_SOCKET_PATH: /ipc/${NODE_SOCKET_NAME}
+    volumes:
+      - ${NODE_DB}:/data
+      - ${NODE_SOCKET_DIR}:/ipc
+      - ${NODE_CONFIGS}:/configs
+    restart: on-failure
+    user: ${USER_ID}:${GROUP_ID}
+    logging:
+      driver: "json-file"
+      options:
+        compress: "true"
+        max-file: "10"
+        max-size: "50m"
+    entrypoint: []
+    command: >
+      cardano-node run --topology /configs/topology.json
+        --database-path /data
+        --socket-path /ipc/node.socket
+        --config /configs/config.json
+        +RTS -N -A16m -qg -qb -RTS
+
+  cardano-wallet:
+    image: cardanofoundation/cardano-wallet:2024.7.7
+    volumes:
+      - ${WALLET_DB}:/wallet-db
+      - ${NODE_SOCKET_DIR}:/ipc
+      - ${NODE_CONFIGS}:/configs
+    entrypoint: []
+    command: >
+      cardano-wallet serve
+        --node-socket /ipc/${NODE_SOCKET_NAME}
+        --database /wallet-db
+        --listen-address 0.0.0.0
+        --mainnet
+        +RTS -N -A16m -qg -qb -RTS
+
+    user: ${USER_ID}:${GROUP_ID}
+    restart: on-failure
+    logging:
+      driver: "json-file"
+      options:
+        compress: "true"
+        max-file: "10"
+        max-size: "50m"

run/mainnet/docker/run.sh

@@ -0,0 +1 @@
+../../common/docker/run.sh

run/mainnet/docker/snapshot.sh

@@ -0,0 +1 @@
+../../common/snapshot.sh

run/mainnet/nix/.env

@@ -0,0 +1 @@
+NETWORK="mainnet"

run/mainnet/nix/configs

@@ -0,0 +1 @@
+../../../configs/cardano/mainnet

run/mainnet/nix/run.sh

@@ -0,0 +1 @@
+../../common/nix/run.sh

run/private/docker/.env

@@ -0,0 +1 @@
+NETWORK="private"

run/private/nix/.env

@@ -0,0 +1 @@
+NETWORK="private"

run/sanchonet/docker/.env

@@ -0,0 +1 @@
+NETWORK="sanchonet"

run/sanchonet/nix/.env

@@ -0,0 +1 @@
+NETWORK="sanchonet"