diff --git a/Cargo.lock b/Cargo.lock index 551cf12a..e7c8cc9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -856,7 +856,7 @@ dependencies = [ name = "blocklist-client" version = "0.0.1" dependencies = [ - "config", + "config 0.11.0", "mockito 0.28.0", "once_cell", "reqwest 0.11.27", @@ -1095,7 +1095,7 @@ checksum = "1b1b9d958c2b1368a663f05538fc1b5975adce1e19f435acceae987aceeeb369" dependencies = [ "lazy_static", "nom 5.1.3", - "rust-ini", + "rust-ini 0.13.0", "serde 1.0.202", "serde-hjson", "serde_json", @@ -1103,18 +1103,67 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "config" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7328b20597b53c2454f0b1919720c25c7339051c02b72b7e05409e00b14132be" +dependencies = [ + "async-trait", + "convert_case", + "json5", + "lazy_static", + "nom 7.1.3", + "pathdiff", + "ron", + "rust-ini 0.19.0", + "serde 1.0.202", + "serde_json", + "toml 0.8.13", + "yaml-rust", +] + [[package]] name = "const-oid" version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.15", + "once_cell", + "tiny-keccak", +] + [[package]] name = "const_fn" version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "373e9fafaa20882876db20562275ff58d50e0caa2590077fe7ce7bef90211d0d" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1369,6 +1418,15 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -1878,6 +1936,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" + [[package]] name = "hashbrown" version = "0.14.5" @@ -2394,6 +2458,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde 1.0.202", +] + [[package]] name = "jsonrpc" version = "0.18.0" @@ -3004,6 +3079,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-multimap" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ed8acf08e98e744e5384c8bc63ceb0364e68a6854187221c18df61c4797690e" +dependencies = [ + "dlv-list", + "hashbrown 0.13.2", +] + [[package]] name = "outref" version = "0.5.1" @@ -3094,6 +3179,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pathdiff" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -3115,6 +3206,51 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pest" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26293c9193fbca7b1a3bf9b79dc1e388e927e6cacaa78b4a3ab705a1d3d41459" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.65", +] + +[[package]] +name = "pest_meta" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + [[package]] name = "petgraph" version = "0.6.5" @@ -3664,6 +3800,18 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "ron" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" +dependencies = [ + "base64 0.21.7", + "bitflags 2.5.0", + "serde 1.0.202", + "serde_derive", +] + [[package]] name = "rsa" version = "0.9.6" @@ -3743,6 +3891,16 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" +[[package]] +name = "rust-ini" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e2a3bcec1f113553ef1c88aae6c020a369d03d55b58de9869a0908930385091" +dependencies = [ + "cfg-if 1.0.0", + "ordered-multimap", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -4282,7 +4440,7 @@ dependencies = [ "bitcoincore-rpc", "blocklist-api", "clap", - "config", + "config 0.14.0", "electrum-client", "fake", "futures", @@ -4310,6 +4468,7 @@ dependencies = [ "tracing", "tracing-attributes", "tracing-subscriber", + "url", "wsts", ] @@ -5008,6 +5167,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -5366,6 +5534,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + [[package]] name = "uint" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index 482bdb8a..c62fc3e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ tokio = "1.32.0" tokio-stream = {version = "0.1.15", features = ["sync"] } tracing = { version = "0.1", default-features = false } tracing-attributes = "0.1" +url = "2.5" wsts = "9.1.0" [workspace.dependencies.tracing-subscriber] diff --git a/devenv/local/docker-compose/bitcoin-miner-sidecar/docker/Dockerfile b/devenv/local/docker-compose/bitcoin-miner-sidecar/docker/Dockerfile index e6916e87..2aaeb4d7 100644 --- a/devenv/local/docker-compose/bitcoin-miner-sidecar/docker/Dockerfile +++ b/devenv/local/docker-compose/bitcoin-miner-sidecar/docker/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:latest -MAINTAINER Gowtham Sundar +LABEL org.opencontainers.image.authors="Gowtham Sundar " RUN apk add --no-cache curl jq diff --git a/devenv/local/docker-compose/bitcoin/docker/Dockerfile b/devenv/local/docker-compose/bitcoin/docker/Dockerfile index 1b673921..2ea3ea2b 100644 --- a/devenv/local/docker-compose/bitcoin/docker/Dockerfile +++ b/devenv/local/docker-compose/bitcoin/docker/Dockerfile @@ -1,5 +1,5 @@ FROM debian:stable-slim as builder -MAINTAINER Gowtham Sundar +LABEL org.opencontainers.image.authors="Gowtham Sundar " ARG VERSION=25.0 diff --git a/devenv/local/docker-compose/electrs/docker/Dockerfile b/devenv/local/docker-compose/electrs/docker/Dockerfile index 9319c0b2..5aba08e3 100644 --- a/devenv/local/docker-compose/electrs/docker/Dockerfile +++ b/devenv/local/docker-compose/electrs/docker/Dockerfile @@ -4,7 +4,7 @@ # -------------------------------------------------------- FROM debian:bookworm-slim as builder -MAINTAINER Gowtham Sundar +LABEL org.opencontainers.image.authors="Gowtham Sundar " RUN apt-get update -qqy RUN apt-get install -qqy librocksdb-dev curl git # -------------------------------------------------------- diff --git a/devenv/local/docker-compose/nakamoto-signer/docker/Dockerfile b/devenv/local/docker-compose/nakamoto-signer/docker/Dockerfile index fb8d57ba..40602bfc 100644 --- a/devenv/local/docker-compose/nakamoto-signer/docker/Dockerfile +++ b/devenv/local/docker-compose/nakamoto-signer/docker/Dockerfile @@ -1,5 +1,5 @@ FROM blockstack/stacks-core:2.5.0.0.3 -MAINTAINER Gowtham Sundar +LABEL org.opencontainers.image.authors="Gowtham Sundar " COPY . . ARG SIGNER_ENDPOINT diff --git a/devenv/local/docker-compose/stacks-api/docker/Dockerfile b/devenv/local/docker-compose/stacks-api/docker/Dockerfile index 984a3337..99c191b6 100644 --- a/devenv/local/docker-compose/stacks-api/docker/Dockerfile +++ b/devenv/local/docker-compose/stacks-api/docker/Dockerfile @@ -1,5 +1,5 @@ FROM node:16-alpine -MAINTAINER Gowtham Sundar +LABEL org.opencontainers.image.authors="Gowtham Sundar " ARG GIT_URI='https://github.com/hirosystems/stacks-blockchain-api.git' ARG GIT_BRANCH='v7.10.0-nakamoto.7' diff --git a/devenv/local/docker-compose/stacks-explorer/docker/Dockerfile b/devenv/local/docker-compose/stacks-explorer/docker/Dockerfile index c2469a03..17230e50 100644 --- a/devenv/local/docker-compose/stacks-explorer/docker/Dockerfile +++ b/devenv/local/docker-compose/stacks-explorer/docker/Dockerfile @@ -1,6 +1,6 @@ # -------------------------------------------------------- FROM node:18-alpine AS build -MAINTAINER Gowtham Sundar +LABEL org.opencontainers.image.authors="Gowtham Sundar " ARG GIT_URI='https://github.com/hirosystems/explorer.git' diff --git a/devenv/local/docker-compose/stacks/docker/Dockerfile b/devenv/local/docker-compose/stacks/docker/Dockerfile index f0cda085..9a61cb1d 100644 --- a/devenv/local/docker-compose/stacks/docker/Dockerfile +++ b/devenv/local/docker-compose/stacks/docker/Dockerfile @@ -1,5 +1,5 @@ FROM blockstack/stacks-core:2.5.0.0.3 -MAINTAINER Gowtham Sundar +LABEL org.opencontainers.image.authors="Gowtham Sundar " COPY . . ARG STACKS_LOG_DEBUG diff --git a/devenv/local/docker-compose/tests/devnet-liveness.sh b/devenv/local/docker-compose/tests/devnet-liveness.sh old mode 100644 new mode 100755 diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 818c6adf..7e919f7a 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -11,17 +11,125 @@ services: - 8333:8333 - 8332:8332 - 18443:18443 + - 18444:18444 - 28332:28332 - 28333:28333 + healthcheck: + test: ["CMD-SHELL", "bitcoin-cli getblockcount"] + interval: 3s + timeout: 1s + retries: 3 - web: + # During proof-of-transfer Stacks nodes spend BTC to stackers. In order to + # do that each node must have some UTXOs at their disposal. The particular + # path that the stacks-node has taken is to be able to sign for some UTXOs + # based on the private key that is written in its configuration file. The + # address that the stacks node searches for when looking for UTXOs is a + # legacy p2pkh address or a p2wpkh segwit address. The private key used here + # is 0000000000000000000000000000000000000000000000000000000000000001. The + # address here is p2pkh of the assoicated public key, although it appears + # to be different from the public key that rust-bitcoin would generate for + # the same key. + # + # We mine 110 bitcoin blocks so that stacks-node epoch 2.5 is activated, + # which is configured (in stacks-node-config.toml) to happen at block 108. + # + # Note that Stacks nodes don't really get up and running until it sees two + # bitcoin blocks (with enough time to react to each of them). + bitcoin-miner: + image: lncm/bitcoind:v27.0 + user: "1000:1000" + stop_grace_period: 5s + volumes: + - ./signer/tests/bitcoin/bitcoin.conf:/data/.bitcoin/bitcoin.conf:ro + entrypoint: + - /bin/sh + - -c + - > + bitcoin-cli -rpcconnect=bitcoind generatetoaddress 110 "mtoKs9V381UAhUia3d7Vb9GNak8Qvmcsme" + + while true; do + bitcoin-cli -rpcconnect=bitcoind generatetoaddress 1 "mtoKs9V381UAhUia3d7Vb9GNak8Qvmcsme" + sleep 5 + done + depends_on: + stacks-node: + condition: service_started + + # This stacks node is configured to use the bitcoind container above + # when looking for UTXOs and for submitting transactions. + stacks-node: + image: blockstack/stacks-core:2.5.0.0.3-debian + user: "1000:1000" + command: stacks-node start --config /config.toml + stop_grace_period: 5s + ports: + - 20443:20443 + - 20444:20444 + depends_on: + bitcoind: + condition: service_healthy + volumes: + - ./signer/tests/stacks/stacks-node-config.toml:/config.toml:ro + environment: + RUST_BACKTRACE: "full" + BLOCKSTACK_DEBUG: 0 + + stacks-signer: + image: blockstack/stacks-core:2.5.0.0.3-debian + user: "1000:1000" + command: stacks-signer run --config /config.toml + stop_grace_period: 5s + ports: + - 30000:30000 + depends_on: + bitcoind: + condition: service_healthy + environment: + RUST_BACKTRACE: "full" + BLOCKSTACK_DEBUG: 0 + volumes: + - ./signer/tests/stacks/stacks-signer-config.toml:/config.toml:ro + + stacks-api: + image: hirosystems/stacks-blockchain-api:7.11.0-beta.7 + user: "1000:1000" + stop_grace_period: 5s + ports: + - 3700:3700 + - 3999:3999 + - 9153:9153 + environment: + NODE_ENV: production + GIT_TAG: master + PG_HOST: postgres + PG_PORT: 5432 + PG_USER: user + PG_PASSWORD: password + PG_DATABASE: signer + STACKS_CHAIN_ID: "0x80000000" + V2_POX_MIN_AMOUNT_USTX: 90000000260 + STACKS_CORE_EVENT_PORT: 3700 + STACKS_CORE_EVENT_HOST: 0.0.0.0 + STACKS_BLOCKCHAIN_API_PORT: 3999 + STACKS_BLOCKCHAIN_API_HOST: 0.0.0.0 + STACKS_BLOCKCHAIN_API_DB: pg #default + STACKS_CORE_RPC_HOST: stacks-node + STACKS_CORE_RPC_PORT: 20443 + depends_on: + stacks-node: + condition: service_started + postgres: + condition: service_healthy + + mempool-web: environment: FRONTEND_HTTP_PORT: "8080" BACKEND_MAINNET_HTTP_HOST: "mempool-api" image: mempool/frontend:v3.0.0-dev8 user: "1000:1000" restart: on-failure - stop_grace_period: 10s + stop_grace_period: 5s command: "./wait-for mempooldb:3306 --timeout=720 -- nginx -g 'daemon off;'" ports: - 80:8080 @@ -42,21 +150,21 @@ services: image: mempool/backend:v3.0.0-dev8 user: "1000:1000" restart: on-failure - stop_grace_period: 10s + stop_grace_period: 5s command: "./wait-for-it.sh mempooldb:3306 --timeout=720 --strict -- ./start.sh" volumes: - /tmp/mempool-api/data:/backend/cache mempooldb: + image: mariadb:10.5.21 + user: "1000:1000" environment: MYSQL_DATABASE: "mempool" MYSQL_USER: "mempool" MYSQL_PASSWORD: "mempool" MYSQL_ROOT_PASSWORD: "admin" - image: mariadb:10.5.21 - user: "1000:1000" restart: on-failure - stop_grace_period: 10s + stop_grace_period: 5s volumes: - /tmp/mysql/data:/var/lib/mysql @@ -71,6 +179,11 @@ services: - "5432:5432" volumes: - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready --username=user --dbname=signer"] + interval: 2s + timeout: 1s + retries: 5 pgadmin: image: dpage/pgadmin4:8.6 @@ -81,7 +194,8 @@ services: ports: - "8080:80" depends_on: - - postgres + postgres: + condition: service_healthy volumes: postgres_data: diff --git a/signer/Cargo.toml b/signer/Cargo.toml index 0abb147d..3a23c14d 100644 --- a/signer/Cargo.toml +++ b/signer/Cargo.toml @@ -24,7 +24,7 @@ bincode.workspace = true bitcoin = { workspace = true, features = ["rand-std"] } blocklist-api = { path = "../.generated-sources/blocklist-api" } clap.workspace = true -config.workspace = true +config = "0.14" electrum-client = "0.19.0" futures.workspace = true once_cell.workspace = true @@ -46,6 +46,7 @@ tokio-stream.workspace = true tracing.workspace = true tracing-attributes.workspace = true tracing-subscriber.workspace = true +url.workspace = true wsts.workspace = true # Only for testing diff --git a/signer/README.md b/signer/README.md index ce682789..d2cb98fb 100644 --- a/signer/README.md +++ b/signer/README.md @@ -17,6 +17,8 @@ You can run the signer by providing the required environment variables. If not specified, the default values from `./src/config/default.toml` will be used. - `SIGNER_BLOCKLIST_CLIENT__HOST`=`` - `SIGNER_BLOCKLIST_CLIENT__PORT`=`` +- `SIGNER_STACKS_API_ENDPOINT`=`` +- `SIGNER_STACKS_NODE_ENDPOINT`=`` #### Inspecting the signer database The signer state will be stored in a `sbtc_signer` schema in the provided database at `$DATABASE_URL`. diff --git a/signer/src/block_observer.rs b/signer/src/block_observer.rs index 05cea7a9..a0d66a83 100644 --- a/signer/src/block_observer.rs +++ b/signer/src/block_observer.rs @@ -19,6 +19,7 @@ use std::collections::HashMap; +use crate::stacks_api::StacksInteract; use crate::storage; use bitcoin::hashes::Hash; @@ -149,7 +150,7 @@ where let stacks_blocks = self .stacks_client .get_blocks_by_bitcoin_block(&block.block_hash()) - .await; + .await?; self.extract_deposit_requests(&block.txdata); self.extract_sbtc_transactions(&block.txdata); @@ -240,15 +241,6 @@ pub trait BitcoinInteract { ) -> impl std::future::Future>; } -/// Placeholder trait -pub trait StacksInteract { - /// Get stacks blocks confirmed by the given bitcoin block - fn get_blocks_by_bitcoin_block( - &mut self, - bitcoin_block_hash: &bitcoin::BlockHash, - ) -> impl std::future::Future>; -} - /// Placeholder trait pub trait EmilyInteract { /// Get deposits @@ -271,6 +263,9 @@ pub enum Error { /// Storage error #[error("storage error")] StorageError, + /// Crate error + #[error("Client error maybe {0}")] + StacksClient(#[from] crate::error::Error), } #[cfg(test)] @@ -406,13 +401,14 @@ mod tests { impl StacksInteract for TestHarness { async fn get_blocks_by_bitcoin_block( - &mut self, + &self, bitcoin_block_hash: &bitcoin::BlockHash, - ) -> Vec { - self.stacks_blocks_per_bitcoin_block + ) -> Result, crate::error::Error> { + Ok(self + .stacks_blocks_per_bitcoin_block .get(bitcoin_block_hash) .cloned() - .unwrap_or_else(Vec::new) + .unwrap_or_else(Vec::new)) } } diff --git a/signer/src/config.rs b/signer/src/config.rs index c0cc9c68..b3644769 100644 --- a/signer/src/config.rs +++ b/signer/src/config.rs @@ -3,6 +3,9 @@ use config::{Config, ConfigError, Environment, File}; use once_cell::sync::Lazy; use serde::Deserialize; +use serde::Deserializer; + +use crate::error::Error; /// Top-level configuration for the signer #[derive(Deserialize, Clone, Debug)] @@ -45,11 +48,15 @@ impl Settings { /// Initializing the global config first with default values and then with provided/overwritten environment variables. /// The explicit separator with double underscores is needed to correctly parse the nested config structure. pub fn new() -> Result { - let mut cfg = Config::new(); - cfg.merge(File::with_name("./src/config/default"))?; - let env = Environment::with_prefix("SIGNER").separator("__"); - cfg.merge(env)?; - let settings: Settings = cfg.try_into()?; + let env = Environment::with_prefix("SIGNER") + .separator("__") + .prefix_separator("_"); + let cfg = Config::builder() + .add_source(File::with_name("./src/config/default")) + .add_source(env) + .build()?; + + let settings: Settings = cfg.try_deserialize()?; settings.validate()?; @@ -73,3 +80,112 @@ impl Settings { Ok(()) } } + +/// A deserializer for the url::Url type. +fn url_deserializer<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + String::deserialize(deserializer)? + .parse() + .map_err(serde::de::Error::custom) +} + +/// A struct for the entries in the signers Config.toml (which is currently +/// located in src/config/default.toml) +#[derive(Debug, Clone, serde::Deserialize)] +pub struct StacksSettings { + /// The configuration entries related to the Stacks API + pub api: StacksApiSettings, + /// The configuration entries related to the Stacks node + pub node: StacksNodeSettings, +} + +/// Whatever +#[derive(Debug, Clone, serde::Deserialize)] +pub struct StacksApiSettings { + /// TODO(225): We'll want to support specifying multiple Stacks API + /// endpoints. + /// + /// The endpoint to use when making requests to the stacks API. + #[serde(deserialize_with = "url_deserializer")] + pub endpoint: url::Url, +} + +/// Settings associated with the stacks node that this signer uses for information +#[derive(Debug, Clone, serde::Deserialize)] +pub struct StacksNodeSettings { + /// TODO(225): We'll want to support specifying multiple Stacks Nodes + /// endpoints. + /// + /// The endpoint to use when making requests to a stacks node. + #[serde(deserialize_with = "url_deserializer")] + pub endpoint: url::Url, +} + +impl StacksSettings { + /// Create a new StacksSettings object by reading the relevant entries + /// in the signer's config.toml. The values there can be overridden by + /// environment variables. + /// + /// # Notes + /// + /// The relevant environment variables and the config entries that are + /// overridden are: + /// + /// * SIGNER_STACKS_API_ENDPOINT <-> stacks.api.endpoint + /// * SIGNER_STACKS_NODE_ENDPOINT <-> stacks.node.endpoint + /// + /// Each of these overrides an entry in the signer's `config.toml` + pub fn new_from_config() -> Result { + let source = File::with_name("./src/config/default"); + let env = Environment::with_prefix("SIGNER") + .prefix_separator("_") + .separator("_"); + + let conf = Config::builder() + .add_source(source) + .add_source(env) + .build() + .map_err(Error::SignerConfig)?; + + conf.get::("stacks") + .map_err(Error::StacksApiConfig) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_config_toml_loads_with_environment() { + // The default toml used here specifies http://localhost:3999 + // as the stacks API endpoint. + let settings = StacksSettings::new_from_config().unwrap(); + let host = settings.api.endpoint.host(); + assert_eq!(host, Some(url::Host::Domain("localhost"))); + assert_eq!(settings.api.endpoint.port(), Some(3999)); + + std::env::set_var("SIGNER_STACKS_API_ENDPOINT", "http://whatever:1234"); + + let settings = StacksSettings::new_from_config().unwrap(); + let host = settings.api.endpoint.host(); + assert_eq!(host, Some(url::Host::Domain("whatever"))); + assert_eq!(settings.api.endpoint.port(), Some(1234)); + + std::env::set_var("SIGNER_STACKS_API_ENDPOINT", "http://127.0.0.1:5678"); + + let settings = StacksSettings::new_from_config().unwrap(); + let ip: std::net::Ipv4Addr = "127.0.0.1".parse().unwrap(); + assert_eq!(settings.api.endpoint.host(), Some(url::Host::Ipv4(ip))); + assert_eq!(settings.api.endpoint.port(), Some(5678)); + + std::env::set_var("SIGNER_STACKS_API_ENDPOINT", "http://[::1]:9101"); + + let settings = StacksSettings::new_from_config().unwrap(); + let ip: std::net::Ipv6Addr = "::1".parse().unwrap(); + assert_eq!(settings.api.endpoint.host(), Some(url::Host::Ipv6(ip))); + assert_eq!(settings.api.endpoint.port(), Some(9101)); + } +} diff --git a/signer/src/config/default.toml b/signer/src/config/default.toml index ec936705..9381284f 100644 --- a/signer/src/config/default.toml +++ b/signer/src/config/default.toml @@ -7,4 +7,10 @@ server = "tcp://localhost:60401" retry_interval = 10 max_retry_attempts = 5 ping_interval = 60 -subscribe_interval = 10 \ No newline at end of file +subscribe_interval = 10 + +[stacks.api] +endpoint = "http://localhost:3999" + +[stacks.node] +endpoint = "http://localhost:20443" diff --git a/signer/src/error.rs b/signer/src/error.rs index aea22221..083cf0f8 100644 --- a/signer/src/error.rs +++ b/signer/src/error.rs @@ -1,4 +1,5 @@ //! Top-level error type for the signer +use blockstack_lib::types::chainstate::StacksBlockId; /// Top-level signer error #[derive(Debug, thiserror::Error)] @@ -15,10 +16,42 @@ pub enum Error { #[error("Failed to get fee estimates from all fee estimate sources")] NoGoodFeeEstimates, + /// Parsing the Hex Error + #[error("Could not parse the Hex string to a StacksBlockId: {0}, original: {1}")] + ParseStacksBlockId(#[source] blockstack_lib::util::HexError, String), + + /// Parsing the Hex Error + #[error("Could not decode the Nakamoto block with ID: {1}; {0}")] + DecodeNakamotoBlock(#[source] blockstack_lib::codec::Error, StacksBlockId), + + /// Could not parse the path part of a url + #[error("{0}")] + PathParse(#[source] url::ParseError), + /// Reqwest error #[error("{0}")] Reqwest(#[from] reqwest::Error), + /// Error when reading the signer config.toml + #[error("Failed to read the signers config file: {0}")] + SignerConfig(#[source] config::ConfigError), + + /// Error when reading the stacks API part of the config.toml + #[error("Failed to parse the stacks.api portion of the config: {0}")] + StacksApiConfig(#[source] config::ConfigError), + + /// Could not make a successful request to the stacks API. + #[error("Failed to make a request to the stacks API at {1}: {0}")] + StacksApiRequest(#[source] reqwest::Error, url::Url), + + /// Could not make a successful request to the stacks node. + #[error("Failed to make a request to the stacks Node at {1}: {0}")] + StacksNodeRequest(#[source] reqwest::Error, url::Url), + + /// Reqwest error + #[error("Response did not conform the expected schema {0}")] + UnexpectedStacksResponse(#[source] reqwest::Error, url::Url), + /// Taproot error #[error("An error occured when constructing the taproot signing digest: {0}")] Taproot(#[from] bitcoin::sighash::TaprootError), diff --git a/signer/src/lib.rs b/signer/src/lib.rs index 408ae396..18b758f5 100644 --- a/signer/src/lib.rs +++ b/signer/src/lib.rs @@ -12,6 +12,7 @@ pub mod fees; pub mod message; pub mod network; pub mod packaging; +pub mod stacks_api; pub mod storage; #[cfg(feature = "testing")] pub mod testing; diff --git a/signer/src/stacks_api.rs b/signer/src/stacks_api.rs new file mode 100644 index 00000000..6c137534 --- /dev/null +++ b/signer/src/stacks_api.rs @@ -0,0 +1,193 @@ +//! A module with structs that interact with the Stacks API. + +use std::future::Future; +use std::time::Duration; + +use blockstack_lib::chainstate::nakamoto::NakamotoBlock; +use blockstack_lib::codec::StacksMessageCodec; +use blockstack_lib::types::chainstate::StacksBlockId; +use futures::StreamExt; +use serde::Deserialize; + +use crate::config::StacksSettings; +use crate::error::Error; + +const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); + +/// A trait detailing the interface with the Stacks API and Stacks Nodes. +pub trait StacksInteract { + /// Get stacks blocks confirmed by the given bitcoin block + fn get_blocks_by_bitcoin_block( + &self, + block_hash: &bitcoin::BlockHash, + ) -> impl Future, Error>> + Send; +} + +/// A client for interacting with Stacks nodes and the Stacks API +pub struct StacksClient { + /// The base URL (with the port) that will be used when making requests + /// for to the Stacks API. + pub api_endpoint: url::Url, + /// The base URL (with the port) that will be used when making requests + /// for to a Stacks node. + pub node_endpoint: url::Url, + /// The client used to make the request. + pub client: reqwest::Client, +} + +impl StacksClient { + /// Create a new instance of the Stacks client using the given + /// StacksSettings. + pub fn new(settings: StacksSettings) -> Self { + Self { + api_endpoint: settings.api.endpoint, + node_endpoint: settings.node.endpoint, + client: reqwest::Client::new(), + } + } + + /// Get Stacks block IDs given the bitcoin block hash. Uses the Stacks API + /// via the GET /extended/v2/burn-blocks/:height_or_hash endpoint. + /// + /// See https://docs.hiro.so/api/get-burn-block + async fn get_block_ids(&self, hash: &bitcoin::BlockHash) -> Result, Error> { + // The Stacks API expects the hash to be hex encoded with the + // leading 0x in the string, which is not produced by the Display + // implementation of bitcoin::hashes::sha256d::Hash (but is for + // the Debug implementation). + let hash: &bitcoin::hashes::sha256d::Hash = hash.as_raw_hash(); + let path = format!("/extended/v2/burn-blocks/0x{}", hash); + let url = self.api_endpoint.join(&path).map_err(Error::PathParse)?; + + tracing::debug!(%hash, "Fetching block IDs confirmed by bitcoin block from stacks API"); + + let response = self + .client + .get(url.clone()) + .timeout(REQUEST_TIMEOUT) + .send() + .await + .map_err(|err| Error::StacksNodeRequest(err, url.clone()))?; + let resp: GetBurnBlockResponse = response + .json() + .await + .map_err(|err| Error::UnexpectedStacksResponse(err, url))?; + + // The Stacks API often returns hex prefixed with 0x. If this is + // the case, we split it off before constructing the block ids. + resp.stacks_blocks + .into_iter() + .map(|hex_string| { + let hex_str: &str = if hex_string.starts_with("0x") { + hex_string.split_at(2).1 + } else { + &hex_string + }; + + StacksBlockId::from_hex(hex_str) + .map_err(|err| Error::ParseStacksBlockId(err, hex_string)) + }) + .collect() + } + + /// Fetch the raw stacks nakamoto block from a Stacks node given the + /// Stacks block ID. + async fn get_block(&self, block_id: StacksBlockId) -> Result { + let path = format!("/v3/blocks/{}", block_id.to_hex()); + let url = self.node_endpoint.join(&path).map_err(Error::PathParse)?; + + tracing::debug!(%block_id, "Making request to the stacks node for the raw nakamoto block"); + + let response = self + .client + .get(url.clone()) + .timeout(REQUEST_TIMEOUT) + .send() + .await + .map_err(|err| Error::StacksNodeRequest(err, url.clone()))?; + let resp = response + .bytes() + .await + .map_err(|err| Error::UnexpectedStacksResponse(err, url))?; + + NakamotoBlock::consensus_deserialize(&mut &*resp) + .map_err(|err| Error::DecodeNakamotoBlock(err, block_id)) + } +} + +impl StacksInteract for StacksClient { + async fn get_blocks_by_bitcoin_block( + &self, + block_hash: &bitcoin::BlockHash, + ) -> Result, Error> { + let block_ids = self.get_block_ids(block_hash).await?; + + let stream = block_ids + .into_iter() + .map(|block_id| self.get_block(block_id)); + let ans: Vec> = futures::stream::iter(stream) + .buffer_unordered(3) + .collect() + .await; + + ans.into_iter().collect() + } +} + +/// Response from the Stacks API for GET /extended/v2/burn-blocks/:height_or_hash +/// requests. +/// +/// See https://docs.hiro.so/api/get-burn-block +#[derive(Clone, Debug, Deserialize)] +pub struct GetBurnBlockResponse { + /// The hash of the bitcoin block. + pub burn_block_hash: String, + /// The hash of the bitcoin block. + pub burn_block_height: u32, + /// Hashes of the Stacks blocks included in the bitcoin block + pub stacks_blocks: Vec, + /// The total number of Stacks transactions included in the stacks + /// blocks. + pub total_tx_count: u64, +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use super::*; + + #[tokio::test] + #[ignore = "This is an integration test that hasn't been setup for CI yet"] + async fn get_blocks_by_bitcoin_block_works() { + let block = bitcoin::BlockHash::from_str( + "00e34f99fc2d8e4857680cec4e8a74b64bebe53fe9d5752a8912dd777677043c", + ) + .unwrap(); + + let settings = StacksSettings::new_from_config().unwrap(); + let client = StacksClient::new(settings); + + let resp = client.get_block_ids(&block).await.unwrap(); + dbg!(resp); + } + + #[tokio::test] + #[ignore = "This is an integration test that hasn't been setup for CI yet"] + async fn get_blocks_works() { + let block = bitcoin::BlockHash::from_str( + "00e34f99fc2d8e4857680cec4e8a74b64bebe53fe9d5752a8912dd777677043c", + ) + .unwrap(); + + let settings = StacksSettings::new_from_config().unwrap(); + let client = StacksClient::new(settings); + + let block_ids = client.get_block_ids(&block).await.unwrap(); + let block_id = block_ids[0]; + + dbg!(&block_id); + let resp = client.get_block(block_id).await.unwrap(); + dbg!(resp); + } +} diff --git a/signer/tests/stacks/stacks-node-config.toml b/signer/tests/stacks/stacks-node-config.toml new file mode 100644 index 00000000..6461536c --- /dev/null +++ b/signer/tests/stacks/stacks-node-config.toml @@ -0,0 +1,235 @@ +[node] +working_dir = "/tmp/stacks" # Change to data directory you would like to use for your node +rpc_bind = "0.0.0.0:20443" +p2p_bind = "0.0.0.0:20444" +# Our other integration tests, and bootstrap bitcoin services mine bitcoin blocks +# with coinbase rewards that spend to an address associated with this private key. +# The docker service spends to a p2pkh address, while the integration tests spend +# to a p2wpkh address. +seed = "0000000000000000000000000000000000000000000000000000000000000001" +local_peer_seed = "0000000000000000000000000000000000000000000000000000000000000001" +mine_microblocks = false +pox_sync_sample_secs = 10 +wait_time_for_blocks = 10 +miner = true +# required if you are running a signer +stacker = true +use_test_genesis_chainstate = true + +[miner] +min_tx_fee = 1 +first_attempt_time_ms = 180_000 +subsequent_attempt_time_ms = 360_000 +wait_for_block_download = false +microblock_attempt_time_ms = 1000 +mining_key = "0000000000000000000000000000000000000000000000000000000000000001" +# There is a bug in stacks-core where regtest segwit addresses are not +# created correctly. Bitcoin p2pkh addresses are identical in regtest and testnet +# but p2wpkh addresses are different in regtest and testnet. Stacks-core generates +# testnet addresses when it should be generating regtest addresses. +segwit = false + +[connection_options] +disable_block_download = true +disable_inbound_handshakes = true +disable_inbound_walks = true +# Should match the value for the `auth_password` key in the stacks-signer config.toml. +block_proposal_token = "helloworld" +private_neighbors = false + +[[events_observer]] +endpoint = "stacks-signer:30000" +retry_count = 10 +include_data_events = false +events_keys = ["stackerdb", "block_proposal"] + +# # Add stacks-api as an event observer +[[events_observer]] +endpoint = "stacks-api:3700" +retry_count = 255 +# include_data_events = false +events_keys = ["*"] + + +[burnchain] +chain = "bitcoin" +mode = "nakamoto-neon" +# This is the same wallet name used in our other integration tests +wallet_name = "integration-tests-wallet" +magic_bytes = "T3" +pox_prepare_length = 8 +pox_reward_length = 20 +peer_host = "bitcoind" +username = "alice" +password = "pw" +rpc_port = 18443 +peer_port = 18444 +pox_2_activation = 104 +commit_anchor_block_within = 5 +burn_fee_cap = 20_000 +poll_time_secs = 1 +timeout = 30 + +[[burnchain.epochs]] +epoch_name = "1.0" +start_height = 0 + +[[burnchain.epochs]] +epoch_name = "2.0" +start_height = 0 + +[[burnchain.epochs]] +epoch_name = "2.05" +start_height = 102 + +[[burnchain.epochs]] +epoch_name = "2.1" +start_height = 103 + +[[burnchain.epochs]] +epoch_name = "2.2" +start_height = 105 + +[[burnchain.epochs]] +epoch_name = "2.3" +start_height = 106 + +[[burnchain.epochs]] +epoch_name = "2.4" +start_height = 107 + +[[burnchain.epochs]] +epoch_name = "2.5" +start_height = 108 + +[[burnchain.epochs]] +epoch_name = "3.0" +start_height = 130 + +[[ustx_balance]] +address = "ST0DZFQ1XGHC5P1BZ6B7HSWQKQJHM74JBGCSDTNA" +amount = 10000000000000000 + +[[ustx_balance]] +address = "ST2G2RJR4B5M95D0ZZAGZJP9J4WH090WHP0C5YW0H" +amount = 10000000000000000 + +[[ustx_balance]] +address = "ST3JCQJE9NZRCAPPE44Q12KR7FH8AY9HTEMWP2G5F" +amount = 10000000000000000 + +[[ustx_balance]] +address = "STA0EP5GD8FC661T8Q0Z382QW7Z6JXDM3E476MB7" +amount = 17500000000000 + +[[ustx_balance]] +address = "ST3MNK12DGQF7JN4Q0STK6926VWE5MN21KJ4EGV0E" +amount = 10000000000000000 + +[[ustx_balance]] +address = "ST484MS3VACPAZ90WHC21XQ7T6XANCV341HJYE0W" +amount = 10000000000000000 + +[[ustx_balance]] +address = "ST2D1M978SCE52GAV07VXSRC9DQBP69X5WHX0DHN5" +amount = 10000000000000000 + +[[ustx_balance]] +address = "ST2A68NMMXVZDWDTDZ5GJGA69M86V8KK0JS9X1QQP" +amount = 10000000000000000 + +[[ustx_balance]] +address = "ST2ME1CR5XR0P332SBTSD90P9HG48F1SK8MZVJ3XW" +amount = 10000000000000000 + +[[ustx_balance]] +address = "ST19MXV72S9HHRSZCDY10K9DMB11JYPTXVVNYAWPH" +amount = 10000000000000000 + +[[ustx_balance]] +address = "ST20Q2N56E1NBWE37R4VGSF89X4HHTB3GSMD8GKYW" +amount = 10000000000000000 + +[[ustx_balance]] +address = "ST2Q6124HQFKVKPJSS5J6156BJR74FD6EC1297HJ1" +amount = 10000000000000000 + +[[ustx_balance]] +address = "ST1114TBQYGNPGFAVXKWBKZAHP0X7ZGX9K6XYYE4F" +amount = 10000000000000000 + +[[ustx_balance]] +address = "ST1NCEQ0T4Z32QTYT88BNXJKC9HR3VWYHJ0TB95TP" +amount = 10000000000000000 + +[[ustx_balance]] +address = "STWF12K119FTA70NDG29MNYWR0CPMF44ZKC2SG2T" +amount = 24378281250000 + +[[ustx_balance]] +address = "ST36G5CRHH1GJVZGFWPTW4H9GSA8VAVWM0ST7AV82" +amount = 24378281250000 + +[[ustx_balance]] +address = "ST2KWFMX0SVXFMZ0W7TXZ3MV0C6V276BNAT49XAQW" +amount = 24378281250000 + +[[ustx_balance]] +address = "ST1ZMVDYKGWF5TFGH46GEFBR273JJ3RRTHEDETKNH" +amount = 24378281250000 + +[[ustx_balance]] +address = "ST3D0TEK871ZMBFFF0998YY609A1QGM6ZTYCQJJFQ" +amount = 24378281250000 + +[[ustx_balance]] +address = "ST372ND8K8M3GKESD0KG8ZWJ6EV0GGXWXC5246MJN" +amount = 24378281250000 + +[[ustx_balance]] +address = "ST33PA4H3TW3DQFHG2RXPGGW1FFG5YQJ704B3DA8M" +amount = 24378281250000 + +[[ustx_balance]] +address = "STJ737JNPK525J86BGSPAW362SRRAYC4SP6F95HC" +amount = 24378281250000 + +[[ustx_balance]] +address = "ST21AJANGK9NA2ZED5D5J1VZPTVW8DY05B0ECMFN" +amount = 24378281250000 + +[[ustx_balance]] +address = "ST30Z74A4S2T8563D844ENSBHBFSVQEVBPV9S0A7E" +amount = 24378281250000 + +[[ustx_balance]] +address = "ST2FGTGYAGJVXJZQX17NBJNSQAM4J2V5JFDHEEAZQ" +amount = 24378281250000 + +[[ustx_balance]] +address = "ST16PC3G9BMQH0G37JGAGDGYZPDB5NGNARBDFPWYB" +amount = 24378281250000 + +[[ustx_balance]] +address = "ST1XJHGBSQPV9B14HFYG98ZBSQGKG8GN0AMB3V2VT" +amount = 24378281250000 + +[[ustx_balance]] +address = "ST2XDC0R30841X2RRECWV2F9KTANKQEERPS4V3H9D" +amount = 24378281250000 + +[[ustx_balance]] +address = "ST2HC6JENRNNE6YVATT7WZVZWVR5J26BGYX67W8G7" +amount = 24378281250000 + +[[ustx_balance]] +address = "STPW2CGZC98EZ95XYC9DE93SFBS5KA2PYYK89VHM" +amount = 24378281250000 + +[[ustx_balance]] +address = "STNX3E9MYTA2ZDQK53YNMMJ3E7783DC019JZNYZZ" +amount = 24378281250000 + +[[ustx_balance]] +address = "ST0D135PF2R0S4B6S4G49QZC69KF19MSZ4Z5RDF5" +amount = 24378281250000 diff --git a/signer/tests/stacks/stacks-signer-config.toml b/signer/tests/stacks/stacks-signer-config.toml new file mode 100644 index 00000000..3f303e0e --- /dev/null +++ b/signer/tests/stacks/stacks-signer-config.toml @@ -0,0 +1,25 @@ +# The IP address and port where your Stacks node can be accessed. +# The port 20443 is the default RPC endpoint for Stacks nodes. +# Note that you must use an IP address - DNS hosts are not supported at this time. +node_host = "stacks-node:20443" + +# This is the location where the signer will expose an RPC endpoint for +# receiving events from your Stacks node. +endpoint = "0.0.0.0:30000" + +# Either “testnet” or “mainnet” +network = "testnet" + +# this is a file path where your signer will persist data. If using Docker, +# this must be within a volume, so that data can be persisted across restarts +db_path = "/var/stacks/signer.sqlite" + +# an authentication token that is used for some HTTP requests made from the +# signer to your Stacks node. You’ll need to use this later on when configuring +# your Stacks node. You create this field yourself, rather than it being generated +# with your private key. +auth_password = "helloworld" + +# This is the hex-encoded privateKey field from the keys you generated in the +# previous step. +stacks_private_key = "7287ba251d44a4d3fd9276c88ce34c5c52a038955511cccaf77e61068649c17801"