Feature: add feature-flag: bt enables backtrace

`--features bt` enables backtrace when generating errors.
By default errors does not contain backtrace info.

Thus openraft can be built on stable rust by default.

To use on stable rust with backtrace, set `RUSTC_BOOTSTRAP=1`, e.g.:
```
RUSTUP_TOOLCHAIN=stable RUSTC_BOOTSTRAP=1 make test
```

- fix: #241

commit-id:933ca964

.github/workflows/ci.yaml

@@ -62,6 +62,63 @@ jobs:
           args: --all-targets -- -D warnings -A clippy::bool-assert-comparison
 
 
+      - name: Upload artifact
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          path: |
+            openraft/_log/
+
+  ut-on-stable-rust:
+    name: unittest on stable rust
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Setup | Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup | Toolchain
+        uses: actions-rs/toolchain@v1.0.6
+        with:
+          toolchain: "stable"
+          override: true
+          components: rustfmt, clippy
+
+      - name: Unit Tests
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+        env:
+          # Parallel tests block each other and result in timeout.
+          RUST_TEST_THREADS: 2
+          RUST_LOG: debug
+          RUST_BACKTRACE: full
+
+
+      - name: Test example-raft-kv
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --manifest-path example-raft-kv/Cargo.toml
+
+
+      - name: Build | Release Mode | No features
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --release
+
+
+      - name: Build | Release Mode | No features
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --release --all-features
+        env:
+          # Enable unstable feature on stalbe rust.
+          RUSTC_BOOTSTRAP: 1
+
+
       - name: Upload artifact
         uses: actions/upload-artifact@v2
         if: failure()

example-raft-kv/Cargo.toml

@@ -20,7 +20,6 @@ path = "src/bin/main.rs"
 
 [dependencies]
 actix-web = "4.0.0-rc.2"
-anyerror = { version = "0.1.4"}
 async-trait = "0.1.36"
 clap = { version = "3.0.13", features = ["derive", "env"] }
 env_logger = "0.9.0"

example-raft-kv/src/store/mod.rs

@@ -5,12 +5,12 @@ use std::ops::RangeBounds;
 use std::sync::Arc;
 use std::sync::Mutex;
 
-use anyerror::AnyError;
 use openraft::async_trait::async_trait;
 use openraft::raft::Entry;
 use openraft::raft::EntryPayload;
 use openraft::storage::LogState;
 use openraft::storage::Snapshot;
+use openraft::AnyError;
 use openraft::EffectiveMembership;
 use openraft::ErrorSubject;
 use openraft::ErrorVerb;

example-raft-kv/tests/cluster/test_cluster.rs

@@ -1,14 +1,14 @@
 use std::thread;
 use std::time::Duration;
 
-use anyerror::AnyError;
 use example_raft_key_value::client::ExampleClient;
 use example_raft_key_value::start_example_raft_node;
 use example_raft_key_value::store::ExampleRequest;
 use example_raft_key_value::ExampleTypeConfig;
 use maplit::btreemap;
 use maplit::btreeset;
 use openraft::error::NodeNotFound;
+use openraft::AnyError;
 use openraft::Node;
 use tokio::runtime::Runtime;
 

memstore/Cargo.toml

@@ -16,7 +16,6 @@ repository = "https://github.com/datafuselabs/openraft"
 readme = "README.md"
 
 [dependencies]
-anyerror = { version = "0.1.4"}
 openraft = { version="0.6", path= "../openraft" }
 async-trait = "0.1.36"
 serde = { version="1.0.114", features=["derive"] }

memstore/src/lib.rs

@@ -1,5 +1,3 @@
-#![feature(backtrace)]
-
 #[cfg(test)]
 mod test;
 
@@ -11,14 +9,14 @@ use std::ops::RangeBounds;
 use std::sync::Arc;
 use std::sync::Mutex;
 
-use anyerror::AnyError;
 use openraft::async_trait::async_trait;
 use openraft::raft::Entry;
 use openraft::raft::EntryPayload;
 use openraft::storage::LogState;
 use openraft::storage::RaftLogReader;
 use openraft::storage::RaftSnapshotBuilder;
 use openraft::storage::Snapshot;
+use openraft::AnyError;
 use openraft::EffectiveMembership;
 use openraft::ErrorSubject;
 use openraft::ErrorVerb;

openraft/Cargo.toml

@@ -16,7 +16,9 @@ repository = "https://github.com/datafuselabs/openraft"
 readme = "../README.md"
 
 [dependencies]
-anyerror = { version = "0.1.4", features = ["anyhow"]}
+anyerror = { version = "0.1.6" }
+# Leave it for debug
+# anyerror = { git = "https://github.com/drmingdrmer/anyerror", branch = "ci",  features = ["anyhow"]}
 async-trait = "0.1.36"
 byte-unit = "4.0.12"
 bytes = "1.0"
@@ -44,5 +46,9 @@ tracing-subscriber = { version = "0.3.3",  features=["env-filter"] }
 [features]
 docinclude = [] # Used only for activating `doc(include="...")` on nightly.
 
+# Enable backtrace when generating an error.
+# Stable rust does not support backtrace.
+bt  = ["anyerror/backtrace", "anyhow/backtrace"]
+
 [package.metadata.docs.rs]
 features = ["docinclude"] # Activate `docinclude` during docs.rs build.

openraft/src/core/mod.rs

@@ -578,7 +578,8 @@ impl<C: RaftTypeConfig, N: RaftNetworkFactory<C>, S: RaftStorage<C>> RaftCore<C,
                     Ok(res) => match res {
                         Ok(snapshot) => {
                             let _ = tx_compaction.try_send(SnapshotUpdate::SnapshotComplete(snapshot.meta.last_log_id));
-                            let _ = chan_tx.send(snapshot.meta.last_log_id.index); // This will always succeed.
+                            // This will always succeed.
+                            let _ = chan_tx.send(snapshot.meta.last_log_id.index);
                         }
                         Err(err) => {
                             tracing::error!({error=%err}, "error while generating snapshot");

openraft/src/lib.rs

@@ -1,5 +1,11 @@
 #![doc = include_str!("../README.md")]
-#![feature(backtrace)]
+#![cfg_attr(feature = "bt", feature(backtrace))]
+
+//! # Feature flags
+//!
+//! - `bt`: Enable backtrace: generate backtrace for errors. This requires a unstable feature `backtrace` thus it can
+//!   not be used with stable rust, unless explicity allowing using unstable features in stable rust with
+//!   `RUSTC_BOOTSTRAP=1`.
 
 mod config;
 mod core;
@@ -28,6 +34,8 @@ mod leader_metrics_test;
 #[cfg(test)]
 mod metrics_wait_test;
 
+pub use anyerror;
+pub use anyerror::AnyError;
 pub use async_trait;
 use serde::de::DeserializeOwned;
 use serde::Serialize;

openraft/src/storage_error.rs

@@ -1,4 +1,3 @@
-use std::backtrace::Backtrace;
 use std::fmt::Formatter;
 use std::ops::Bound;
 
@@ -45,15 +44,15 @@ pub struct DefensiveError<C: RaftTypeConfig> {
     /// The description of the violation.
     pub violation: Violation<C>,
 
-    pub backtrace: String,
+    pub backtrace: Option<String>,
 }
 
 impl<C: RaftTypeConfig> DefensiveError<C> {
     pub fn new(subject: ErrorSubject<C>, violation: Violation<C>) -> Self {
         Self {
             subject,
             violation,
-            backtrace: format!("{:?}", Backtrace::capture()),
+            backtrace: anyerror::backtrace_str(),
         }
     }
 }
@@ -168,15 +167,15 @@ pub enum StorageError<C: RaftTypeConfig> {
     #[error(transparent)]
     Defensive {
         #[from]
-        #[backtrace]
+        #[cfg_attr(feature = "bt", backtrace)]
         source: DefensiveError<C>,
     },
 
     /// An error raised by io operation.
     #[error(transparent)]
     IO {
         #[from]
-        #[backtrace]
+        #[cfg_attr(feature = "bt", backtrace)]
         source: StorageIOError<C>,
     },
 }
@@ -209,7 +208,7 @@ pub struct StorageIOError<C: RaftTypeConfig> {
     subject: ErrorSubject<C>,
     verb: ErrorVerb,
     source: AnyError,
-    backtrace: String,
+    backtrace: Option<String>,
 }
 
 impl<C: RaftTypeConfig> std::fmt::Display for StorageIOError<C> {
@@ -224,8 +223,7 @@ impl<C: RaftTypeConfig> StorageIOError<C> {
             subject,
             verb,
             source,
-            // TODO: use crate backtrace instead of std::backtrace.
-            backtrace: format!("{:?}", Backtrace::capture()),
+            backtrace: anyerror::backtrace_str(),
         }
     }
 }

openraft/tests/append_entries/t50_append_entries_with_bigger_term.rs

@@ -42,14 +42,17 @@ async fn append_entries_with_bigger_term() -> Result<()> {
     assert!(resp.is_success());
 
     // after append entries, check hard state in term 2 and vote for node 1
+    let mut store = router.get_storage_handle(&0)?;
+
     router
-        .assert_storage_state_in_node(
-            0,
+        .assert_storage_state_with_sto(
+            &mut store,
+            &0,
             2,
             log_index,
             Some(1),
             LogId::new(LeaderId::new(1, 0), log_index),
-            None,
+            &None,
         )
         .await?;
 

openraft/tests/fixtures/mod.rs

@@ -839,39 +839,6 @@ where
         Ok(())
     }
 
-    /// Assert against the state of the storage system one node in the cluster.
-    pub async fn assert_storage_state_in_node(
-        &self,
-        node_id: C::NodeId,
-        expect_term: u64,
-        expect_last_log: u64,
-        expect_voted_for: Option<C::NodeId>,
-        expect_sm_last_applied_log: LogId<C::NodeId>,
-        expect_snapshot: Option<(ValueTest<u64>, u64)>,
-    ) -> anyhow::Result<()> {
-        let mut rt = self.routing_table.lock().unwrap();
-
-        for (id, (_node, storage)) in rt.iter_mut() {
-            if *id != node_id {
-                continue;
-            }
-            self.assert_storage_state_with_sto(
-                storage,
-                id,
-                expect_term,
-                expect_last_log,
-                expect_voted_for,
-                expect_sm_last_applied_log,
-                &expect_snapshot,
-            )
-            .await?;
-
-            break;
-        }
-
-        Ok(())
-    }
-
     /// Assert against the state of the storage system per node in the cluster.
     pub async fn assert_storage_state(
         &self,
@@ -881,12 +848,18 @@ where
         expect_sm_last_applied_log: LogId<C::NodeId>,
         expect_snapshot: Option<(ValueTest<u64>, u64)>,
     ) -> anyhow::Result<()> {
-        let mut rt = self.routing_table.lock().unwrap();
+        let node_ids = {
+            let rt = self.routing_table.lock().unwrap();
+            let node_ids = rt.keys().cloned().collect::<Vec<_>>();
+            node_ids
+        };
+
+        for id in node_ids {
+            let mut storage = self.get_storage_handle(&id)?;
 
-        for (id, (_node, storage)) in rt.iter_mut() {
             self.assert_storage_state_with_sto(
-                storage,
-                id,
+                &mut storage,
+                &id,
                 expect_term,
                 expect_last_log,
                 expect_voted_for,

openraft/tests/snapshot/snapshot_chunk_size.rs

@@ -101,27 +101,31 @@ async fn snapshot_chunk_size() -> Result<()> {
 
         // after add_learner, log_index + 1,
         // leader has only log_index log in snapshot, cause it has compacted before add_learner
+        let mut store = router.get_storage_handle(&0)?;
         router
-            .assert_storage_state_in_node(
-                0,
+            .assert_storage_state_with_sto(
+                &mut store,
+                &0,
                 1,
                 log_index,
                 Some(0),
                 LogId::new(LeaderId::new(1, 0), log_index),
-                Some(((log_index - 1).into(), 1)),
+                &Some(((log_index - 1).into(), 1)),
             )
             .await?;
 
         // learner has log_index + 1 log in snapshot, cause it do compact after add_learner,
         // so learner's snapshot include add_learner log
+        let mut store = router.get_storage_handle(&1)?;
         router
-            .assert_storage_state_in_node(
-                1,
+            .assert_storage_state_with_sto(
+                &mut store,
+                &1,
                 1,
                 log_index,
                 Some(0),
                 LogId::new(LeaderId::new(1, 0), log_index),
-                Some(((log_index).into(), 1)),
+                &Some(((log_index).into(), 1)),
             )
             .await?;
     }

rust-toolchain

@@ -1 +1 @@
-nightly-2022-02-07
+nightly-2022-03-03