exposed internals of scenario results

input-output-hk · Nov 19, 2020 · e47157f · e47157f
1 parent 2dec5df
commit e47157f
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 16 deletions.
diff --git a/testing/jormungandr-scenario-tests/src/scenario/repository/mod.rs b/testing/jormungandr-scenario-tests/src/scenario/repository/mod.rs
@@ -102,20 +102,24 @@ impl ScenariosRepository {
     ) -> ScenarioSuiteResult {
         let mut suite_result = ScenarioSuiteResult::new();
         for scenario_to_run in available_scenarios {
+            if scenario_to_run.tags().contains(&Tag::Unstable) {
+                let scenario_result = ScenarioResult::ignored(scenario_to_run.name());
+
+                if self.report_unstable {
+                    println!("Scenario '{}' {}", scenario_to_run.name(), scenario_result);
+                }
+
+                suite_result.push(scenario_result);
+
+                continue;
+            }
+
             suite_result.push(self.run_single_scenario(
                 &scenario_to_run.name(),
                 &available_scenarios,
                 &mut context,
             ));
         }
-
-        if self.report_unstable {
-            for scenario in self.scenarios_tagged_by(Tag::Unstable) {
-                let scenario_result = ScenarioResult::ignored();
-                println!("Scenario '{}' {}", scenario.name(), scenario_result);
-                suite_result.push(scenario_result);
-            }
-        }
         suite_result
     }
 
@@ -146,7 +150,7 @@ impl ScenariosRepository {
                 std::panic::catch_unwind(|| scenario_to_run(context.clone().derive()))
             }
         };
-        let scenario_result = ScenarioResult::from_result(result);
+        let scenario_result = ScenarioResult::from_result(scenario.name(), result);
         println!("Scenario '{}' {}", scenario.name(), scenario_result);
         scenario_result
     }
@@ -183,7 +187,7 @@ fn scenarios_repository() -> Vec<Scenario> {
     repository.push(Scenario::new(
         "leader_restart",
         leader_restart,
-        vec![Tag::Unstable],
+        vec![Tag::Short, Tag::Unstable],
     ));
     repository.push(Scenario::new(
         "passive_node_is_updated",

diff --git a/testing/jormungandr-scenario-tests/src/scenario/repository/result.rs b/testing/jormungandr-scenario-tests/src/scenario/repository/result.rs
@@ -3,24 +3,28 @@ use std::{any::Any, fmt};
 
 #[derive(Clone, Debug)]
 pub struct ScenarioResult {
+    pub name: String,
     pub scenario_status: ScenarioStatus,
 }
 
 impl ScenarioResult {
-    pub fn passed() -> Self {
+    pub fn passed<S: Into<String>>(name: S) -> Self {
         ScenarioResult {
+            name: name.into(),
             scenario_status: ScenarioStatus::Passed,
         }
     }
 
-    pub fn failed<S: Into<String>>(reason: S) -> Self {
+    pub fn failed<P: Into<String>, S: Into<String>>(name: P, reason: S) -> Self {
         ScenarioResult {
+            name: name.into(),
             scenario_status: ScenarioStatus::Failed(reason.into()),
         }
     }
 
-    pub fn ignored() -> Self {
+    pub fn ignored<S: Into<String>>(name: S) -> Self {
         ScenarioResult {
+            name: name.into(),
             scenario_status: ScenarioStatus::Ignored,
         }
     }
@@ -41,15 +45,20 @@ impl ScenarioResult {
         matches!(*self.scenario_status(), ScenarioStatus::Passed)
     }
 
-    pub fn from_result(
+    pub fn name(&self) -> String {
+        self.name.to_string()
+    }
+
+    pub fn from_result<S: Into<String>>(
+        name: S,
         result: std::result::Result<Result<ScenarioResult>, std::boxed::Box<dyn Any + Send>>,
     ) -> ScenarioResult {
         match result {
             Ok(inner) => match inner {
                 Ok(scenario_result) => scenario_result,
-                Err(err) => ScenarioResult::failed(err.to_string()),
+                Err(err) => ScenarioResult::failed(name, err.to_string()),
             },
-            Err(_) => ScenarioResult::failed("no data".to_string()),
+            Err(_) => ScenarioResult::failed(name, "no data"),
         }
     }
 }

diff --git a/testing/jormungandr-scenario-tests/src/scenario/repository/scenario.rs b/testing/jormungandr-scenario-tests/src/scenario/repository/scenario.rs
@@ -17,6 +17,10 @@ impl Scenario {
         }
     }
 
+    pub fn tags(&self) -> &[Tag] {
+        &self.tags
+    }
+
     pub fn has_tag(&self, tag: Tag) -> bool {
         self.tags.iter().any(|t| *t == tag)
     }

diff --git a/testing/jormungandr-scenario-tests/src/scenario/repository/suite_result.rs b/testing/jormungandr-scenario-tests/src/scenario/repository/suite_result.rs
@@ -20,6 +20,10 @@ impl ScenarioSuiteResult {
         self.count_failed() > 0
     }
 
+    pub fn passed(&self) -> bool {
+        !self.is_failed()
+    }
+
     pub fn count_passed(&self) -> usize {
         self.results.iter().filter(|x| x.is_passed()).count()
     }
@@ -55,4 +59,8 @@ impl ScenarioSuiteResult {
         suite_result.push(result);
         suite_result
     }
+
+    pub fn results(&self) -> &[ScenarioResult] {
+        &self.results
+    }
 }