Add async test helper to timeout and provide a task_executor automati…

…cally (paritytech#6651)

* Initial commit

Forked at: 60e3a69
Parent branch: origin/master

* Add async test helper to timeout and provide a task_executor automatically

* simplify error message to avoid difference between CI and locally

* forgot env var

* Use runtime env var instead of build env var

* Rename variable to SUBSTRATE_TEST_TIMEOUT

* CLEANUP

Forked at: 60e3a69
Parent branch: origin/master

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Re-export from test-utils

* Default value to 120

* fix wrong crate in ci

* Revert "Default value to 120"

This reverts commit 8e45871.

* Fix version

* WIP

Forked at: 60e3a69
Parent branch: origin/master

* WIP

Forked at: 60e3a69
Parent branch: origin/master

* WIP

Forked at: 60e3a69
Parent branch: origin/master

* remove feature flag

* fix missing dependency

* CLEANUP

Forked at: 60e3a69
Parent branch: origin/master

* fix test

* Removed autotests=false

* Some doc...

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* WIP

Forked at: 60e3a69
Parent branch: origin/master

* WIP

Forked at: 60e3a69
Parent branch: origin/master

* Update test-utils/src/lib.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

.gitlab-ci.yml

@@ -263,6 +263,7 @@ test-linux-stable:                 &test-linux
   script:
     # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
     - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml
+    - WASM_BUILD_NO_COLOR=1 SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
     - sccache -s
 
 unleash-check:
@@ -727,7 +728,7 @@ deploy-kubernetes-alerting-rules:
     RULES:                         .maintain/monitoring/alerting-rules/alerting-rules.yaml
   script:
     - echo "deploying prometheus alerting rules"
-    - kubectl -n ${NAMESPACE} patch prometheusrule ${PROMETHEUSRULE} 
+    - kubectl -n ${NAMESPACE} patch prometheusrule ${PROMETHEUSRULE}
         --type=merge --patch "$(sed 's/^/  /;1s/^/spec:\n/' ${RULES})"
   only:
     refs:

Cargo.toml

@@ -171,9 +171,11 @@ members = [
 	"primitives/utils",
 	"primitives/wasm-interface",
 	"test-utils/client",
+	"test-utils/derive",
 	"test-utils/runtime",
 	"test-utils/runtime/client",
 	"test-utils/runtime/transaction-pool",
+	"test-utils/test-crate",
 	"utils/browser",
 	"utils/build-script-utils",
 	"utils/fork-tree",

test-utils/Cargo.toml

@@ -9,3 +9,12 @@ repository = "https://github.com/paritytech/substrate/"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+futures = { version = "0.3.1", features = ["compat"] }
+substrate-test-utils-derive = { path = "./derive" }
+tokio = { version = "0.2.13", features = ["macros"] }
+
+[dev-dependencies]
+sc-service = { path = "../client/service" }
+trybuild = { version = "1.0", features = ["diff"] }

test-utils/derive/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "substrate-test-utils-derive"
+version = "0.8.0-rc5"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "Apache-2.0"
+homepage = "https://substrate.dev"
+repository = "https://github.com/paritytech/substrate/"
+
+[dependencies]
+quote = "1.0.6"
+syn = { version = "1.0.33", features = ["full"] }
+proc-macro-crate = "0.1.4"
+
+[lib]
+proc-macro = true

test-utils/derive/src/lib.rs

@@ -0,0 +1,107 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use proc_macro::{Span, TokenStream};
+use proc_macro_crate::crate_name;
+use quote::quote;
+use std::env;
+
+#[proc_macro_attribute]
+pub fn test(args: TokenStream, item: TokenStream) -> TokenStream {
+	impl_test(args, item)
+}
+
+fn impl_test(args: TokenStream, item: TokenStream) -> TokenStream {
+	let input = syn::parse_macro_input!(item as syn::ItemFn);
+	let args = syn::parse_macro_input!(args as syn::AttributeArgs);
+
+	parse_knobs(input, args).unwrap_or_else(|e| e.to_compile_error().into())
+}
+
+fn parse_knobs(
+	mut input: syn::ItemFn,
+	args: syn::AttributeArgs,
+) -> Result<TokenStream, syn::Error> {
+	let sig = &mut input.sig;
+	let body = &input.block;
+	let attrs = &input.attrs;
+	let vis = input.vis;
+
+	if sig.inputs.len() != 1 {
+		let msg = "the test function accepts only one argument of type sc_service::TaskExecutor";
+		return Err(syn::Error::new_spanned(&sig, msg));
+	}
+	let (task_executor_name, task_executor_type) = match sig.inputs.pop().map(|x| x.into_value()) {
+		Some(syn::FnArg::Typed(x)) => (x.pat, x.ty),
+		_ => {
+			let msg =
+				"the test function accepts only one argument of type sc_service::TaskExecutor";
+			return Err(syn::Error::new_spanned(&sig, msg));
+		}
+	};
+
+	let crate_name = if env::var("CARGO_PKG_NAME").unwrap() == "substrate-test-utils" {
+		syn::Ident::new("substrate_test_utils", Span::call_site().into())
+	} else {
+		let crate_name = crate_name("substrate-test-utils")
+			.map_err(|e| syn::Error::new_spanned(&sig, e))?;
+
+		syn::Ident::new(&crate_name, Span::call_site().into())
+	};
+
+	let header = {
+		quote! {
+			#[#crate_name::tokio::test(#(#args)*)]
+		}
+	};
+
+	let result = quote! {
+		#header
+		#(#attrs)*
+		#vis #sig {
+			use #crate_name::futures::future::FutureExt;
+
+			let #task_executor_name: #task_executor_type = (|fut, _| {
+				#crate_name::tokio::spawn(fut).map(drop)
+			})
+			.into();
+			let timeout_task = #crate_name::tokio::time::delay_for(
+				std::time::Duration::from_secs(
+					std::env::var("SUBSTRATE_TEST_TIMEOUT")
+						.ok()
+						.and_then(|x| x.parse().ok())
+						.unwrap_or(600))
+			).fuse();
+			let actual_test_task = async move {
+				#body
+			}
+			.fuse();
+
+			#crate_name::futures::pin_mut!(timeout_task, actual_test_task);
+
+			#crate_name::futures::select! {
+				_ = timeout_task => {
+					panic!("The test took too long!");
+				},
+				_ = actual_test_task => {},
+			}
+		}
+	};
+
+	Ok(result.into())
+}

test-utils/src/lib.rs

@@ -17,6 +17,29 @@
 
 //! Test utils
 
+#[doc(hidden)]
+pub use futures;
+/// Marks async function to be executed by an async runtime and provide a `TaskExecutor`, suitable
+/// to test environment.
+///
+/// # Requirements
+///
+///	You must have tokio in the `[dev-dependencies]` of your crate to use this macro.
+///
+/// # Example
+///
+/// ```
+/// #[substrate_test_utils::test]
+/// async fn basic_test(task_executor: TaskExecutor) {
+///     assert!(true);
+///     // create your node in here and use task_executor
+///     // then don't forget to gracefully shutdown your node before exit
+/// }
+/// ```
+pub use substrate_test_utils_derive::test;
+#[doc(hidden)]
+pub use tokio;
+
 /// Panic when the vectors are different, without taking the order into account.
 ///
 /// # Examples

test-utils/test-crate/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "substrate-test-utils-test-crate"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "Apache-2.0"
+homepage = "https://substrate.dev"
+repository = "https://github.com/paritytech/substrate/"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dev-dependencies]
+tokio = { version = "0.2.13", features = ["macros"] }
+test-utils = { path = "..", package = "substrate-test-utils" }
+sc-service = { path = "../../client/service" }

test-utils/test-crate/src/main.rs

@@ -0,0 +1,25 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+#[cfg(test)]
+#[test_utils::test]
+async fn basic_test(_: sc_service::TaskExecutor) {
+	assert!(true);
+}
+
+fn main() {}

test-utils/tests/basic.rs

@@ -0,0 +1,58 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use sc_service::{TaskExecutor, TaskType};
+
+#[substrate_test_utils::test]
+async fn basic_test(_: TaskExecutor) {
+	assert!(true);
+}
+
+#[substrate_test_utils::test]
+#[should_panic(expected = "boo!")]
+async fn panicking_test(_: TaskExecutor) {
+	panic!("boo!");
+}
+
+#[substrate_test_utils::test(max_threads = 2)]
+async fn basic_test_with_args(_: TaskExecutor) {
+	assert!(true);
+}
+
+#[substrate_test_utils::test]
+async fn rename_argument(ex: TaskExecutor) {
+	let ex2 = ex.clone();
+	ex2.spawn(Box::pin(async { () }), TaskType::Blocking);
+	assert!(true);
+}
+
+#[substrate_test_utils::test]
+#[should_panic(expected = "test took too long")]
+// NOTE: enable this test only after setting SUBSTRATE_TEST_TIMEOUT to a smaller value
+//
+// SUBSTRATE_TEST_TIMEOUT=1 cargo test -- --ignored timeout
+#[ignore]
+async fn timeout(_: TaskExecutor) {
+	tokio::time::delay_for(std::time::Duration::from_secs(
+		std::env::var("SUBSTRATE_TEST_TIMEOUT")
+			.expect("env var SUBSTRATE_TEST_TIMEOUT has been provided by the user")
+			.parse::<u64>()
+			.unwrap() + 1,
+	))
+	.await;
+}

test-utils/tests/ui.rs

@@ -0,0 +1,24 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+#[test]
+fn substrate_test_utils_derive_trybuild() {
+	let t = trybuild::TestCases::new();
+	t.compile_fail("tests/ui/missing-func-parameter.rs");
+	t.compile_fail("tests/ui/too-many-func-parameters.rs");
+}