feat: add metrics for trusted_operation execution

tee-worker/core-primitives/enclave-metrics/Cargo.toml

@@ -12,6 +12,7 @@ sgx_tstd = { branch = "master", git = "https://github.com/apache/teaclave-sgx-sd
 
 # no-std dependencies
 codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "full"] }
+ita-stf = { path = "../../app-libs/stf", default-features = false }
 lc-stf-task-sender = { path = "../../litentry/core/stf-task/sender", default-features = false }
 substrate-fixed = { default-features = false, git = "https://github.com/encointer/substrate-fixed", tag = "v0.5.9" }
 
@@ -20,7 +21,9 @@ default = ["std"]
 std = [
     "substrate-fixed/std",
     "codec/std",
+    "ita-stf/std",
 ]
 sgx = [
     "sgx_tstd",
+    "ita-stf/sgx",
 ]

tee-worker/core-primitives/enclave-metrics/src/lib.rs

@@ -24,6 +24,7 @@ compile_error!("feature \"std\" and feature \"sgx\" cannot be enabled at the sam
 extern crate sgx_tstd as std;
 
 use codec::{Decode, Encode};
+use ita_stf::TrustedCall;
 use lc_stf_task_sender::RequestType;
 use std::{boxed::Box, string::String};
 use substrate_fixed::types::U32F32;
@@ -39,6 +40,8 @@ pub enum EnclaveMetric {
 	TopPoolSizeDecrement,
 	ExchangeRateOracle(ExchangeRateOracleMetric),
 	StfTaskExecutionTime(Box<RequestType>, f64),
+	SuccessfulTrustedOperationIncrement(TrustedCall),
+	FailedTrustedOperationIncrement(TrustedCall),
 	// OracleMetric(OracleMetric<MetricsInfo>),
 }
 

tee-worker/core-primitives/stf-executor/Cargo.toml

@@ -12,6 +12,7 @@ sgx_types = { branch = "master", git = "https://github.com/apache/teaclave-sgx-s
 
 # local dependencies
 ita-stf = { path = "../../app-libs/stf", default-features = false }
+itp-enclave-metrics = { path = "../enclave-metrics", default-features = false }
 itp-node-api = { path = "../node-api", default-features = false }
 itp-ocall-api = { path = "../ocall-api", default-features = false }
 itp-sgx-crypto = { path = "../sgx/crypto", default-features = false }

tee-worker/core-primitives/stf-executor/src/executor.rs

@@ -26,8 +26,9 @@ use ita_stf::{
 	stf_sgx::{shards_key_hash, storage_hashes_to_update_per_shard},
 	ParentchainHeader, TrustedCallSigned, TrustedOperation,
 };
+use itp_enclave_metrics::EnclaveMetric;
 use itp_node_api::metadata::{provider::AccessNodeMetadata, NodeMetadataTrait};
-use itp_ocall_api::{EnclaveAttestationOCallApi, EnclaveOnChainOCallApi};
+use itp_ocall_api::{EnclaveAttestationOCallApi, EnclaveMetricsOCallApi, EnclaveOnChainOCallApi};
 use itp_sgx_externalities::{SgxExternalitiesTrait, StateHash};
 use itp_stf_interface::{
 	parentchain_pallet::ParentchainPalletInterface, runtime_upgrade::RuntimeUpgradeInterface,
@@ -54,7 +55,7 @@ pub struct StfExecutor<OCallApi, StateHandler, NodeMetadataRepository, Stf> {
 impl<OCallApi, StateHandler, NodeMetadataRepository, Stf>
 	StfExecutor<OCallApi, StateHandler, NodeMetadataRepository, Stf>
 where
-	OCallApi: EnclaveAttestationOCallApi + EnclaveOnChainOCallApi,
+	OCallApi: EnclaveAttestationOCallApi + EnclaveOnChainOCallApi + EnclaveMetricsOCallApi,
 	StateHandler: HandleState<HashType = H256>,
 	StateHandler::StateT: SgxExternalitiesTrait + Encode,
 	NodeMetadataRepository: AccessNodeMetadata,
@@ -138,11 +139,17 @@ where
 			self.node_metadata_repo.clone(),
 		) {
 			Err(e) => {
+				if let Err(e) = self.ocall_api.update_metric(EnclaveMetric::FailedTrustedOperationIncrement(trusted_call.call.clone())) {
+					warn!("Failed to update metric for failed trusted operations: {:?}", e);
+				}
 				error!("Stf execute failed: {:?}", e);
 				let rpc_response_value: Vec<u8> = e.encode();
 				Ok(ExecutedOperation::failed(operation_hash, top_or_hash, extrinsic_call_backs, rpc_response_value))
 			},
 			Ok(result) => {
+				if let Err(e) = self.ocall_api.update_metric(EnclaveMetric::SuccessfulTrustedOperationIncrement(trusted_call.call.clone())) {
+					warn!("Failed to update metric for succesfull trusted operations: {:?}", e);
+				}
 				let force_connection_wait = result.force_connection_wait();
 				let rpc_response_value: Vec<u8> = result.get_encoded_result();
 				if let StatePostProcessing::Prune = post_processing {
@@ -236,7 +243,7 @@ where
 impl<OCallApi, StateHandler, NodeMetadataRepository, Stf> StateUpdateProposer
 	for StfExecutor<OCallApi, StateHandler, NodeMetadataRepository, Stf>
 where
-	OCallApi: EnclaveAttestationOCallApi + EnclaveOnChainOCallApi,
+	OCallApi: EnclaveAttestationOCallApi + EnclaveOnChainOCallApi + EnclaveMetricsOCallApi,
 	StateHandler: HandleState<HashType = H256>,
 	StateHandler::StateT: SgxExternalitiesTrait + Encode + StateHash,
 	<StateHandler::StateT as SgxExternalitiesTrait>::SgxExternalitiesType: Encode,

tee-worker/service/src/prometheus_metrics.rs

@@ -29,6 +29,7 @@ use codec::{Decode, Encode};
 #[cfg(feature = "dcap")]
 use core::time::Duration;
 use frame_support::scale_info::TypeInfo;
+use ita_stf::TrustedCall;
 #[cfg(feature = "dcap")]
 use itc_rest_client::{
 	http_client::{DefaultSend, HttpClient},
@@ -41,8 +42,8 @@ use lc_stf_task_sender::RequestType;
 use litentry_primitives::{Assertion, Identity};
 use log::*;
 use prometheus::{
-	proto::MetricFamily, register_histogram_vec, register_int_gauge, register_int_gauge_vec,
-	HistogramVec, IntGauge, IntGaugeVec,
+	proto::MetricFamily, register_counter_vec, register_histogram_vec, register_int_gauge,
+	register_int_gauge_vec, CounterVec, HistogramVec, IntGauge, IntGaugeVec,
 };
 use serde::{Deserialize, Serialize};
 use std::{net::SocketAddr, sync::Arc};
@@ -66,6 +67,12 @@ lazy_static! {
 	static ref ENCLAVE_STF_TASKS_EXECUTION: HistogramVec =
 		register_histogram_vec!("litentry_worker_enclave_stf_tasks_execution_times", "Litentry Stf Tasks Exeuction Time", &["request_type", "variant"])
 			.unwrap();
+	static ref ENCLAVE_SUCCESSFUL_TRUSTED_OPERATION: CounterVec =
+		register_counter_vec!("litentry_worker_enclave_successful_trusted_operation", "Litentry Successful Trusted Operation", &["call"])
+			.unwrap();
+	static ref ENCLAVE_FAILED_TRUSTED_OPERATION: CounterVec =
+		register_counter_vec!("litentry_worker_enclave_failed_trusted_operation", "Litentry Failed Trusted Operation", &["call"])
+			.unwrap();
 }
 
 pub async fn start_metrics_server<MetricsHandler>(
@@ -185,6 +192,12 @@ impl ReceiveEnclaveMetrics for EnclaveMetricsReceiver {
 			EnclaveMetric::StfTaskExecutionTime(req, time) => {
 				handle_stf_call_request(*req, time);
 			},
+			EnclaveMetric::SuccessfulTrustedOperationIncrement(calls) => {
+				handle_trusted_operation(calls, inc_successful_trusted_operation_counter);
+			},
+			EnclaveMetric::FailedTrustedOperationIncrement(calls) => {
+				handle_trusted_operation(calls, inc_failed_trusted_operation_counter);
+			},
 			#[cfg(feature = "teeracle")]
 			EnclaveMetric::ExchangeRateOracle(m) => update_teeracle_metrics(m)?,
 			#[cfg(not(feature = "teeracle"))]
@@ -244,6 +257,56 @@ fn handle_stf_call_request(req: RequestType, time: f64) {
 	observe_execution_time(category, label, time)
 }
 
+// This function will increment the metric with provided label values.
+fn inc_successful_trusted_operation_counter(operation: &str) {
+	ENCLAVE_SUCCESSFUL_TRUSTED_OPERATION.with_label_values(&[operation]).inc();
+}
+
+fn inc_failed_trusted_operation_counter(operation: &str) {
+	ENCLAVE_FAILED_TRUSTED_OPERATION.with_label_values(&[operation]).inc();
+}
+
+fn handle_trusted_operation<F>(call: TrustedCall, record_metric_fn: F)
+where
+	F: Fn(&str),
+{
+	match call {
+		TrustedCall::link_identity(..) => {
+			record_metric_fn("link_identity");
+		},
+		TrustedCall::request_vc(..) => {
+			record_metric_fn("request_vc");
+		},
+		TrustedCall::set_user_shielding_key(..) => {
+			record_metric_fn("set_user_shielding_key");
+		},
+		TrustedCall::set_user_shielding_key_with_networks(..) => {
+			record_metric_fn("set_user_shielding_key_with_networks");
+		},
+		TrustedCall::link_identity_callback(..) => {
+			record_metric_fn("link_identity_callback");
+		},
+		TrustedCall::request_vc_callback(..) => {
+			record_metric_fn("request_vc_callback");
+		},
+		TrustedCall::handle_vcmp_error(..) => {
+			record_metric_fn("handle_vcmp_error");
+		},
+		TrustedCall::handle_imp_error(..) => {
+			record_metric_fn("handle_icmp_error");
+		},
+		TrustedCall::deactivate_identity(..) => {
+			record_metric_fn("deactivate_identity");
+		},
+		TrustedCall::activate_identity(..) => {
+			record_metric_fn("activate_identity");
+		},
+		_ => {
+			record_metric_fn("unsupported_trusted_operation");
+		},
+	}
+}
+
 #[derive(Serialize, Deserialize, Debug)]
 struct PrometheusMarblerunEvents(pub Vec<PrometheusMarblerunEvent>);