Skip to content

Commit bd0ab9a

Browse files
isumisum
authored
Add deployment_status metric (#5720)
* graph, core: add deployment_status metric * update news with the implemented changes
1 parent 2be1cc3 commit bd0ab9a

File tree

7 files changed

+262
-97
lines changed

7 files changed

+262
-97
lines changed

‎NEWS.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,24 @@
11
# NEWS
22

3+
## v0.36.1
4+
5+
### What's new
6+
7+
- A new `deployment_status` metric is added [(#5720)](https://github.com/graphprotocol/graph-node/pull/5720) with the
8+
following behavior:
9+
- Once graph-node has figured out that it should index a deployment, `deployment_status` is set to `1` _(starting)_;
10+
- When the block stream is created and blocks are ready to be processed, `deployment_status` is set to `2` _(
11+
running)_;
12+
- When a deployment is unassigned, `deployment_status` is set to `3` _(stopped)_;
13+
- If a temporary or permanent failure occurs, `deployment_status` is set to `4` _(failed)_;
14+
- If indexing manages to recover from a temporary failure, the `deployment_status` is set back to `2` _(
15+
running)_;
16+
17+
### Breaking changes
18+
19+
- The `deployment_failed` metric is removed and the failures are reported by the new `deployment_status`
20+
metric. [(#5720)](https://github.com/graphprotocol/graph-node/pull/5720)
21+
322
## v0.36.0
423

524
### Note on Firehose Extended Block Details

‎core/src/subgraph/instance_manager.rs‎

Lines changed: 111 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ use crate::subgraph::runner::SubgraphRunner;
1212
use graph::blockchain::block_stream::BlockStreamMetrics;
1313
use graph::blockchain::{Blockchain, BlockchainKind, DataSource, NodeCapabilities};
1414
use graph::components::metrics::gas::GasMetrics;
15+
use graph::components::metrics::subgraph::DeploymentStatusMetric;
1516
use graph::components::subgraph::ProofOfIndexingVersion;
1617
use graph::data::subgraph::{UnresolvedSubgraphManifest, SPEC_VERSION_0_0_6};
1718
use graph::data::value::Word;
@@ -69,77 +70,91 @@ impl<S: SubgraphStore> SubgraphInstanceManagerTrait for SubgraphInstanceManager<
6970
let err_logger = logger.clone();
7071
let instance_manager = self.cheap_clone();
7172

72-
let subgraph_start_future = async move {
73-
match BlockchainKind::from_manifest(&manifest)? {
74-
BlockchainKind::Arweave => {
75-
let runner = instance_manager
76-
.build_subgraph_runner::<graph_chain_arweave::Chain>(
77-
logger.clone(),
78-
self.env_vars.cheap_clone(),
79-
loc.clone(),
80-
manifest,
81-
stop_block,
82-
Box::new(SubgraphTriggerProcessor {}),
83-
)
84-
.await?;
85-
86-
self.start_subgraph_inner(logger, loc, runner).await
87-
}
88-
BlockchainKind::Ethereum => {
89-
let runner = instance_manager
90-
.build_subgraph_runner::<graph_chain_ethereum::Chain>(
91-
logger.clone(),
92-
self.env_vars.cheap_clone(),
93-
loc.clone(),
94-
manifest,
95-
stop_block,
96-
Box::new(SubgraphTriggerProcessor {}),
97-
)
98-
.await?;
99-
100-
self.start_subgraph_inner(logger, loc, runner).await
101-
}
102-
BlockchainKind::Near => {
103-
let runner = instance_manager
104-
.build_subgraph_runner::<graph_chain_near::Chain>(
105-
logger.clone(),
106-
self.env_vars.cheap_clone(),
107-
loc.clone(),
108-
manifest,
109-
stop_block,
110-
Box::new(SubgraphTriggerProcessor {}),
111-
)
112-
.await?;
113-
114-
self.start_subgraph_inner(logger, loc, runner).await
115-
}
116-
BlockchainKind::Cosmos => {
117-
let runner = instance_manager
118-
.build_subgraph_runner::<graph_chain_cosmos::Chain>(
119-
logger.clone(),
120-
self.env_vars.cheap_clone(),
121-
loc.clone(),
122-
manifest,
123-
stop_block,
124-
Box::new(SubgraphTriggerProcessor {}),
125-
)
126-
.await?;
127-
128-
self.start_subgraph_inner(logger, loc, runner).await
129-
}
130-
BlockchainKind::Substreams => {
131-
let runner = instance_manager
132-
.build_subgraph_runner::<graph_chain_substreams::Chain>(
133-
logger.clone(),
134-
self.env_vars.cheap_clone(),
135-
loc.cheap_clone(),
136-
manifest,
137-
stop_block,
138-
Box::new(graph_chain_substreams::TriggerProcessor::new(loc.clone())),
139-
)
140-
.await?;
141-
142-
self.start_subgraph_inner(logger, loc, runner).await
73+
let deployment_status_metric = self.new_deployment_status_metric(&loc);
74+
deployment_status_metric.starting();
75+
76+
let subgraph_start_future = {
77+
let deployment_status_metric = deployment_status_metric.clone();
78+
79+
async move {
80+
match BlockchainKind::from_manifest(&manifest)? {
81+
BlockchainKind::Arweave => {
82+
let runner = instance_manager
83+
.build_subgraph_runner::<graph_chain_arweave::Chain>(
84+
logger.clone(),
85+
self.env_vars.cheap_clone(),
86+
loc.clone(),
87+
manifest,
88+
stop_block,
89+
Box::new(SubgraphTriggerProcessor {}),
90+
deployment_status_metric,
91+
)
92+
.await?;
93+
94+
self.start_subgraph_inner(logger, loc, runner).await
95+
}
96+
BlockchainKind::Ethereum => {
97+
let runner = instance_manager
98+
.build_subgraph_runner::<graph_chain_ethereum::Chain>(
99+
logger.clone(),
100+
self.env_vars.cheap_clone(),
101+
loc.clone(),
102+
manifest,
103+
stop_block,
104+
Box::new(SubgraphTriggerProcessor {}),
105+
deployment_status_metric,
106+
)
107+
.await?;
108+
109+
self.start_subgraph_inner(logger, loc, runner).await
110+
}
111+
BlockchainKind::Near => {
112+
let runner = instance_manager
113+
.build_subgraph_runner::<graph_chain_near::Chain>(
114+
logger.clone(),
115+
self.env_vars.cheap_clone(),
116+
loc.clone(),
117+
manifest,
118+
stop_block,
119+
Box::new(SubgraphTriggerProcessor {}),
120+
deployment_status_metric,
121+
)
122+
.await?;
123+
124+
self.start_subgraph_inner(logger, loc, runner).await
125+
}
126+
BlockchainKind::Cosmos => {
127+
let runner = instance_manager
128+
.build_subgraph_runner::<graph_chain_cosmos::Chain>(
129+
logger.clone(),
130+
self.env_vars.cheap_clone(),
131+
loc.clone(),
132+
manifest,
133+
stop_block,
134+
Box::new(SubgraphTriggerProcessor {}),
135+
deployment_status_metric,
136+
)
137+
.await?;
138+
139+
self.start_subgraph_inner(logger, loc, runner).await
140+
}
141+
BlockchainKind::Substreams => {
142+
let runner = instance_manager
143+
.build_subgraph_runner::<graph_chain_substreams::Chain>(
144+
logger.clone(),
145+
self.env_vars.cheap_clone(),
146+
loc.cheap_clone(),
147+
manifest,
148+
stop_block,
149+
Box::new(graph_chain_substreams::TriggerProcessor::new(
150+
loc.clone(),
151+
)),
152+
deployment_status_metric,
153+
)
154+
.await?;
155+
156+
self.start_subgraph_inner(logger, loc, runner).await
157+
}
143158
}
144159
}
145160
};
@@ -152,12 +167,16 @@ impl<S: SubgraphStore> SubgraphInstanceManagerTrait for SubgraphInstanceManager<
152167
graph::spawn(async move {
153168
match subgraph_start_future.await {
154169
Ok(()) => {}
155-
Err(err) => error!(
156-
err_logger,
157-
"Failed to start subgraph";
158-
"error" => format!("{:#}", err),
159-
"code" => LogCode::SubgraphStartFailure
160-
),
170+
Err(err) => {
171+
deployment_status_metric.failed();
172+
173+
error!(
174+
err_logger,
175+
"Failed to start subgraph";
176+
"error" => format!("{:#}", err),
177+
"code" => LogCode::SubgraphStartFailure
178+
);
179+
}
161180
}
162181
});
163182
}
@@ -217,6 +236,7 @@ impl<S: SubgraphStore> SubgraphInstanceManager<S> {
217236
manifest: serde_yaml::Mapping,
218237
stop_block: Option<BlockNumber>,
219238
tp: Box<dyn TriggerProcessor<C, RuntimeHostBuilder<C>>>,
239+
deployment_status_metric: DeploymentStatusMetric,
220240
) -> anyhow::Result<SubgraphRunner<C, RuntimeHostBuilder<C>>>
221241
where
222242
C: Blockchain,
@@ -387,6 +407,7 @@ impl<S: SubgraphStore> SubgraphInstanceManager<S> {
387407
registry.cheap_clone(),
388408
deployment.hash.as_str(),
389409
stopwatch_metrics.clone(),
410+
deployment_status_metric,
390411
));
391412

392413
let block_stream_metrics = Arc::new(BlockStreamMetrics::new(
@@ -496,7 +517,7 @@ impl<S: SubgraphStore> SubgraphInstanceManager<S> {
496517
<C as Blockchain>::MappingTrigger: ToAscPtr,
497518
{
498519
let registry = self.metrics_registry.cheap_clone();
499-
let subgraph_metrics_unregister = runner.metrics.subgraph.cheap_clone();
520+
let subgraph_metrics = runner.metrics.subgraph.cheap_clone();
500521

501522
// Keep restarting the subgraph until it terminates. The subgraph
502523
// will usually only run once, but is restarted whenever a block
@@ -513,20 +534,30 @@ impl<S: SubgraphStore> SubgraphInstanceManager<S> {
513534
// https://github.com/tokio-rs/tokio/issues/3493.
514535
graph::spawn_thread(deployment.to_string(), move || {
515536
match graph::block_on(task::unconstrained(runner.run())) {
516-
Ok(()) => {}
537+
Ok(()) => {
538+
subgraph_metrics.deployment_status.stopped();
539+
}
517540
Err(SubgraphRunnerError::Duplicate) => {
518541
// We do not need to unregister metrics because they are unique per subgraph
519542
// and another runner is still active.
520543
return;
521544
}
522545
Err(err) => {
523546
error!(&logger, "Subgraph instance failed to run: {:#}", err);
547+
subgraph_metrics.deployment_status.failed();
524548
}
525549
}
526550

527-
subgraph_metrics_unregister.unregister(registry);
551+
subgraph_metrics.unregister(registry);
528552
});
529553

530554
Ok(())
531555
}
556+
557+
pub fn new_deployment_status_metric(
558+
&self,
559+
deployment: &DeploymentLocator,
560+
) -> DeploymentStatusMetric {
561+
DeploymentStatusMetric::register(&self.metrics_registry, deployment)
562+
}
532563
}

‎core/src/subgraph/runner.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,8 @@ where
244244

245245
debug!(self.logger, "Starting block stream");
246246

247+
self.metrics.subgraph.deployment_status.running();
248+
247249
// Process events from the stream as long as no restart is needed
248250
loop {
249251
let event = {
@@ -876,7 +878,7 @@ where
876878
self.state.should_try_unfail_non_deterministic = false;
877879

878880
if let UnfailOutcome::Unfailed = outcome {
879-
self.metrics.stream.deployment_failed.set(0.0);
881+
self.metrics.subgraph.deployment_status.running();
880882
self.state.backoff.reset();
881883
}
882884
}
@@ -909,7 +911,7 @@ where
909911

910912
// Handle unexpected stream errors by marking the subgraph as failed.
911913
Err(e) => {
912-
self.metrics.stream.deployment_failed.set(1.0);
914+
self.metrics.subgraph.deployment_status.failed();
913915
let last_good_block = self
914916
.inputs
915917
.store

‎graph/src/blockchain/block_stream.rs‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -573,7 +573,6 @@ where
573573
#[derive(Clone)]
574574
pub struct BlockStreamMetrics {
575575
pub deployment_head: Box<Gauge>,
576-
pub deployment_failed: Box<Gauge>,
577576
pub reverted_blocks: Gauge,
578577
pub stopwatch: StopwatchMetrics,
579578
}
@@ -605,16 +604,8 @@ impl BlockStreamMetrics {
605604
labels.clone(),
606605
)
607606
.expect("failed to create `deployment_head` gauge");
608-
let deployment_failed = registry
609-
.new_gauge(
610-
"deployment_failed",
611-
"Boolean gauge to indicate whether the deployment has failed (1 == failed)",
612-
labels,
613-
)
614-
.expect("failed to create `deployment_failed` gauge");
615607
Self {
616608
deployment_head,
617-
deployment_failed,
618609
reverted_blocks,
619610
stopwatch,
620611
}

0 commit comments

Comments
 (0)