Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix empty shotover_chain_latency metric #980

Merged
merged 5 commits into from Jan 9, 2023
Merged

Fix empty shotover_chain_latency metric #980

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4b3fe52
Fix empty shotover_chain_latency metric
rukai Dec 22, 2022
7042eec
Merge branch 'main' into fix_empty_shotover_chain_latency
rukai Dec 22, 2022
cecec8c
Merge branch 'main' into fix_empty_shotover_chain_latency
conorbros Jan 9, 2023
fca85a5
Merge branch 'main' into fix_empty_shotover_chain_latency
conorbros Jan 9, 2023
e7b07ab
Merge branch 'main' into fix_empty_shotover_chain_latency
conorbros Jan 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 3 additions & 1 deletion shotover-proxy/src/server.rs
View file
Open in desktop
Expand Up @@ -569,7 +569,9 @@ impl<C: Codec + 'static> Handler<C> {
);

let modified_messages = if reverse_chain {
self.chain.process_request_rev(wrapper).await
self.chain
.process_request_rev(wrapper, self.client_details.clone())
.await
} else {
self.chain
.process_request(wrapper, self.client_details.clone())
Expand Down
10 changes: 7 additions & 3 deletions shotover-proxy/src/transforms/chain.rs
View file
Open in desktop
Expand Up @@ -189,7 +189,11 @@ impl TransformChain {
result
}

pub async fn process_request_rev(&mut self, mut wrapper: Wrapper<'_>) -> ChainResponse {
pub async fn process_request_rev(
&mut self,
mut wrapper: Wrapper<'_>,
client_details: String,
) -> ChainResponse {
let start = Instant::now();
wrapper.reset_rev(&mut self.chain);

Expand All @@ -199,7 +203,7 @@ impl TransformChain {
self.chain_failures.increment(1);
}

histogram!("shotover_chain_latency", start.elapsed(), "chain" => self.name.clone());
histogram!("shotover_chain_latency", start.elapsed(), "chain" => self.name.clone(), "client_details" => client_details);
result
}
}
Expand All @@ -226,7 +230,7 @@ impl TransformChainBuilder {

let chain_total = register_counter!("shotover_chain_total", "chain" => name.clone());
let chain_failures = register_counter!("shotover_chain_failures", "chain" => name.clone());
register_histogram!("shotover_chain_latency", "chain" => name.clone());
// Cant register shotover_chain_latency because a unique one is created for each client ip address

TransformChainBuilder {
name,
Expand Down
17 changes: 4 additions & 13 deletions shotover-proxy/tests/runner/observability_int_tests.rs
View file
Open in desktop
@@ -1,7 +1,6 @@
use crate::helpers::ShotoverManager;
use itertools::Itertools;
use serial_test::serial;
use test_helpers::connection::redis_connection;
use test_helpers::{connection::redis_connection, shotover_process::shotover_from_topology_file};

async fn http_request_metrics() -> String {
let url = "http://localhost:9001/metrics";
Expand Down Expand Up @@ -65,8 +64,7 @@ async fn assert_metrics_key_value(key: &str, value: &str) {
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn test_metrics() {
let _shotover_manager =
ShotoverManager::from_topology_file("example-configs/null-redis/topology.yaml");
let shotover = shotover_from_topology_file("example-configs/null-redis/topology.yaml").await;
let mut connection = redis_connection::new_async(6379).await;

// Expected string looks unnatural because it is sorted in alphabetical order to make it match the sorted error output
Expand All @@ -83,23 +81,14 @@ query_count{name="redis-chain"}
shotover_available_connections{source="RedisSource"}
shotover_chain_failures{chain="redis_chain"}
shotover_chain_latency_count{chain="redis_chain",client_details="127.0.0.1"}
shotover_chain_latency_count{chain="redis_chain"}
shotover_chain_latency_sum{chain="redis_chain",client_details="127.0.0.1"}
shotover_chain_latency_sum{chain="redis_chain"}
shotover_chain_latency{chain="redis_chain",client_details="127.0.0.1",quantile="0"}
shotover_chain_latency{chain="redis_chain",client_details="127.0.0.1",quantile="0.5"}
shotover_chain_latency{chain="redis_chain",client_details="127.0.0.1",quantile="0.9"}
shotover_chain_latency{chain="redis_chain",client_details="127.0.0.1",quantile="0.95"}
shotover_chain_latency{chain="redis_chain",client_details="127.0.0.1",quantile="0.99"}
shotover_chain_latency{chain="redis_chain",client_details="127.0.0.1",quantile="0.999"}
shotover_chain_latency{chain="redis_chain",client_details="127.0.0.1",quantile="1"}
shotover_chain_latency{chain="redis_chain",quantile="0"}
shotover_chain_latency{chain="redis_chain",quantile="0.5"}
shotover_chain_latency{chain="redis_chain",quantile="0.9"}
shotover_chain_latency{chain="redis_chain",quantile="0.95"}
shotover_chain_latency{chain="redis_chain",quantile="0.99"}
shotover_chain_latency{chain="redis_chain",quantile="0.999"}
shotover_chain_latency{chain="redis_chain",quantile="1"}
shotover_chain_total{chain="redis_chain"}
shotover_transform_failures{transform="Null"}
shotover_transform_failures{transform="QueryCounter"}
Expand Down Expand Up @@ -163,4 +152,6 @@ query_count{name="redis-chain",query="SET",type="redis"}
"2",
)
.await;

shotover.shutdown_and_then_consume_events(&[]).await;
}