Revert "fix: remove trace from ops (#780)" (#805)

<!-- Please make sure there is an issue that this PR is correlated to. -->

## Changes

<!-- If there are frontend changes, please include screenshots. -->

lib/api-helper/build/src/macro_util.rs

@@ -332,6 +332,7 @@ pub async fn __with_ctx<A: auth::ApiAuth + Send>(
 		ts,
 		ts,
 		(),
+		Vec::new(),
 	);
 
 	// Create auth

lib/chirp/client/src/client.rs

@@ -238,10 +238,6 @@ impl Client {
 	pub fn ts(&self) -> i64 {
 		self.ts
 	}
-
-	pub fn trace(&self) -> &[chirp::TraceEntry] {
-		&self.trace
-	}
 }
 
 impl Drop for Client {

lib/chirp/worker/src/manager.rs

@@ -743,6 +743,7 @@ where
 					ts,
 					req_ts,
 					req_body,
+					trace,
 				),
 				dont_log_body,
 				allow_recursive,

lib/chirp/worker/src/test.rs

@@ -37,6 +37,7 @@ impl TestCtx {
 			rivet_util::timestamp::now(),
 			rivet_util::timestamp::now(),
 			(),
+			Vec::new(),
 		);
 
 		Ok(TestCtx {

lib/connection/src/lib.rs

@@ -30,7 +30,7 @@ impl Connection {
 		&self,
 		parent_req_id: Uuid,
 		ray_id: Uuid,
-		trace_entry: chirp_client::TraceEntry,
+		trace: Vec<chirp_client::TraceEntry>,
 	) -> GlobalResult<Connection> {
 		// Not the same as the operation ctx's ts because this cannot be overridden by debug start ts
 		let ts = rivet_util::timestamp::now();
@@ -41,11 +41,7 @@ impl Connection {
 				parent_req_id,
 				ray_id,
 				ts,
-				{
-					let mut x = self.client.trace().to_vec();
-					x.push(trace_entry);
-					x
-				},
+				trace,
 				chirp_perf::PerfCtxInner::new(redis_cache, ts, parent_req_id, ray_id),
 			),
 			self.pools.clone(),

lib/operation/core/src/lib.rs

@@ -40,6 +40,9 @@ where
 	ts: i64,
 	req_ts: i64,
 	body: B,
+	// Trace of all requests not including this request. The client does include
+	// this request in the trace, though.
+	trace: Vec<chirp_client::TraceEntry>,
 }
 
 impl<B> OperationContext<B>
@@ -55,6 +58,7 @@ where
 		ts: i64,
 		req_ts: i64,
 		body: B,
+		trace: Vec<chirp_client::TraceEntry>,
 	) -> Self {
 		OperationContext {
 			name,
@@ -65,6 +69,7 @@ where
 			ts,
 			req_ts,
 			body,
+			trace,
 		}
 	}
 
@@ -131,25 +136,46 @@ where
 	/// Adds trace and correctly wraps `Connection` (and subsequently `chirp_client::Client`).
 	fn wrap<O: Operation>(&self, body: O::Request) -> GlobalResult<OperationContext<O::Request>> {
 		let ray_id = Uuid::new_v4();
-		let trace_entry = chirp_client::TraceEntry {
-			context_name: self.name.clone(),
-			req_id: Some(self.req_id.into()),
-			ts: rivet_util::timestamp::now(),
-			run_context: match rivet_util::env::run_context() {
-				rivet_util::env::RunContext::Service => chirp_client::RunContext::Service,
-				rivet_util::env::RunContext::Test => chirp_client::RunContext::Test,
-			} as i32,
+		// Add self to new operation's trace
+		let trace = {
+			let mut x = self.trace.clone();
+			x.push(chirp_client::TraceEntry {
+				context_name: self.name.clone(),
+				req_id: Some(self.req_id.into()),
+				ts: rivet_util::timestamp::now(),
+				run_context: match rivet_util::env::run_context() {
+					rivet_util::env::RunContext::Service => chirp_client::RunContext::Service,
+					rivet_util::env::RunContext::Test => chirp_client::RunContext::Test,
+				} as i32,
+			});
+			x
 		};
 
 		Ok(OperationContext {
 			name: O::NAME.to_string(),
 			timeout: O::TIMEOUT,
-			conn: self.conn.wrap(self.req_id, ray_id, trace_entry)?,
+			conn: self.conn.wrap(self.req_id, ray_id, {
+				let mut x = trace.clone();
+
+				// Add new operation's trace to its connection (and chirp client)
+				x.push(chirp_client::TraceEntry {
+					context_name: O::NAME.to_string(),
+					req_id: Some(self.req_id.into()),
+					ts: rivet_util::timestamp::now(),
+					run_context: match rivet_util::env::run_context() {
+						rivet_util::env::RunContext::Service => chirp_client::RunContext::Service,
+						rivet_util::env::RunContext::Test => chirp_client::RunContext::Test,
+					} as i32,
+				});
+
+				x
+			})?,
 			req_id: self.req_id,
 			ray_id,
 			ts: util::timestamp::now(),
 			req_ts: self.req_ts,
 			body,
+			trace,
 		})
 	}
 
@@ -165,6 +191,7 @@ where
 			ts: self.ts,
 			req_ts: self.req_ts,
 			body: (),
+			trace: self.trace.clone(),
 		}
 	}
 
@@ -204,11 +231,11 @@ where
 	}
 
 	pub fn trace(&self) -> &[chirp_client::TraceEntry] {
-		self.conn.trace()
+		&self.trace
 	}
 
 	pub fn test(&self) -> bool {
-		self.trace()
+		self.trace
 			.iter()
 			.any(|x| x.run_context == chirp_client::RunContext::Test as i32)
 	}

svc/api/admin/tests/basic.rs

@@ -39,6 +39,7 @@ impl Ctx {
 			util::timestamp::now(),
 			util::timestamp::now(),
 			(),
+			Vec::new(),
 		);
 
 		Ctx { op_ctx }

svc/api/auth/tests/basic.rs

@@ -46,6 +46,7 @@
 // 			util::timestamp::now(),
 // 			util::timestamp::now(),
 // 			(),
+// 			Vec::new(),
 // 		);
 
 // 		let (user_id, user_token) = Self::issue_user_token(&op_ctx).await;

svc/api/cf-verification/tests/basic.rs

@@ -37,6 +37,7 @@
 // 			util::timestamp::now(),
 // 			util::timestamp::now(),
 // 			(),
+// 			Vec::new(),
 // 		);
 
 // 		let http_client = rivet_cf_verification::Config::builder()

svc/api/cloud/tests/basic.rs

@@ -46,6 +46,7 @@
 // 			util::timestamp::now(),
 // 			util::timestamp::now(),
 // 			(),
+// 			Vec::new(),
 // 		);
 
 // 		// Create temp team

svc/api/group/tests/basic.rs

@@ -40,6 +40,7 @@ impl Ctx {
 			util::timestamp::now(),
 			util::timestamp::now(),
 			(),
+			Vec::new(),
 		);
 
 		let (_user_id, user_token) = Self::issue_user_token(&op_ctx).await;

svc/api/identity/tests/basic.rs

@@ -47,6 +47,7 @@
 // 			util::timestamp::now(),
 // 			util::timestamp::now(),
 // 			(),
+// 			Vec::new(),
 // 		);
 
 // 		let (primary_region_id, _) = Self::setup_region(&op_ctx).await;

svc/api/job/tests/basic.rs

@@ -41,6 +41,7 @@
 // 			util::timestamp::now(),
 // 			util::timestamp::now(),
 // 			(),
+// 			Vec::new(),
 // 		);
 
 // 		let nomad_config = nomad_util::config_from_env().unwrap();

svc/api/kv/tests/basic.rs

@@ -52,6 +52,7 @@
 // 			util::timestamp::now(),
 // 			util::timestamp::now(),
 // 			(),
+// 			Vec::new(),
 // 		);
 
 // 		let (primary_region_id, _) = Self::setup_region(&op_ctx).await;

svc/api/matchmaker/tests/common.rs

@@ -56,6 +56,7 @@ impl Ctx {
 			util::timestamp::now(),
 			util::timestamp::now(),
 			(),
+			Vec::new(),
 		);
 
 		let (primary_region_id, primary_region_name_id) = Self::setup_region(&op_ctx).await;

svc/api/portal/tests/basic.rs

@@ -39,6 +39,7 @@ impl Ctx {
 			util::timestamp::now(),
 			util::timestamp::now(),
 			(),
+			Vec::new(),
 		);
 
 		let (_user_id, user_token) = Self::issue_user_token(&op_ctx).await;

svc/api/status/tests/basic.rs

@@ -40,6 +40,7 @@
 // 			util::timestamp::now(),
 // 			util::timestamp::now(),
 // 			(),
+// 			Vec::new(),
 // 		);
 
 // 		let http_client = rivet_status::Config::builder()

svc/api/traefik-provider/tests/basic.rs

@@ -53,6 +53,7 @@ impl Ctx {
 			util::timestamp::now(),
 			util::timestamp::now(),
 			(),
+			Vec::new(),
 		);
 
 		Ctx { op_ctx }

svc/pkg/build/standalone/default-create/src/lib.rs

@@ -82,6 +82,7 @@ pub async fn run_from_env() -> GlobalResult<()> {
 		util::timestamp::now(),
 		util::timestamp::now(),
 		(),
+		Vec::new(),
 	);
 
 	for build in DEFAULT_BUILDS {

svc/pkg/cluster/standalone/datacenter-tls-renew/src/lib.rs

@@ -19,6 +19,7 @@ pub async fn run_from_env(pools: rivet_pools::Pools) -> GlobalResult<()> {
 		util::timestamp::now(),
 		util::timestamp::now(),
 		(),
+		Vec::new(),
 	);
 
 	let updated_datacenter_ids = rivet_pools::utils::crdb::tx(&ctx.crdb().await?, |tx| {

svc/pkg/cluster/standalone/default-update/src/lib.rs

@@ -110,6 +110,7 @@ pub async fn run_from_env(use_autoscaler: bool) -> GlobalResult<()> {
 		util::timestamp::now(),
 		util::timestamp::now(),
 		(),
+		Vec::new(),
 	);
 
 	// Read config from env

svc/pkg/cluster/standalone/gc/src/lib.rs

@@ -23,6 +23,7 @@ pub async fn run_from_env(ts: i64, pools: rivet_pools::Pools) -> GlobalResult<()
 		util::timestamp::now(),
 		util::timestamp::now(),
 		(),
+		Vec::new(),
 	);
 
 	let datacenter_ids = rivet_pools::utils::crdb::tx(&ctx.crdb().await?, |tx| {

svc/pkg/cluster/standalone/metrics-publish/src/lib.rs

@@ -57,6 +57,7 @@ pub async fn run_from_env(_ts: i64, pools: rivet_pools::Pools) -> GlobalResult<(
 		util::timestamp::now(),
 		util::timestamp::now(),
 		(),
+		Vec::new(),
 	);
 
 	let servers = select_servers(&ctx).await?;

svc/pkg/job/standalone/gc/src/lib.rs

@@ -42,6 +42,7 @@ pub async fn run_from_env(ts: i64, pools: rivet_pools::Pools) -> GlobalResult<()
 		ts,
 		ts,
 		(),
+		Vec::new(),
 	);
 
 	// Find jobs to stop.

svc/pkg/linode/standalone/gc/src/lib.rs

@@ -29,6 +29,7 @@ pub async fn run_from_env(pools: rivet_pools::Pools) -> GlobalResult<()> {
 		util::timestamp::now(),
 		util::timestamp::now(),
 		(),
+		Vec::new(),
 	);
 
 	let dc_rows = sql_fetch_all!(

svc/pkg/load-test/standalone/api-cloud/src/lib.rs

@@ -18,6 +18,7 @@ pub async fn run_from_env(_ts: i64) -> GlobalResult<()> {
 		util::timestamp::now(),
 		util::timestamp::now(),
 		(),
+		Vec::new(),
 	);
 
 	// Create temp team

svc/pkg/load-test/standalone/mm-sustain/src/lib.rs

@@ -22,6 +22,7 @@ pub async fn run_from_env(_ts: i64) -> GlobalResult<()> {
 		util::timestamp::now(),
 		util::timestamp::now(),
 		(),
+		Vec::new(),
 	);
 
 	// Region

svc/pkg/load-test/standalone/mm/src/lib.rs

@@ -72,6 +72,7 @@ pub async fn run_from_env(_ts: i64) -> GlobalResult<()> {
 		util::timestamp::now(),
 		util::timestamp::now(),
 		(),
+		Vec::new(),
 	);
 
 	// Setup

svc/pkg/load-test/standalone/sqlx/src/lib.rs

@@ -14,6 +14,7 @@ pub async fn run_from_env(_ts: i64) -> GlobalResult<()> {
 		util::timestamp::now(),
 		util::timestamp::now(),
 		(),
+		Vec::new(),
 	);
 
 	let mut interval = tokio::time::interval(std::time::Duration::from_millis(100));

svc/pkg/load-test/standalone/watch-requests/src/lib.rs

@@ -18,6 +18,7 @@ pub async fn run_from_env(_ts: i64) -> GlobalResult<()> {
 		util::timestamp::now(),
 		util::timestamp::now(),
 		(),
+		Vec::new(),
 	);
 
 	// Create temp team

svc/pkg/mm/standalone/gc/src/lib.rs

@@ -15,6 +15,7 @@ pub async fn run_from_env(ts: i64, pools: rivet_pools::Pools) -> GlobalResult<()
 		ts,
 		ts,
 		(),
+		Vec::new(),
 	);
 	let redis_mm = ctx.redis_mm().await?;
 

svc/pkg/telemetry/standalone/beacon/src/lib.rs

@@ -28,6 +28,7 @@ pub async fn run_from_env(ts: i64) -> GlobalResult<()> {
 		util::timestamp::now(),
 		util::timestamp::now(),
 		(),
+		Vec::new(),
 	);
 
 	if std::env::var("RIVET_TELEMETRY_DISABLE")

svc/pkg/upload/standalone/provider-fill/src/lib.rs

@@ -17,6 +17,7 @@ pub async fn run_from_env() -> GlobalResult<()> {
 		util::timestamp::now(),
 		util::timestamp::now(),
 		(),
+		Vec::new(),
 	);
 
 	let Ok(backfill_provider) = std::env::var("S3_BACKFILL_PROVIDER") else {

svc/pkg/user/standalone/delete-pending/src/lib.rs

@@ -18,6 +18,7 @@ pub async fn run_from_env(ts: i64) -> GlobalResult<()> {
 		util::timestamp::now(),
 		util::timestamp::now(),
 		(),
+		Vec::new(),
 	);
 
 	let user_ids = sql_fetch_all!(

svc/pkg/user/standalone/search-user-gc/src/lib.rs

@@ -18,6 +18,7 @@ pub async fn run_from_env(ts: i64) -> GlobalResult<()> {
 		util::timestamp::now(),
 		util::timestamp::now(),
 		(),
+		Vec::new(),
 	);
 
 	let mut total_removed = 0;