fix: test isolation and install script hashing (#671)

<!-- Please make sure there is an issue that this PR is correlated to. -->

## Changes

<!-- If there are frontend changes, please include screenshots. -->

.gitignore

@@ -29,13 +29,11 @@ Bolt.local.toml
 !/secrets/README.md
 
 # Generated code
-gen/hash.txt
 gen/build_script.sh
 gen/svc/
 gen/tf/
 gen/docker/
 gen/k8s/
-svc/pkg/cluster/util/gen/hash.txt
 
 # Rust
 lib/**/Cargo.lock
@@ -52,4 +50,3 @@ lib/**/Cargo.lock
 **/node_modules/
 
 tests/basic-game/.env
-

lib/bolt/core/src/tasks/test.rs

@@ -561,12 +561,13 @@ struct TaggedObjectsListResponse {
 struct TaggedObject {
 	#[serde(rename = "type")]
 	_type: String,
-	data: Data,
+	data: TaggedObjectData,
 }
 
 #[derive(Debug, Deserialize)]
-struct Data {
+struct TaggedObjectData {
 	id: u64,
+	tags: Vec<String>,
 }
 
 #[derive(Debug, Deserialize)]
@@ -588,6 +589,8 @@ async fn cleanup_servers(ctx: &ProjectContext) -> Result<()> {
 	eprintln!();
 	rivet_term::status::progress("Cleaning up servers", "");
 
+	let ns = ctx.ns_id();
+
 	// Create client
 	let api_token = ctx.read_secret(&["linode", "token"]).await?;
 	let auth = format!("Bearer {}", api_token);
@@ -597,10 +600,10 @@ async fn cleanup_servers(ctx: &ProjectContext) -> Result<()> {
 		.default_headers(headers)
 		.build()?;
 
-	// Fetch all objects with the tag "test" and ssh keys with "test-" in them
 	let test_tag = "test";
 	let (objects_res, ssh_keys_res) = tokio::try_join!(
 		async {
+			// Fetch all objects with the tag "test"
 			let res = client
 				.get(format!("https://api.linode.com/v4/tags/{test_tag}"))
 				.send()
@@ -618,11 +621,12 @@ async fn cleanup_servers(ctx: &ProjectContext) -> Result<()> {
 			Ok(res.json::<TaggedObjectsListResponse>().await?)
 		},
 		async {
+			// Fetch all ssh keys with "test-" in their label and are in this namespace
 			let res = client
 				.get(format!("https://api.linode.com/v4/profile/sshkeys"))
 				.header(
 					"X-Filter",
-					format!(r#"{{ "label": {{ "+contains": "{test_tag}-" }} }}"#),
+					format!(r#"{{ "label": {{ "+contains": "{test_tag}-{ns}-" }} }}"#),
 				)
 				.send()
 				.await?;
@@ -643,6 +647,8 @@ async fn cleanup_servers(ctx: &ProjectContext) -> Result<()> {
 	let deletions = objects_res
 		.data
 		.into_iter()
+		// Only delete test objects from this namespace
+		.filter(|object| object.data.tags.iter().any(|tag| tag == ns))
 		.map(|object| {
 			let client = client.clone();
 			let obj_type = object._type;

svc/pkg/cluster/util/build.rs

@@ -1,3 +1,5 @@
+use std::path::PathBuf;
+
 use tokio::{fs, process::Command};
 use sha2::{Digest, Sha256};
 
@@ -6,6 +8,7 @@ use sha2::{Digest, Sha256};
 // Get a hash of the server install worker
 #[tokio::main]
 async fn main() {
+	let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
 	let current_dir = std::env::current_dir().unwrap();
 	let server_install_path = {
 		let mut dir = current_dir.clone();
@@ -18,9 +21,9 @@ async fn main() {
 	};
 
 	// Add rereun statement
-	println!("cargo:rerun-if-changed={}", server_install_path.display(),);
+	println!("cargo:rerun-if-changed={}", server_install_path.display());
 
-	let mut util_path = std::env::current_dir().unwrap();
+	let mut util_path = current_dir.clone();
 	util_path.pop();
 	let util_path = util_path.join("worker").join("src").join("workers").join("server_install");
 
@@ -62,8 +65,7 @@ async fn main() {
 		hex::encode(Sha256::digest(source_diff.as_bytes()))
 	};
 
-	fs::create_dir_all(current_dir.join("gen")).await.unwrap();
-	fs::write(current_dir.join("gen").join("hash.txt"), source_hash.trim())
+	fs::write(out_dir.join("hash.txt"), source_hash.trim())
 		.await
 		.unwrap();
 }

svc/pkg/cluster/util/src/lib.rs

@@ -5,7 +5,7 @@ pub mod test;
 
 // Use the hash of the server install script in the image variant so that if the install scripts are updated
 // we won't be using the old image anymore
-pub const INSTALL_SCRIPT_HASH: &str = include_str!("../gen/hash.txt");
+pub const INSTALL_SCRIPT_HASH: &str = include_str!(concat!(env!("OUT_DIR"), "/hash.txt"));
 
 // NOTE: We don't reserve CPU because Nomad is running as a higher priority process than the rest and
 // shouldn't be doing much heavy lifting.

svc/pkg/linode/ops/server-provision/src/lib.rs

@@ -63,9 +63,10 @@ pub async fn handle(
 	let client = util_linode::Client::new(datacenter.provider_api_token.clone()).await?;
 
 	// Create SSH key
+	let ssh_key_label = format!("{ns}-{server_id}");
 	let ssh_key_res = api::create_ssh_key(
 		&client,
-		&server_id.to_string(),
+		&ssh_key_label,
 		ctx.tags.iter().any(|tag| tag == "test"),
 	)
 	.await?;
@@ -227,7 +228,7 @@ async fn get_custom_image(
 				RETURNING provider, install_hash, datacenter_id, pool_type
 			),
 			selected AS (
-				SELECT provider, install_hash, datacenter_id, pool_type, image_id
+				SELECT provider, install_hash, datacenter_id, pool_type, provider_image_id
 				FROM db_cluster.server_images
 				WHERE
 					provider = $1 AND

svc/pkg/linode/worker/src/workers/prebake_provision.rs

@@ -85,8 +85,12 @@ async fn provision(
 	datacenter_id: Uuid,
 	server: &api::ProvisionCtx,
 ) -> GlobalResult<String> {
+	let server_id = Uuid::new_v4();
+	let ns = util::env::namespace();
+
 	// Create SSH key
-	let ssh_key_res = api::create_ssh_key(client, &Uuid::new_v4().to_string(), false).await?;
+	let ssh_key_label = format!("{ns}-{server_id}");
+	let ssh_key_res = api::create_ssh_key(client, &ssh_key_label, false).await?;
 
 	// Write SSH key id
 	sql_execute!(