fix: expand prebake image variant system

infra/tf/dns/cert_packs.tf

@@ -8,6 +8,10 @@ locals {
  	# 
  	# If the CDN domain is already at the root of the zone, then Cloudflare exposes a cert back by default and we don't need to create a new one.
  	needs_cdn_cert_pack = data.cloudflare_zone.cdn.name != var.domain_cdn
+
+	# Should be `lets_encrypt` to be consistent with job node certs, change to `google` if experiencing rate
+	# limits
+	certificate_authority = "lets_encrypt"
 }
 
 # TODO: Only if we use deprecated subdomains
@@ -22,7 +26,7 @@ resource "cloudflare_certificate_pack" "main" {
 		create_before_destroy = true
 	}
 
-	certificate_authority = "lets_encrypt"
+	certificate_authority = local.certificate_authority
 	# The certificate must include the root domain in it.
 	#
 	# We convert to set then back to list to remove potential duplicates of the root zoon.
@@ -52,7 +56,7 @@ resource "cloudflare_certificate_pack" "cdn" {
 		create_before_destroy = true
 	}
 
-	certificate_authority = "lets_encrypt"
+	certificate_authority = local.certificate_authority
 	# The certificate must include the root domain in it.
 	#
 	# We convert to set then back to list to remove potential duplicates of the root zoon.

svc/pkg/cluster/db/cluster/migrations/20231201000927_init.up.sql

@@ -65,21 +65,31 @@ CREATE TABLE cloudflare_misc (
 );
 
 CREATE TABLE server_images (
-	-- A string denoting what type of image this is (ex. "linode-us-southeast-job")
-	variant TEXT PRIMARY KEY,
+	provider INT,
+	install_hash TEXT,
+	datacenter_id UUID,
+	pool_type INT,
+
 	create_ts INT NOT NULL,
-	image_id TEXT
+	image_id TEXT,
+
+	PRIMARY KEY (provider, install_hash, datacenter_id, pool_type)
 );
 
+-- Stores data for destroying linode prebake resources and creating custom images
 CREATE TABLE server_images_linode_misc (
-	variant TEXT PRIMARY KEY,
+	install_hash TEXT,
+	datacenter_id UUID,
+	pool_type INT,
+
 	ssh_key_id INT NOT NULL,
 	linode_id INT,
 	firewall_id INT,
 	disk_id INT,
 	public_ip TEXT,
 	image_id TEXT,
 
+	PRIMARY KEY (install_hash, datacenter_id, pool_type),
 	INDEX (public_ip),
 	INDEX (image_id)
 );

svc/pkg/cluster/standalone/gc/src/lib.rs

@@ -88,8 +88,7 @@ pub async fn run_from_env(ts: i64, pools: rivet_pools::Pools) -> GlobalResult<()
 		"
 		UPDATE db_cluster.servers
 		SET cloud_destroy_ts = $2
-		WHERE
-			server_id = ANY($1)
+		WHERE server_id = ANY($1)
 		",
 		&destroy_server_ids,
 		util::timestamp::now(),

svc/pkg/cluster/types/msg/server-install-complete.proto

@@ -11,8 +11,8 @@ import "proto/backend/cluster.proto";
 /// ]
 message Message {
     string public_ip = 1;
+    rivet.common.Uuid datacenter_id = 2;
     // If set in server install message
-    optional rivet.common.Uuid server_id = 2;
-    rivet.backend.cluster.Provider provider = 3;
-    optional string provider_api_token = 4;
+    optional rivet.common.Uuid server_id = 3;
+    rivet.backend.cluster.Provider provider = 4;
 }

svc/pkg/cluster/types/msg/server-install.proto

@@ -12,11 +12,13 @@ import "proto/backend/cluster.proto";
 message Message {
     string public_ip = 1;
     rivet.backend.cluster.PoolType pool_type = 2;
-    // Only if installing on a "server" (see cluster database). Used to check if
-    // the server is currently being deleted
+
+    rivet.common.Uuid datacenter_id = 6;
+    // Unset when installing prebake servers since they don't have an id. Used to check if
+    // the server is currently being deleted to prevent installation
     optional rivet.common.Uuid server_id = 3;
+
     // Simply passed to the install complete message
     rivet.backend.cluster.Provider provider = 4;
-    optional string provider_api_token = 5;
-    bool initialize_immediately = 6;
+    bool initialize_immediately = 5;
 }

svc/pkg/cluster/util/src/lib.rs

@@ -3,6 +3,10 @@ use uuid::Uuid;
 
 pub mod test;
 
+// Use the hash of the server install script in the image variant so that if the install scripts are updated
+// we won't be using the old image anymore
+pub const INSTALL_SCRIPT_HASH: &str = include_str!("../gen/hash.txt");
+
 // NOTE: We don't reserve CPU because Nomad is running as a higher priority process than the rest and
 // shouldn't be doing much heavy lifting.
 const RESERVE_SYSTEM_MEMORY: u64 = 512;
@@ -71,39 +75,3 @@ pub fn server_name(
 	)
 }
 
-// Use the hash of the server install script in the image variant so that if the install scripts are updated
-// we wont be using the old image anymore
-const CLUSTER_SERVER_INSTALL_HASH: &str = include_str!("../gen/hash.txt");
-
-// Used for linode labels which have to be between 3 and 64 characters for some reason
-pub fn simple_image_variant(
-	provider_datacenter_id: &str,
-	pool_type: backend::cluster::PoolType,
-) -> String {
-	let ns = rivet_util::env::namespace();
-	let pool_type_str = match pool_type {
-		backend::cluster::PoolType::Job => "job",
-		backend::cluster::PoolType::Gg => "gg",
-		backend::cluster::PoolType::Ats => "ats",
-	};
-
-	format!("{ns}-{provider_datacenter_id}-{pool_type_str}")
-}
-
-pub fn image_variant(
-	provider: backend::cluster::Provider,
-	provider_datacenter_id: &str,
-	pool_type: backend::cluster::PoolType,
-) -> String {
-	let ns = rivet_util::env::namespace();
-	let provider_str = match provider {
-		backend::cluster::Provider::Linode => "linode",
-	};
-	let pool_type_str = match pool_type {
-		backend::cluster::PoolType::Job => "job",
-		backend::cluster::PoolType::Gg => "gg",
-		backend::cluster::PoolType::Ats => "ats",
-	};
-
-	format!("{ns}-{CLUSTER_SERVER_INSTALL_HASH}-{provider_str}-{provider_datacenter_id}-{pool_type_str}")
-}

svc/pkg/cluster/worker/src/workers/nomad_node_drain_complete.rs

@@ -12,8 +12,7 @@ async fn worker(
 		"
 		UPDATE db_cluster.servers
 		SET cloud_destroy_ts = $2
-		WHERE
-			server_id = $1
+		WHERE server_id = $1
 		RETURNING datacenter_id
 		",
 		&server_id,

svc/pkg/cluster/worker/src/workers/server_destroy.rs

@@ -44,7 +44,7 @@ async fn worker(ctx: &OperationContext<cluster::msg::server_destroy::Message>) -
 
 			op!([ctx] linode_server_destroy {
 				server_id: ctx.server_id,
-				api_token: datacenter.provider_api_token.clone(),
+				datacenter_id: Some(server.datacenter_id.into()),
 			})
 			.await?;
 		}

svc/pkg/cluster/worker/src/workers/server_install/mod.rs

@@ -135,9 +135,9 @@ async fn worker(ctx: &OperationContext<cluster::msg::server_install::Message>) -
 
 	msg!([ctx] cluster::msg::server_install_complete(&ctx.public_ip) {
 		public_ip: ctx.public_ip.clone(),
+		datacenter_id: ctx.datacenter_id,
 		server_id: ctx.server_id,
 		provider: ctx.provider,
-		provider_api_token: ctx.provider_api_token.clone(),
 	})
 	.await?;
 

svc/pkg/cluster/worker/src/workers/server_install_complete.rs

@@ -12,7 +12,7 @@ async fn worker(
 			if ctx.server_id.is_none() {
 				msg!([ctx] linode::msg::prebake_install_complete(&ctx.public_ip) {
 					public_ip: ctx.public_ip.clone(),
-					api_token: ctx.provider_api_token.clone(),
+					datacenter_id: ctx.datacenter_id,
 				})
 				.await?;
 			}

svc/pkg/cluster/worker/src/workers/server_provision.rs

@@ -81,13 +81,13 @@ async fn worker(
 				);
 
 				let res = op!([ctx] linode_server_provision {
+					datacenter_id: ctx.datacenter_id,
 					server_id: ctx.server_id,
 					provider_datacenter_id: datacenter.provider_datacenter_id.clone(),
 					hardware: Some(hardware.clone()),
 					pool_type: ctx.pool_type,
 					vlan_ip: vlan_ip.clone(),
 					tags: ctx.tags.clone(),
-					api_token: datacenter.provider_api_token.clone(),
 				})
 				.await;
 
@@ -139,10 +139,10 @@ async fn worker(
 		if !provision_res.already_installed {
 			msg!([ctx] cluster::msg::server_install(&provision_res.public_ip) {
 				public_ip: provision_res.public_ip,
+				datacenter_id: ctx.datacenter_id,
 				server_id: ctx.server_id,
 				pool_type: ctx.pool_type,
 				provider: ctx.provider,
-				provider_api_token: datacenter.provider_api_token.clone(),
 				initialize_immediately: true,
 			})
 			.await?;

svc/pkg/linode/ops/server-destroy/Cargo.toml

@@ -12,6 +12,8 @@ reqwest = { version = "0.11", features = ["json"] }
 util-cluster = { package = "rivet-util-cluster", path = "../../../cluster/util" }
 util-linode = { package = "rivet-util-linode", path = "../../util" }
 
+cluster-datacenter-get = { path = "../../../cluster/ops/datacenter-get" }
+
 [dependencies.sqlx]
 version = "0.7"
 default-features = false

svc/pkg/linode/ops/server-destroy/src/lib.rs

@@ -15,6 +15,13 @@ pub async fn handle(
 ) -> GlobalResult<linode::server_destroy::Response> {
 	let crdb = ctx.crdb().await?;
 	let server_id = unwrap_ref!(ctx.server_id).as_uuid();
+	let datacenter_id = unwrap!(ctx.datacenter_id);
+
+	let datacenter_res = op!([ctx] cluster_datacenter_get {
+		datacenter_ids: vec![datacenter_id],
+	})
+	.await?;
+	let datacenter = unwrap!(datacenter_res.datacenters.first());
 
 	let data = sql_fetch_optional!(
 		[ctx, LinodeData, &crdb]
@@ -33,7 +40,7 @@ pub async fn handle(
 	};
 
 	// Build HTTP client
-	let client = util_linode::Client::new(ctx.api_token.clone()).await?;
+	let client = util_linode::Client::new(datacenter.provider_api_token.clone()).await?;
 
 	if let Some(linode_id) = data.linode_id {
 		api::delete_instance(&client, linode_id).await?;

svc/pkg/linode/ops/server-provision/Cargo.toml

@@ -12,6 +12,8 @@ reqwest = { version = "0.11", features = ["json"] }
 util-cluster = { package = "rivet-util-cluster", path = "../../../cluster/util" }
 util-linode = { package = "rivet-util-linode", path = "../../util" }
 
+cluster-datacenter-get = { path = "../../../cluster/ops/datacenter-get" }
+
 [dependencies.sqlx]
 version = "0.7"
 default-features = false

svc/pkg/linode/ops/server-provision/README.md

@@ -0,0 +1,3 @@
+# linode-server-provision
+
+This was meant to be agnostic to all other packages and simply create a server on Linode, but because of custom API keys and prebake images we need to include a `datacenter_id` in the request. In the future and if needed this can be made optional so that this endpoint does not require a `datacenter_id`.