feat(bolt): build svcs as docker containers locally

infra/tf/k8s_cluster_k3d/main.tf

@@ -7,6 +7,11 @@ terraform {
 	}
 }
 
+locals {
+	repo_host = "svc"
+	repo_port = 5001
+}
+
 resource "k3d_cluster" "main" {
 	name = "rivet-${var.namespace}"
 
@@ -80,6 +85,13 @@ resource "k3d_cluster" "main" {
 		node_filters = ["server:0"]
 	}
 
+	registries {
+		create {
+			name = "svc"
+			host_port = local.repo_port
+		}
+	}
+
 	k3s {
 		extra_args {
 			arg = "--disable=traefik"

infra/tf/k8s_cluster_k3d/output.tf

@@ -2,3 +2,10 @@ output "traefik_external_ip" {
 	value = var.public_ip
 }
 
+output "repo_host" {
+	value = local.repo_host
+}
+
+output "repo_port" {
+	value = local.repo_port
+}

lib/bolt/config/src/ns.rs

@@ -306,6 +306,7 @@ pub struct Docker {
 	///
 	/// See [here](https://docs.docker.com/docker-hub/download-rate-limit) for
 	/// more information on Docker Hub's rate limits.
+	#[serde(default)]
 	pub authenticate_all_docker_hub_pulls: bool,
 	/// Docker repository to upload builds to. Must end in a slash.
 	#[serde(default = "default_docker_repo")]
@@ -341,14 +342,21 @@ pub struct Kubernetes {
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub enum KubernetesProvider {
 	#[serde(rename = "k3d")]
-	K3d {},
+	K3d {
+		/// Tells bolt to use the K3d managed registry for svc builds. This will override ns.docker.repository
+		/// for image uploads.
+		#[serde(default)]
+		use_local_repo: bool,
+	},
 	#[serde(rename = "aws_eks")]
 	AwsEks {},
 }
 
 impl Default for KubernetesProvider {
 	fn default() -> Self {
-		Self::K3d {}
+		Self::K3d {
+			use_local_repo: false,
+		}
 	}
 }
 

lib/bolt/core/src/context/project.rs

@@ -11,7 +11,7 @@ use sha2::{Digest, Sha256};
 use tokio::{fs, sync::Mutex};
 
 use super::{RunContext, ServiceContext};
-use crate::{config, context, utils::command_helper::CommandHelper};
+use crate::{config, context, dep::terraform, utils::command_helper::CommandHelper};
 
 pub type ProjectContext = Arc<ProjectContextData>;
 
@@ -343,6 +343,11 @@ impl ProjectContextData {
 				"cannot enable billing without emailing"
 			);
 		}
+
+		assert!(
+			self.ns().docker.repository.ends_with('/'),
+			"docker repository must end with slash"
+		);
 	}
 
 	// Traverses from FS root to CWD, returns first directory with Bolt.toml
@@ -1017,6 +1022,36 @@ impl ProjectContextData {
 	}
 }
 
+impl ProjectContextData {
+	/// Gets the correct repo to push svc builds to/pull from.
+	pub async fn docker_repos(self: &Arc<Self>) -> (String, String) {
+		match self.ns().kubernetes.provider {
+			config::ns::KubernetesProvider::K3d { use_local_repo } if use_local_repo => {
+				let output = terraform::output::read_k8s_cluster_k3d(self).await;
+				let local_repo = format!("localhost:{}/", *output.repo_port);
+				let internal_repo = format!("{}:{}/", *output.repo_host, *output.repo_port);
+
+				(local_repo, internal_repo)
+			}
+			_ => (
+				self.ns().docker.repository.clone(),
+				self.ns().docker.repository.clone(),
+			),
+		}
+	}
+
+	/// Whether or not to build svc images locally vs inside of docker.
+	pub fn build_svcs_locally(&self) -> bool {
+		match self.ns().kubernetes.provider {
+			config::ns::KubernetesProvider::K3d { use_local_repo } if use_local_repo => false,
+			_ => matches!(
+				&self.ns().cluster.kind,
+				config::ns::ClusterKind::SingleNode { .. }
+			),
+		}
+	}
+}
+
 impl ProjectContextData {
 	pub fn leader_count(&self) -> usize {
 		match &self.ns().cluster.kind {

lib/bolt/core/src/context/service.rs

@@ -384,48 +384,52 @@ pub enum ServiceBuildPlan {
 	ExistingUploadedBuild { image_tag: String },
 
 	/// Build the service and upload to Docker.
-	BuildAndUpload { image_tag: String },
+	BuildAndUpload {
+		/// Push location (local repo)
+		push_image_tag: String,
+		/// Pull location (inside of k8s)
+		pull_image_tag: String,
+	},
 }
 
 impl ServiceContextData {
 	/// Determines if this service needs to be recompiled.
 	pub async fn build_plan(&self, build_context: &BuildContext) -> Result<ServiceBuildPlan> {
+		let project_ctx = self.project().await;
+
 		// Check if build exists on docker.io
-		let pub_image_tag = self.docker_image_tag(Some("docker.io/rivetgg/")).await?;
+		let pub_image_tag = self.docker_image_tag(&project_ctx, "docker.io/rivetgg/")?;
 		if docker::cli::container_exists(&pub_image_tag).await {
 			return Ok(ServiceBuildPlan::ExistingUploadedBuild {
 				image_tag: pub_image_tag,
 			});
 		}
 
-		// Check if build exists in custom repo
-		let image_tag = self.docker_image_tag(None).await?;
+		// Check if build exists in config repo
+		let image_tag = self.docker_image_tag(&project_ctx, &project_ctx.ns().docker.repository)?;
 		if docker::cli::container_exists(&image_tag).await {
 			return Ok(ServiceBuildPlan::ExistingUploadedBuild { image_tag });
 		}
 
-		let project_ctx = self.project().await;
+		if project_ctx.build_svcs_locally() {
+			// Derive the build path
+			let optimization = match &build_context {
+				BuildContext::Bin { optimization } => optimization,
+				BuildContext::Test { .. } => &BuildOptimization::Debug,
+			};
+			let output_path = self.rust_bin_path(optimization).await;
 
-		match &project_ctx.ns().cluster.kind {
-			// Build locally
-			config::ns::ClusterKind::SingleNode { .. } => {
-				// Derive the build path
-				let optimization = match &build_context {
-					BuildContext::Bin { optimization } => optimization,
-					BuildContext::Test { .. } => &BuildOptimization::Debug,
-				};
-				let output_path = self.rust_bin_path(optimization).await;
+			// Rust libs always attempt to rebuild (handled by cargo)
+			Ok(ServiceBuildPlan::BuildLocally {
+				exec_path: output_path,
+			})
+		} else {
+			let (push_repo, pull_repo) = project_ctx.docker_repos().await;
 
-				// Rust libs always attempt to rebuild (handled by cargo)
-				Ok(ServiceBuildPlan::BuildLocally {
-					exec_path: output_path,
-				})
-			}
-			// Build and upload to S3
-			config::ns::ClusterKind::Distributed { .. } => {
-				// Default to building
-				Ok(ServiceBuildPlan::BuildAndUpload { image_tag })
-			}
+			Ok(ServiceBuildPlan::BuildAndUpload {
+				push_image_tag: self.docker_image_tag(&project_ctx, &push_repo)?,
+				pull_image_tag: self.docker_image_tag(&project_ctx, &pull_repo)?,
+			})
 		}
 	}
 }
@@ -1320,12 +1324,10 @@ impl ServiceContextData {
 }
 
 impl ServiceContextData {
-	pub async fn docker_image_tag(&self, override_repo: Option<&str>) -> Result<String> {
-		let project_ctx = self.project().await;
+	pub fn docker_image_tag(&self, project_ctx: &ProjectContext, repo: &str) -> Result<String> {
+		ensure!(repo.ends_with('/'), "docker repository must end with slash");
 
 		let source_hash = project_ctx.source_hash();
-		let repo = override_repo.unwrap_or(&project_ctx.ns().docker.repository);
-		ensure!(repo.ends_with('/'), "docker repository must end with slash");
 
 		Ok(format!(
 			"{}{}:{}",
@@ -1336,7 +1338,9 @@ impl ServiceContextData {
 	}
 
 	pub async fn upload_build(&self) -> Result<()> {
-		let image_tag = self.docker_image_tag(None).await?;
+		let project_ctx = self.project().await;
+		let (push_repo, _) = project_ctx.docker_repos().await;
+		let image_tag = self.docker_image_tag(&project_ctx, &push_repo)?;
 
 		let mut cmd = Command::new("docker");
 		cmd.arg("push");