chore(dev): respect CARGO_TARGET_DIR in bolt & use non-mounted target in dev container

.devcontainer/devcontainer.json

@@ -26,5 +26,11 @@
 		"vscode": {
 			"extensions": ["rust-lang.rust-analyzer"]
 		}
-	}
+	},
+	// docs/infrastructure/devcontainers/RUST_WORKAROUND.md
+	"containerEnv": {
+		"CARGO_TARGET_DIR": "/target"
+	},
+	// docs/infrastructure/devcontainers/RUST_WORKAROUND.md
+	"postCreateCommand": "echo 'export PATH=\"/target/debug:/target/release:${PATH}\"' >> ~/.bashrc"
 }

docs/infrastructure/devcontainers/RUST_WORKAROUND.md

@@ -0,0 +1,23 @@
+# Rust Workaround
+
+## Overview
+
+This issue is specific to Apple Silicon machines.
+
+Dev containers have a bug with incremental builds which lead to nonsensical Rust errors.
+
+These are documented in better detail here:
+
+-   https://stackoverflow.com/questions/72448053/rust-incremental-build-not-working-in-vscode-devcontainer
+-   https://github.com/docker/for-mac/issues/7059
+
+## Moving target dir out of mounted dir
+
+In order to fix this, we move the target path to `/target` which is not mounted to the host system. This way the virtual fs does not trigger issues with Rust.
+
+This complicates things, since we now compile binaries in non-standard paths.
+
+This is done by updating `devcontainer.json` to:
+
+-   Overriding `CARGO_TARGET_DIR` to `/target`
+-   Updating `PATH` to include `/target/debug` and `/target/release` so we can still access `bolt`

infra/tf/k8s_cluster_k3d/main.tf

@@ -12,8 +12,8 @@ resource "k3d_cluster" "main" {
 
 	# Mount repository in to k3d so we can access the built binaries
 	volume {
-		source = var.project_root
-		destination = "/rivet-src"
+		source = var.cargo_target_dir
+		destination = "/target"
 		node_filters = ["server:0"]
 	}
 

infra/tf/k8s_cluster_k3d/vars.tf

@@ -6,6 +6,10 @@ variable "project_root" {
 	type = string
 }
 
+variable "cargo_target_dir" {
+	type = string
+}
+
 variable "public_ip" {
 	type = string
 }

lib/bolt/core/src/context/project.rs

@@ -2,6 +2,7 @@ use std::{
 	collections::{HashMap, HashSet},
 	path::{Path, PathBuf},
 	process::Command,
+	str::FromStr,
 	sync::{Arc, Weak},
 };
 
@@ -68,6 +69,13 @@ impl ProjectContextData {
 	pub fn path(&self) -> &Path {
 		self.path.as_path()
 	}
+
+	pub fn cargo_target_dir(&self) -> PathBuf {
+		std::env::var("CARGO_TARGET_DIR").map_or_else(
+			|_| self.path().join("target"),
+			|x| PathBuf::from_str(&x).unwrap(),
+		)
+	}
 }
 
 impl ProjectContextData {

lib/bolt/core/src/context/service.rs

@@ -242,8 +242,7 @@ impl ServiceContextData {
 	pub async fn rust_bin_path(&self, optimization: &BuildOptimization) -> PathBuf {
 		self.project()
 			.await
-			.path()
-			.join("target")
+			.cargo_target_dir()
 			.join(match optimization {
 				BuildOptimization::Release => "release",
 				BuildOptimization::Debug => "debug",

lib/bolt/core/src/dep/cargo/cli.rs

@@ -88,7 +88,7 @@ pub async fn build<'a, T: AsRef<str>>(ctx: &ProjectContext, opts: BuildOpts<'a,
 		#!/bin/bash
 		set -euf
 
-		export CARGO_TARGET_DIR=$(readlink -f ./target)
+		[ -z "$CARGO_TARGET_DIR" ] && export CARGO_TARGET_DIR=$(readlink -f ./target)
 		# Used for Tokio Console. See https://github.com/tokio-rs/console#using-it
 		export RUSTFLAGS="--cfg tokio_unstable"
 		# Used for debugging

lib/bolt/core/src/dep/k8s/gen.rs

@@ -317,7 +317,7 @@ pub async fn gen_svc(exec_ctx: &ExecServiceContext) -> Vec<serde_json::Value> {
 			"IfNotPresent",
 			format!(
 				"{} {}",
-				Path::new("/rivet-src").join(exec_path).display(),
+				Path::new("/target").join(exec_path).display(),
 				args.join(" ")
 			),
 		),
@@ -644,9 +644,9 @@ async fn build_volumes(
 		ExecServiceDriver::LocalBinaryArtifact { .. } => {
 			// Volumes
 			volumes.push(json!({
-				"name": "rivet-src",
+				"name": "target",
 				"hostPath": {
-					"path": "/rivet-src",
+					"path": "/target",
 					"type": "Directory"
 				}
 			}));
@@ -660,8 +660,8 @@ async fn build_volumes(
 
 			// Mounts
 			volume_mounts.push(json!({
-				"name": "rivet-src",
-				"mountPath": "/rivet-src",
+				"name": "target",
+				"mountPath": "/target",
 				"readOnly": true
 			}));
 			volume_mounts.push(json!({

lib/bolt/core/src/dep/terraform/gen.rs

@@ -255,6 +255,12 @@ async fn vars(ctx: &ProjectContext) {
 		vars.insert("tunnels".into(), json!(&tunnels));
 	}
 
+	// Rust
+	vars.insert(
+		"cargo_target_dir".into(),
+		json!(ctx.cargo_target_dir().display().to_string()),
+	);
+
 	// Services
 	{
 		let mut services = HashMap::new();

lib/bolt/core/src/tasks/check.rs

@@ -130,7 +130,7 @@ async fn check_svcs(
 			let mut cmd = Command::new("cargo");
 			cmd.current_dir(path);
 			cmd.env("RUSTFLAGS", "--cfg tokio_unstable");
-			cmd.env("CARGO_TARGET_DIR", ctx.path().join("target"));
+			cmd.env("CARGO_TARGET_DIR", ctx.cargo_target_dir());
 			cmd.arg("clippy");
 
 			// Check tests, which will also check the main module. Using

lib/bolt/core/src/tasks/test.rs

@@ -231,9 +231,9 @@ async fn run_test(
 	// Convert path relative to project
 	let relative_path = test_binary
 		.path
-		.strip_prefix(ctx.path())
+		.strip_prefix(ctx.cargo_target_dir())
 		.context("path not in project")?;
-	let container_path = Path::new("/rivet-src").join(relative_path);
+	let container_path = Path::new("/target").join(relative_path);
 
 	// Build exec ctx
 	let test_id = gen_test_id();
@@ -571,6 +571,10 @@ struct Data {
 
 // TODO: This only deletes linodes and firewalls, the ssh key still remains
 async fn cleanup_servers(ctx: &ProjectContext) -> Result<()> {
+	if ctx.ns().rivet.dynamic_servers.is_none() {
+		return Ok(());
+	}
+
 	eprintln!();
 	rivet_term::status::progress("Cleaning up servers", "");
 

lib/bolt/core/src/tasks/up.rs

@@ -373,7 +373,7 @@ async fn derive_local_build_driver(
 		RuntimeKind::Rust {} => ExecServiceDriver::LocalBinaryArtifact {
 			// Convert path to be relative to the project root
 			exec_path: exec_path
-				.strip_prefix(svc_ctx.project().await.path())
+				.strip_prefix(svc_ctx.project().await.cargo_target_dir())
 				.expect("rust binary path not inside of project dir")
 				.to_owned(),
 			args: Vec::new(),