feat(deployer): connect deployer to builder service (shuttle-hq#1248)

* feat(deployer): connect deployer to builder service

* feat(gateway): pass builder hostname via gateway

Containerfile

@@ -44,6 +44,7 @@ COPY --from=planner /build .
 RUN cargo build \
     $(if [ "$CARGO_PROFILE" = "release" ]; then echo --release; fi) \
     --bin shuttle-auth \
+    --bin shuttle-builder \
     --bin shuttle-deployer \
     --bin shuttle-gateway \
     --bin shuttle-logger \
@@ -110,3 +111,8 @@ FROM shuttle-crate-base AS shuttle-resource-recorder
 ARG CARGO_PROFILE
 COPY --from=builder /build/target/${CARGO_PROFILE}/shuttle-resource-recorder /usr/local/bin/service
 FROM shuttle-resource-recorder AS shuttle-resource-recorder-dev
+
+FROM shuttle-crate-base AS shuttle-builder
+ARG CARGO_PROFILE
+COPY --from=builder /build/target/${CARGO_PROFILE}/shuttle-builder /usr/local/bin/service
+FROM shuttle-builder AS shuttle-builder-dev

Makefile

@@ -23,6 +23,7 @@ RUSTUP_TOOLCHAIN=1.72.0
 
 TAG?=$(shell git describe --tags --abbrev=0)
 AUTH_TAG?=$(TAG)
+BUILDER_TAG?=$(TAG)
 DEPLOYER_TAG?=$(TAG)
 GATEWAY_TAG?=$(TAG)
 LOGGER_TAG?=$(TAG)
@@ -106,6 +107,7 @@ endif
 DOCKER_COMPOSE_ENV=\
 	STACK=$(STACK)\
 	AUTH_TAG=$(AUTH_TAG)\
+	BUILDER_TAG=$(BUILDER_TAG)\
 	DEPLOYER_TAG=$(DEPLOYER_TAG)\
 	GATEWAY_TAG=$(GATEWAY_TAG)\
 	LOGGER_TAG=$(LOGGER_TAG)\
@@ -138,7 +140,7 @@ clean:
 
 images: the-shuttle-images postgres panamax otel
 
-the-shuttle-images: shuttle-auth shuttle-deployer shuttle-gateway shuttle-logger shuttle-provisioner shuttle-resource-recorder
+the-shuttle-images: shuttle-auth shuttle-builder shuttle-deployer shuttle-gateway shuttle-logger shuttle-provisioner shuttle-resource-recorder
 
 shuttle-%:
 	$(DOCKER_BUILD) \

deployer/src/args.rs

@@ -59,6 +59,10 @@ pub struct Args {
     #[clap(long, default_value = "http://127.0.0.1:8008")]
     pub auth_uri: Uri,
 
+    /// Address to reach the builder service at
+    #[clap(long, default_value = "http://builder:8000")]
+    pub builder_uri: Endpoint,
+
     /// Uri to folder to store all artifacts
     #[clap(long, default_value = "/tmp")]
     pub artifacts_path: PathBuf,

deployer/src/main.rs

@@ -11,7 +11,10 @@ use shuttle_deployer::{
     start, start_proxy, Args, DeployLayer, Persistence, ResourceManager, RuntimeManager,
 };
 use shuttle_deployer::{start, start_proxy, Args, Persistence, RuntimeManager, StateChangeLayer};
-use shuttle_proto::logger::{logger_client::LoggerClient, Batcher};
+use shuttle_proto::{
+    builder::builder_client::BuilderClient,
+    logger::{logger_client::LoggerClient, Batcher},
+};
 use tokio::select;
 use tower::ServiceBuilder;
 use tracing::{error, trace};
@@ -46,6 +49,19 @@ async fn main() {
     let logger_client = LoggerClient::new(channel);
     let logger_batcher = Batcher::wrap(logger_client.clone());
 
+    let channel = ServiceBuilder::new()
+        .layer(ClaimLayer)
+        .layer(InjectPropagationLayer)
+        .service(
+            args.builder_uri
+                .connect()
+                .await
+                .expect("failed to connect to builder"),
+        );
+
+    // TODO: use builder client to send project archives to the builder service.
+    let _builder_client = BuilderClient::new(channel);
+
     setup_tracing(
         tracing_subscriber::registry()
             .with(StateChangeLayer {

docker-compose.yml

@@ -1,6 +1,7 @@
 version: "3.7"
 volumes:
   auth-vol:
+  builder-store-vol:
   gateway-vol:
   postgres-vol:
   panamax-crates-vol:
@@ -45,6 +46,36 @@ services:
       - "--state=/var/lib/shuttle-auth"
       - "start"
       - "--address=0.0.0.0:8000"
+  builder:
+    image: "${CONTAINER_REGISTRY}/builder:${BUILDER_TAG}"
+    depends_on:
+      - auth
+    ports:
+      - 8009:8000
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+        max_attempts: 3
+      update_config:
+        order: start-first
+        failure_action: rollback
+        delay: 10s
+      rollback_config:
+        parallelism: 0
+        order: stop-first
+      placement:
+        constraints:
+          - node.hostname==builder
+    networks:
+      user-net:
+    volumes:
+      - builder-store-vol:/nix/store
+    environment:
+      - RUST_LOG=${RUST_LOG}
+    command:
+      - "--address=0.0.0.0:8000"
+      - "--auth-uri=http://auth:8000"
   gateway:
     image: "${CONTAINER_REGISTRY}/gateway:${GATEWAY_TAG}"
     depends_on:
@@ -91,6 +122,7 @@ services:
       - "--auth-uri=http://auth:8000"
       - "--deploys-api-key=${DEPLOYS_API_KEY}"
       - "--provisioner-host=provisioner"
+      - "--builder-host=builder"
       - "--proxy-fqdn=${APPS_FQDN}"
       - "--use-tls=${USE_TLS}"
     healthcheck:

gateway/src/args.rs

@@ -56,6 +56,9 @@ pub struct ContextArgs {
     /// the provisioner service
     #[arg(long, default_value = "provisioner")]
     pub provisioner_host: String,
+    /// The address at which a deployer container will find the builder service
+    #[arg(long, default_value = "builder")]
+    pub builder_host: String,
     /// Address to reach the authentication service at
     #[arg(long, default_value = "http://127.0.0.1:8008")]
     pub auth_uri: Uri,

gateway/src/lib.rs

@@ -606,6 +606,7 @@ pub mod tests {
                 env::var("SHUTTLE_TESTS_NETWORK").unwrap_or_else(|_| "shuttle_default".to_string());
 
             let provisioner_host = "provisioner".to_string();
+            let builder_host = "builder".to_string();
 
             let docker_host = "/var/run/docker.sock".to_string();
 
@@ -619,6 +620,7 @@ pub mod tests {
                     image,
                     prefix,
                     provisioner_host,
+                    builder_host,
                     auth_uri: auth_uri.clone(),
                     network_name,
                     proxy_fqdn: FQDN::from_str("test.shuttleapp.rs").unwrap(),

gateway/src/project.rs

@@ -698,6 +698,7 @@ impl ProjectCreating {
             image: default_image,
             prefix,
             provisioner_host,
+            builder_host,
             auth_uri,
             fqdn: public,
             ..
@@ -750,6 +751,8 @@ impl ProjectCreating {
                         "/opt/shuttle/deployer.sqlite",
                         "--auth-uri",
                         auth_uri,
+                        "--builder-uri",
+                        format!("http://{builder_host}:8000"),
                         "--project-id",
                         self.project_id.to_string()
                     ],

gateway/src/service.rs

@@ -63,6 +63,7 @@ pub struct ContainerSettingsBuilder {
     prefix: Option<String>,
     image: Option<String>,
     provisioner: Option<String>,
+    builder: Option<String>,
     auth_uri: Option<String>,
     network_name: Option<String>,
     fqdn: Option<String>,
@@ -80,6 +81,7 @@ impl ContainerSettingsBuilder {
             prefix: None,
             image: None,
             provisioner: None,
+            builder: None,
             auth_uri: None,
             network_name: None,
             fqdn: None,
@@ -91,6 +93,7 @@ impl ContainerSettingsBuilder {
             prefix,
             network_name,
             provisioner_host,
+            builder_host,
             auth_uri,
             image,
             proxy_fqdn,
@@ -99,6 +102,7 @@ impl ContainerSettingsBuilder {
         self.prefix(prefix)
             .image(image)
             .provisioner_host(provisioner_host)
+            .builder_host(builder_host)
             .auth_uri(auth_uri)
             .network_name(network_name)
             .fqdn(proxy_fqdn)
@@ -121,6 +125,11 @@ impl ContainerSettingsBuilder {
         self
     }
 
+    pub fn builder_host<S: ToString>(mut self, host: S) -> Self {
+        self.builder = Some(host.to_string());
+        self
+    }
+
     pub fn auth_uri<S: ToString>(mut self, auth_uri: S) -> Self {
         self.auth_uri = Some(auth_uri.to_string());
         self
@@ -140,6 +149,7 @@ impl ContainerSettingsBuilder {
         let prefix = self.prefix.take().unwrap();
         let image = self.image.take().unwrap();
         let provisioner_host = self.provisioner.take().unwrap();
+        let builder_host = self.builder.take().unwrap();
         let auth_uri = self.auth_uri.take().unwrap();
 
         let network_name = self.network_name.take().unwrap();
@@ -149,6 +159,7 @@ impl ContainerSettingsBuilder {
             prefix,
             image,
             provisioner_host,
+            builder_host,
             auth_uri,
             network_name,
             fqdn,
@@ -161,6 +172,7 @@ pub struct ContainerSettings {
     pub prefix: String,
     pub image: String,
     pub provisioner_host: String,
+    pub builder_host: String,
     pub auth_uri: String,
     pub network_name: String,
     pub fqdn: String,