feat: dedicated logger service

.circleci/config.yml

@@ -383,6 +383,9 @@ jobs:
       deploys-api-key:
         description: "Shuttle api key used by the gateway to authorise starting idle deployments"
         type: env_var_name
+      logger-postgres-uri:
+        description: "URI used to connect to the logger RDS postgres database."
+        type: string
       production:
         description: "Push and deploy to production"
         type: boolean
@@ -411,6 +414,7 @@ jobs:
             DD_API_KEY=$DD_API_KEY \
             HONEYCOMB_API_KEY=${<< parameters.honeycomb-api-key >>} \
             DEPLOYS_API_KEY=${<< parameters.deploys-api-key >>} \
+            LOGGER_POSTGRES_URI=${<< parameters.logger-postgres-uri >>} \
             make deploy
       - when:
           condition: << parameters.production >>
@@ -709,6 +713,7 @@ workflows:
                 # need to spawn docker containers, therefore in a machine
                 - cargo-shuttle
                 - shuttle-provisioner
+                - shuttle-logger
       - e2e-test:
           requires:
             - test-standalone
@@ -739,6 +744,7 @@ workflows:
           mongodb-password: DEV_MONGO_INITDB_ROOT_PASSWORD
           honeycomb-api-key: DEV_HONEYCOMB_API_KEY
           deploys-api-key: DEV_DEPLOYS_API_KEY
+          logger-postgres-uri: DEV_LOGGER_POSTGRES_URI
           requires:
             - build-and-push-unstable
   release:
@@ -818,6 +824,7 @@ workflows:
           mongodb-password: PROD_MONGO_INITDB_ROOT_PASSWORD
           honeycomb-api-key: PROD_HONEYCOMB_API_KEY
           deploys-api-key: PROD_DEPLOYS_API_KEY
+          logger-postgres-uri: PROD_LOGGER_POSTGRES_URI
           ssh-fingerprint: 6a:c5:33:fe:5b:c9:06:df:99:64:ca:17:0d:32:18:2e
           ssh-config-script: production-ssh-config.sh
           ssh-host: shuttle.prod.internal

Cargo.toml

@@ -9,6 +9,7 @@ members = [
   "common-tests",
   "deployer",
   "gateway",
+  "logger",
   "proto",
   "provisioner",
   "resource-recorder",
@@ -46,6 +47,7 @@ bytes = "1.3.0"
 cap-std = "1.0.2"
 cargo_metadata = "0.15.3"
 chrono = { version = "0.4.23", default-features = false }
+colored = "2.0.0"
 clap = { version = "4.2.7", features = ["derive"] }
 crossbeam-channel = "0.5.7"
 crossterm = "0.26.0"
@@ -64,6 +66,8 @@ jsonwebtoken = { version = "8.2.0" }
 once_cell = "1.16.0"
 opentelemetry = { version = "0.19.0", features = ["rt-tokio"] }
 opentelemetry-http = "0.8.0"
+opentelemetry-otlp = "0.12.0"
+opentelemetry-proto = "0.2.0"
 pin-project = "1.0.12"
 pipe = "0.4.0"
 portpicker = "0.1.1"
@@ -83,6 +87,7 @@ tempfile = "3.4.0"
 thiserror = "1.0.37"
 tar = "0.4.38"
 tokio = { version = "1.22.0" }
+tokio-stream = "0.1.11"
 toml = "0.5.9"
 toml_edit = "0.16.0"
 tonic = "0.8.3"
@@ -93,7 +98,6 @@ tracing = { version = "0.1.37", default-features = false }
 tracing-opentelemetry = "0.19.0"
 tracing-subscriber = { version = "0.3.16", default-features = false, features = [
   "registry",
-  "std",
 ] }
 ttl_cache = "0.5.1"
 utoipa = { version = "3.2.1", features = [ "uuid", "chrono" ] }

Containerfile

@@ -45,8 +45,9 @@ RUN cargo build \
     $(if [ "$CARGO_PROFILE" = "release" ]; then echo --release; fi) \
     --bin shuttle-auth \
     --bin shuttle-deployer \
-    --bin shuttle-provisioner \
     --bin shuttle-gateway \
+    --bin shuttle-logger \
+    --bin shuttle-provisioner \
     --bin shuttle-resource-recorder \
     --bin shuttle-next -F next
 
@@ -92,6 +93,9 @@ FROM shuttle-gateway AS shuttle-gateway-dev
 # For testing certificates locally
 COPY --from=planner /build/*.pem /usr/src/shuttle/
 
+FROM shuttle-crate-base AS shuttle-logger
+FROM shuttle-logger AS shuttle-logger-dev
+
 FROM shuttle-crate-base AS shuttle-provisioner
 ARG CARGO_PROFILE
 COPY --from=builder /build/target/${CARGO_PROFILE}/shuttle-provisioner /usr/local/bin/service

DEVELOPING.md

@@ -228,7 +228,7 @@ USE_PANAMAX=disable make images
 make docker-compose.rendered.yml
 
 # then run
-docker compose -f docker-compose.rendered.yml up provisioner resource-recorder
+docker compose -f docker-compose.rendered.yml up provisioner resource-recorder logger otel-collector
 ```
 
 This starts the provisioner and the auth service, while preventing `gateway` from starting up.
@@ -237,7 +237,7 @@ Make sure an admin user is inserted into auth and that the key is used by cargo-
 We're now ready to start a local run of the deployer:
 
 ```bash
-cargo run -p shuttle-deployer -- --provisioner-address http://localhost:3000 --auth-uri http://localhost:8008 --resource-recorder http://localhost:8007 --proxy-fqdn local.rs --admin-secret dh9z58jttoes3qvt --local --project-id "01H7WHDK23XYGSESCBG6XWJ1V0" --project <name>
+OTLP_ADDRESS=http://127.0.0.1:4317 cargo run -p shuttle-deployer -- --provisioner-address http://localhost:3000 --auth-uri http://localhost:8008 --resource-recorder http://localhost:8007 --logger-uri http://localhost:8080 --proxy-fqdn local.rs --admin-secret dh9z58jttoes3qvt --local --project-id "01H7WHDK23XYGSESCBG6XWJ1V0" --project <name>
 ```
 
 The `<name>` needs to match the name of the project that will be deployed to this deployer.

Makefile

@@ -22,8 +22,10 @@ BUILDX_FLAGS=$(BUILDX_OP) $(PLATFORM_FLAGS) $(CACHE_FLAGS)
 RUSTUP_TOOLCHAIN=1.72.0
 
 TAG?=$(shell git describe --tags --abbrev=0)
-BACKEND_TAG?=$(TAG)
+AUTH_TAG?=$(TAG)
 DEPLOYER_TAG?=$(TAG)
+GATEWAY_TAG?=$(TAG)
+LOGGER_TAG?=$(TAG)
 PROVISIONER_TAG?=$(TAG)
 RESOURCE_RECORDER_TAG?=$(TAG)
 
@@ -68,6 +70,11 @@ CARGO_PROFILE?=debug
 RUST_LOG?=shuttle=debug,info
 DEV_SUFFIX=-dev
 DEPLOYS_API_KEY?=gateway4deployes
+
+# this should use the same version as our prod RDS database
+LOGGER_POSTGRES_TAG?=15
+LOGGER_POSTGRES_PASSWORD?=postgres
+LOGGER_POSTGRES_URI?=postgres://postgres:${LOGGER_POSTGRES_PASSWORD}@logger-postgres:5432/postgres
 endif
 
 ifeq ($(CI),true)
@@ -98,11 +105,16 @@ endif
 
 DOCKER_COMPOSE_ENV=\
 	STACK=$(STACK)\
-	BACKEND_TAG=$(BACKEND_TAG)\
+	AUTH_TAG=$(AUTH_TAG)\
 	DEPLOYER_TAG=$(DEPLOYER_TAG)\
+	GATEWAY_TAG=$(GATEWAY_TAG)\
+	LOGGER_TAG=$(LOGGER_TAG)\
 	PROVISIONER_TAG=$(PROVISIONER_TAG)\
 	RESOURCE_RECORDER_TAG=$(RESOURCE_RECORDER_TAG)\
 	POSTGRES_TAG=${POSTGRES_TAG}\
+	LOGGER_POSTGRES_TAG=${LOGGER_POSTGRES_TAG}\
+	LOGGER_POSTGRES_PASSWORD=${LOGGER_POSTGRES_PASSWORD}\
+	LOGGER_POSTGRES_URI=${LOGGER_POSTGRES_URI}\
 	PANAMAX_TAG=${PANAMAX_TAG}\
 	OTEL_TAG=${OTEL_TAG}\
 	APPS_FQDN=$(APPS_FQDN)\
@@ -118,15 +130,31 @@ DOCKER_COMPOSE_ENV=\
 	COMPOSE_PROFILES=$(COMPOSE_PROFILES)\
 	DOCKER_SOCK=$(DOCKER_SOCK)
 
-.PHONY: images clean src up down deploy shuttle-% shuttle-images postgres docker-compose.rendered.yml test bump-% deploy-examples publish publish-% --validate-version
+.PHONY: clean images the-shuttle-images shuttle-% postgres panamax otel deploy test docker-compose.rendered.yml up down
 
 clean:
 	rm .shuttle-*
 	rm docker-compose.rendered.yml
 
-images: shuttle-images postgres panamax otel
+images: the-shuttle-images postgres panamax otel
 
-shuttle-images: shuttle-auth shuttle-deployer shuttle-gateway shuttle-provisioner shuttle-resource-recorder
+the-shuttle-images: shuttle-auth shuttle-deployer shuttle-gateway shuttle-logger shuttle-provisioner shuttle-resource-recorder
+
+shuttle-%:
+	$(DOCKER_BUILD) \
+		--target $(@)$(DEV_SUFFIX) \
+		--build-arg folder=$(*) \
+		--build-arg crate=$(@) \
+		--build-arg prepare_args=$(PREPARE_ARGS) \
+		--build-arg PROD=$(PROD) \
+		--build-arg RUSTUP_TOOLCHAIN=$(RUSTUP_TOOLCHAIN) \
+		--build-arg CARGO_PROFILE=$(CARGO_PROFILE) \
+		--tag $(CONTAINER_REGISTRY)/$(*):$(COMMIT_SHA) \
+		--tag $(CONTAINER_REGISTRY)/$(*):$(TAG) \
+		--tag $(CONTAINER_REGISTRY)/$(*):latest \
+		$(BUILDX_FLAGS) \
+		-f Containerfile \
+		.
 
 postgres:
 	$(DOCKER_BUILD) \
@@ -175,7 +203,11 @@ up: $(DOCKER_COMPOSE_FILES)
 	$(SHUTTLE_DETACH)
 
 down: $(DOCKER_COMPOSE_FILES)
-	$(DOCKER_COMPOSE_ENV) $(DOCKER_COMPOSE) $(addprefix -f ,$(DOCKER_COMPOSE_FILES)) -p $(STACK) down
+	$(DOCKER_COMPOSE_ENV) \
+	$(DOCKER_COMPOSE) \
+	$(addprefix -f ,$(DOCKER_COMPOSE_FILES)) \
+	-p $(STACK) \
+	down
 
 shuttle-%:
 	$(DOCKER_BUILD) \

auth/src/main.rs

@@ -1,7 +1,7 @@
 use std::io;
 
 use clap::Parser;
-use shuttle_common::backends::tracing::setup_tracing;
+use shuttle_common::{backends::tracing::setup_tracing, log::Backend};
 use sqlx::migrate::Migrator;
 use tracing::{info, trace};
 
@@ -15,7 +15,7 @@ async fn main() -> io::Result<()> {
 
     trace!(args = ?args, "parsed args");
 
-    setup_tracing(tracing_subscriber::registry(), "auth");
+    setup_tracing(tracing_subscriber::registry(), Backend::Auth);
 
     let db_path = args.state.join("authentication.sqlite");
 

cargo-shuttle/src/args.rs

@@ -350,7 +350,7 @@ pub(crate) fn parse_init_path(path: OsString) -> Result<PathBuf, io::Error> {
         )
     })?;
 
-    parse_path(path.clone()).map_err(|e| io::Error::new(ErrorKind::InvalidInput, e))
+    parse_path(path).map_err(|e| io::Error::new(ErrorKind::InvalidInput, e))
 }
 
 #[cfg(test)]