systemd + vault certs

Makefile

@@ -1,46 +1,93 @@
-# Self-signed certificate generation for fastcache api (mTLS).
+# Self-signed certificate generation + Vault publishing for fastcache api (mTLS).
 #   make certs        # generate CA, server, and client certs into certs/
+#   make verify-certs # verify the generated chain locally
+#   make vault-push   # verify, then push them to Vault from the cert builder
+#   make vault-pull   # pull them from Vault onto the fastcache_api host
 #   make clean-certs  # remove the certs/ directory
 
 CERT_DIR = certs
+CERT_DAYS ?= 14
+CERT_HOST ?= sdfdtn002.sdf.slac.stanford.edu
+CERT_VALIDITY = $(shell echo $$(($(CERT_DAYS) * 24))h)
 
-.PHONY: certs ca server client clean-certs
+VAULT_ADDR ?= https://vault.slac.stanford.edu
+VAULT_SECRET_PATH ?= lcls/psdm/lclstream/dev
+
+.PHONY: certs ca server client clean-certs check-certs verify-certs vault-push vault-pull
 
 certs: ca server client
 
 ca: $(CERT_DIR)/ca.crt
 
 $(CERT_DIR)/ca.crt:
 	mkdir -p $(CERT_DIR)
-	openssl req -x509 -newkey rsa:4096 -days 2 -nodes \
-		-keyout $(CERT_DIR)/ca.key -out $(CERT_DIR)/ca.crt \
-		-subj "/CN=fastcache-ca"
+	step certificate create "fastcache-ca" $(CERT_DIR)/ca.crt $(CERT_DIR)/ca.key \
+		--profile root-ca --not-after $(CERT_VALIDITY) --no-password --insecure -f
 	chmod 400 $(CERT_DIR)/ca.key $(CERT_DIR)/ca.crt
 
 server: $(CERT_DIR)/server.crt
 
 $(CERT_DIR)/server.crt: $(CERT_DIR)/ca.crt
-	openssl req -newkey rsa:4096 -nodes \
-		-keyout $(CERT_DIR)/server.key -out $(CERT_DIR)/server.csr \
-		-subj "/CN=$(shell hostname -f)" \
-		-addext "subjectAltName=DNS:$(shell hostname -f)"
-	openssl x509 -req -in $(CERT_DIR)/server.csr \
-		-CA $(CERT_DIR)/ca.crt -CAkey $(CERT_DIR)/ca.key -CAcreateserial \
-		-days 2 -out $(CERT_DIR)/server.crt -copy_extensions copyall
-	rm -f $(CERT_DIR)/server.csr
+	step certificate create "$(CERT_HOST)" $(CERT_DIR)/server.crt $(CERT_DIR)/server.key \
+		--profile leaf --ca $(CERT_DIR)/ca.crt --ca-key $(CERT_DIR)/ca.key \
+		--san $(CERT_HOST) --not-after $(CERT_VALIDITY) --no-password --insecure -f
 	chmod 400 $(CERT_DIR)/server.key $(CERT_DIR)/server.crt
 
 client: $(CERT_DIR)/client.crt
 
 $(CERT_DIR)/client.crt: $(CERT_DIR)/ca.crt
-	openssl req -newkey rsa:4096 -nodes \
-		-keyout $(CERT_DIR)/client.key -out $(CERT_DIR)/client.csr \
-		-subj "/CN=lclstream-client"
-	openssl x509 -req -in $(CERT_DIR)/client.csr \
-		-CA $(CERT_DIR)/ca.crt -CAkey $(CERT_DIR)/ca.key -CAcreateserial \
-		-days 2 -out $(CERT_DIR)/client.crt
-	rm -f $(CERT_DIR)/client.csr
+	step certificate create "lclstream-client" $(CERT_DIR)/client.crt $(CERT_DIR)/client.key \
+		--profile leaf --ca $(CERT_DIR)/ca.crt --ca-key $(CERT_DIR)/ca.key \
+		--not-after $(CERT_VALIDITY) --no-password --insecure -f
 	chmod 400 $(CERT_DIR)/client.key $(CERT_DIR)/client.crt
 
+check-certs:
+	@for f in ca.crt server.crt server.key client.crt client.key; do \
+		test -f $(CERT_DIR)/$$f || { echo "missing $(CERT_DIR)/$$f (run: make certs)"; exit 1; }; \
+	done
+
+verify-certs: check-certs
+	step certificate verify $(CERT_DIR)/server.crt --roots $(CERT_DIR)/ca.crt
+	step certificate verify $(CERT_DIR)/client.crt --roots $(CERT_DIR)/ca.crt
+
+vault-push: verify-certs
+	@if vault token lookup -address=$(VAULT_ADDR) > /dev/null 2>&1; then \
+		echo "using existing token"; \
+	else \
+		echo "Note: you must log in to vault"; \
+		vault login -method=oidc -address=$(VAULT_ADDR); \
+	fi
+	@echo "Pushing certs to $(VAULT_ADDR) secret/$(VAULT_SECRET_PATH)"
+	@vault kv put -address=$(VAULT_ADDR) -mount=secret $(VAULT_SECRET_PATH) \
+		ca.crt=@$(CERT_DIR)/ca.crt \
+		server.crt=@$(CERT_DIR)/server.crt \
+		server.key=@$(CERT_DIR)/server.key \
+		client.crt=@$(CERT_DIR)/client.crt \
+		client.key=@$(CERT_DIR)/client.key
+	@echo "Done. ca.key was NOT pushed (stays local only)."
+
+vault-pull:
+	@if vault token lookup -address=$(VAULT_ADDR) > /dev/null 2>&1; then \
+		echo "using existing token"; \
+	else \
+		echo "Note: you must log in to vault"; \
+		vault login -method=oidc -address=$(VAULT_ADDR); \
+	fi
+	@echo "Pulling certs from $(VAULT_ADDR) secret/$(VAULT_SECRET_PATH)"
+	@mkdir -p $(CERT_DIR)
+	@set -e; \
+		tmp_dir=$$(mktemp -d "$(CERT_DIR)/.vault-pull.XXXXXX"); \
+		trap 'rm -rf "$$tmp_dir"' EXIT; \
+		vault kv get -address=$(VAULT_ADDR) -field=ca.crt -mount=secret $(VAULT_SECRET_PATH) > "$$tmp_dir/ca.crt"; \
+		vault kv get -address=$(VAULT_ADDR) -field=server.crt -mount=secret $(VAULT_SECRET_PATH) > "$$tmp_dir/server.crt"; \
+		vault kv get -address=$(VAULT_ADDR) -field=server.key -mount=secret $(VAULT_SECRET_PATH) > "$$tmp_dir/server.key"; \
+		vault kv get -address=$(VAULT_ADDR) -field=client.crt -mount=secret $(VAULT_SECRET_PATH) > "$$tmp_dir/client.crt"; \
+		vault kv get -address=$(VAULT_ADDR) -field=client.key -mount=secret $(VAULT_SECRET_PATH) > "$$tmp_dir/client.key"; \
+		chmod 400 "$$tmp_dir"/ca.crt "$$tmp_dir"/server.crt "$$tmp_dir"/server.key "$$tmp_dir"/client.crt "$$tmp_dir"/client.key; \
+		mv -f "$$tmp_dir"/ca.crt "$$tmp_dir"/server.crt "$$tmp_dir"/server.key "$$tmp_dir"/client.crt "$$tmp_dir"/client.key $(CERT_DIR)/; \
+		trap - EXIT; \
+		rmdir "$$tmp_dir"
+	@echo "Done. ca.key was NOT pulled; keep it only on the cert builder."
+
 clean-certs:
 	rm -rf $(CERT_DIR)

src/fastcache_api/config.py

@@ -2,7 +2,7 @@
 from typing import Annotated, Any, Literal
 
 from pydantic import BeforeValidator, model_validator
-from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
 
 
 def parse_comma_list(v: Any) -> list[str] | str:
@@ -37,13 +37,17 @@ class Settings(BaseSettings):
     SQLITE_PATH: Path = Path("fastcache_api.sqlite")
 
     # Verified email addresses allowed to access the service.
-    EXPECTED_USERS: Annotated[list[str], BeforeValidator(parse_comma_list)] = []
+    EXPECTED_USERS: Annotated[
+        list[str], BeforeValidator(parse_comma_list), NoDecode
+    ] = []
 
     # OIDC token validation. Identity provider is configured via environment;
     # no provider URL is baked into source.
     OIDC_ISSUER_URL: str
     OIDC_JWKS_URI: str | None = None
-    OIDC_AUDIENCES: Annotated[list[str], BeforeValidator(parse_comma_list)] = []
+    OIDC_AUDIENCES: Annotated[
+        list[str], BeforeValidator(parse_comma_list), NoDecode
+    ] = []
 
     CACHE_PORT_START: int = 30000
     CACHE_PORT_END: int = 30100
@@ -82,5 +86,3 @@ def tls_enabled(self) -> bool:
 
 
 settings = Settings()  # type: ignore
-
-print(settings)

systemd/fastcache-api.service

@@ -0,0 +1,55 @@
+# fastcache_api systemd user unit
+#
+# Install:
+#   mkdir -p ~/.config/systemd/user
+#   cp systemd/fastcache-api.service ~/.config/systemd/user/
+#   mkdir -p ~/.config/fastcache_api
+#   cp systemd/fastcache_api.env.example ~/.config/fastcache_api/fastcache_api.env
+#   $EDITOR ~/.config/fastcache_api/fastcache_api.env
+#   systemctl --user daemon-reload
+#   systemctl --user enable --now fastcache-api.service
+#
+# This is a *user* unit, so it normally only runs while you're logged in.
+# To keep it running across logouts (e.g. on an S3DF login node), enable
+# lingering once:
+#   loginctl enable-linger "$USER"
+#
+# Certs are NOT fetched automatically here: Vault OIDC login needs a browser,
+# so an ExecStartPre that calls Vault would block startup/restarts whenever
+# the cached token (~/.vault-token) has expired. Instead, point SSL_CERTFILE/
+# SSL_KEYFILE/SSL_CA_CERTS (in the env file) at wherever your certs already
+# live -- e.g. this repo's certs/ dir after `make certs` -- and refresh them
+# yourself periodically (`make certs && make vault-push`, or run
+# systemd/fetch-certs.sh by hand), then `systemctl --user restart
+# fastcache-api.service` to pick up the new files.
+
+[Unit]
+Description=LCLStream fastcache API
+After=network-online.target
+Wants=network-online.target
+StartLimitIntervalSec=300
+StartLimitBurst=5
+
+[Service]
+Type=simple
+WorkingDirectory=%h/gits/lclstream/fastcache_api
+EnvironmentFile=%h/.config/fastcache_api/fastcache_api.env
+# systemd 239 here predates StandardOutput=append: (needs v240+), so redirect
+# via a shell instead. Logs (uvicorn + our own logging) land in a real file
+# under .local/state instead of only the (often unreadable) user journal.
+ExecStart=/bin/sh -c 'exec "%h/gits/lclstream/fastcache_api/.venv/bin/fastcache-api" >> "%h/.local/state/fastcache_api/fastcache-api.log" 2>&1'
+Restart=on-failure
+RestartSec=5
+TimeoutStopSec=30
+
+# Light hardening. Avoid ProtectHome/ProtectSystem here: the app reads/writes
+# SQLITE_PATH, the certs, the API log, and request-supplied cache log dirs.
+NoNewPrivileges=true
+PrivateTmp=true
+# Owner-only by default for anything the process creates (sqlite db, API log,
+# and per-cache config.json/log files next to each request's log_path).
+# Directories made ahead of time should also be chmod 700.
+UMask=0077
+
+[Install]
+WantedBy=default.target

systemd/fastcache_api.env.example

@@ -0,0 +1,51 @@
+# fastcache_api environment for the systemd user unit.
+#
+# Copy to ~/.config/fastcache_api/fastcache_api.env and adjust.
+# NOTE: unlike the unit file, systemd does NOT expand "%h" (or $HOME) inside
+# an EnvironmentFile, so replace every /home/YOUR_USER below with your actual
+# home directory, e.g. /sdf/home/s/swelborn.
+#
+# This file (and the directories it points at: SQLITE_PATH, cert paths, and
+# wherever fastcache_api.env itself lives) can contain user identities and
+# internal paths, so lock them down:
+#   chmod 700 ~/.config/fastcache_api ~/.local/share/fastcache_api \
+#             ~/.local/state/fastcache_api
+#   chmod 600 ~/.config/fastcache_api/fastcache_api.env
+# The unit also sets UMask=0077, so anything the process itself creates
+# (the sqlite db, API log, and per-cache config.json/log files) is owner-only
+# by default.
+
+# --- General ---------------------------------------------------------------
+ENVIRONMENT=staging
+PROJECT_NAME=lclstream_fastcache_api
+API_V1_STR=/api/v1
+
+# --- Server / TLS ------------------------------------------------------------
+HOST=0.0.0.0
+PORT=29000
+
+# Point these at wherever your certs live -- e.g. this repo's certs/ dir 
+# after `make certs` -- and refresh manually (`make certs && make vault-push`, or run
+# systemd/fetch-certs.sh by hand), then `systemctl --user restart
+# fastcache-api.service`.
+SSL_CERTFILE=/home/YOUR_USER/.config/fastcache_api/certs/server.crt
+SSL_KEYFILE=/home/YOUR_USER/.config/fastcache_api/certs/server.key
+SSL_CA_CERTS=/home/YOUR_USER/.config/fastcache_api/certs/ca.crt
+REQUIRE_CLIENT_CERT=true
+
+# --- Database --------------------------------------------------------------
+# XDG_DATA_HOME-style location: this is persistent app data, not config.
+SQLITE_PATH=/home/YOUR_USER/.local/share/fastcache_api/fastcache_api.sqlite
+
+# --- Authentication / Authorization -----------------------------------------
+# REQUIRED: verified email address(es) allowed to use the service, e.g.
+# you@slac.stanford.edu (comma-separated for more than one).
+EXPECTED_USERS=
+OIDC_ISSUER_URL=
+OIDC_JWKS_URI=
+OIDC_AUDIENCES=
+
+# --- Cache process management ------------------------------------------------
+FASTCACHE_BINARY=lclstream-fastcache
+SHUTDOWN_GRACE_SECONDS=5.0
+CACHE_POLL_SECONDS=5.0