Implement team-specific multitenant runtime wiring and standardize Kubernetes tests on kind

.github/workflows/ci.yml

@@ -13,32 +13,38 @@ permissions:
 
 jobs:
   lint:
+    name: lint
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - uses: extractions/setup-just@f8a3cce218d9f83db3a2ecd90e41ac3de6cdfd9b # v3
       - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
       - uses: golangci/golangci-lint-action@9fae48acfc02a90574d7c304a1758ef9895495fa # v7.0.1
         with:
           version: v2.8.0
 
-  unit-tests:
+  test-unit:
+    name: test-unit
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+      - name: Set up just
+        uses: extractions/setup-just@f8a3cce218d9f83db3a2ecd90e41ac3de6cdfd9b # v3
 
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
 
       - name: Run unit tests
-        run: go test -v . ./server/... ./transpiler/...
+        run: just test-unit
 
-  integration-tests:
-    needs: unit-tests
+  test-standalone-integration:
+    name: test-standalone-integration
+    needs: test-unit
     runs-on: ubuntu-latest
 
     services:
@@ -75,6 +81,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+      - name: Set up just
+        uses: extractions/setup-just@f8a3cce218d9f83db3a2ecd90e41ac3de6cdfd9b # v3
 
       - name: Set up Go
         uses: actions/setup-go@v5
@@ -106,21 +114,48 @@ jobs:
           /tmp/mc alias set minio http://localhost:39000 minioadmin minioadmin
           /tmp/mc mb minio/ducklake --ignore-existing
 
-      - name: Run integration tests
-        run: go test -v ./tests/integration/...
+      - name: Run standalone integration tests
+        run: just test-standalone-integration
 
-  controlplane-tests:
-    needs: unit-tests
+  test-control-plane:
+    name: test-control-plane
+    needs: test-unit
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+      - name: Set up just
+        uses: extractions/setup-just@f8a3cce218d9f83db3a2ecd90e41ac3de6cdfd9b # v3
 
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
 
-      - name: Run control plane tests
-        run: go test -v -timeout 300s ./tests/controlplane/...
+      - name: Run control-plane tests
+        run: just test-control-plane
+
+  test-k8s-integration:
+    name: test-k8s-integration
+    needs: test-unit
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      DUCKGRES_K8S_TEST_SETUP: kind
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Set up just
+        uses: extractions/setup-just@f8a3cce218d9f83db3a2ecd90e41ac3de6cdfd9b # v3
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Set up kind
+        uses: engineerd/setup-kind@v0.6.2
+        with:
+          skipClusterCreation: true
+      - name: Run Kubernetes integration tests
+        run: just test-k8s-integration

README.md

@@ -597,7 +597,7 @@ kill -USR2 <control-plane-pid>
 
 ### Remote Worker Backend
 
-In Kubernetes environments, the control plane can spawn worker pods instead of local processes using `--worker-backend remote`. The control plane creates worker pods via the Kubernetes API, communicates with them over gRPC (Arrow Flight SQL), and uses owner references for automatic garbage collection when the control plane pod is deleted.
+In Kubernetes environments, the control plane can spawn worker pods instead of local processes using `--worker-backend remote`. That mode is now the multitenant config-store path: `--worker-backend remote` requires `--config-store`, and the control plane creates per-team worker pods via the Kubernetes API and routes queries to them over gRPC (Arrow Flight SQL).
 
 ```bash
 # Build with Kubernetes support
@@ -607,9 +607,11 @@ docker build --build-arg BUILD_TAGS=kubernetes -t duckgres:latest .
 kubectl apply -f k8s/
 ```
 
-See [`k8s/README.md`](k8s/README.md) for the full architecture, configuration reference, manifest details, and the default local OrbStack config-store workflow via `just run-multitenant-local`.
+See [`k8s/README.md`](k8s/README.md) for the full architecture, configuration reference, manifest layout, and the default local `kind` multitenant workflow via `just run-multitenant-kind`. OrbStack remains available as an optional alternate local workflow.
 
-On the multi-tenant path, the config store now keeps per-team managed-warehouse metadata in addition to team/user auth and limits. That team-scoped contract is intended to become the source of truth for the tenant warehouse DB, the tenant DuckLake metadata store (which may live on shared Aurora or a dedicated RDS instance), object-store settings, worker identity, secret references, and provisioning state. The older config-store `DuckLakeConfig` singleton remains only as a legacy cluster-wide setting and should not be treated as authoritative for multi-tenant runtime wiring.
+On the multi-tenant path, the config store now keeps per-team managed-warehouse metadata in addition to team/user auth and limits. That team-scoped contract is intended to become the source of truth for the tenant warehouse DB, the tenant DuckLake metadata store (which may live on shared Aurora or a dedicated RDS instance), object-store settings, worker identity, secret references, and provisioning state.
+
+Managed-warehouse teams now fail closed unless provisioning has produced a valid team-scoped `runtime_config` Secret reference. The multitenant worker path no longer falls back to a shared worker runtime `ConfigMap` for DuckLake / warehouse / S3 behavior.
 
 Managed-warehouse contract notes:
 

ci_workflow_test.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"os"
+	"regexp"
+	"testing"
+)
+
+func TestCIWorkflowPinsSetupJustToFullSHA(t *testing.T) {
+	t.Parallel()
+
+	const (
+		workflowPath = ".github/workflows/ci.yml"
+		wantSHA      = "f8a3cce218d9f83db3a2ecd90e41ac3de6cdfd9b"
+	)
+
+	data, err := os.ReadFile(workflowPath)
+	if err != nil {
+		t.Fatalf("read %s: %v", workflowPath, err)
+	}
+
+	matches := regexp.MustCompile(`(?m)^\s*uses:\s*extractions/setup-just@([^\s#]+)`).FindAllSubmatch(data, -1)
+	if len(matches) == 0 {
+		t.Fatalf("expected at least one extractions/setup-just use in %s", workflowPath)
+	}
+
+	for _, match := range matches {
+		got := string(match[1])
+		if got != wantSHA {
+			t.Fatalf("expected extractions/setup-just to be pinned to %s, got %s", wantSHA, got)
+		}
+	}
+}

cmd/init-config-store/main.go

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/posthog/duckgres/controlplane/configstore"
+)
+
+func main() {
+	configStoreConn := flag.String("config-store", "postgres://duckgres:duckgres@127.0.0.1:5434/duckgres_config?sslmode=disable", "PostgreSQL config-store connection string")
+	flag.Parse()
+
+	if _, err := configstore.NewConfigStore(*configStoreConn, time.Hour); err != nil {
+		exitf("initialize config store: %v", err)
+	}
+}
+
+func exitf(format string, args ...any) {
+	_, _ = fmt.Fprintf(os.Stderr, format+"\n", args...)
+	os.Exit(1)
+}

cmd/render-multitenant-local-runtime/main.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/posthog/duckgres/controlplane"
+	"github.com/posthog/duckgres/controlplane/configstore"
+)
+
+func main() {
+	var (
+		configStoreConn      = flag.String("config-store", "postgres://duckgres:duckgres@127.0.0.1:5434/duckgres_config?sslmode=disable", "PostgreSQL config-store connection string")
+		teamName             = flag.String("team", "local", "Team name to render runtime artifacts for")
+		namespace            = flag.String("namespace", "duckgres", "Default Kubernetes namespace for generated artifacts")
+		outputPath           = flag.String("output", "", "Write the rendered manifest to this path instead of stdout")
+		dataDir              = flag.String("data-dir", "/data", "Worker data_dir value for the rendered duckgres.yaml")
+		extensionsCSV        = flag.String("extensions", "ducklake", "Comma-separated worker extensions to include in duckgres.yaml")
+		warehouseDBPassword  = flag.String("warehouse-db-password", "duckgres", "Warehouse database password used for local artifact rendering")
+		metadataStorePass    = flag.String("metadata-store-password", "ducklake", "Metadata-store password used for local artifact rendering")
+		s3AccessKey          = flag.String("s3-access-key", "minioadmin", "S3 access key used for local artifact rendering")
+		s3SecretKey          = flag.String("s3-secret-key", "minioadmin", "S3 secret key used for local artifact rendering")
+	)
+	flag.Parse()
+
+	store, err := configstore.NewConfigStore(*configStoreConn, time.Hour)
+	if err != nil {
+		exitf("connect config store: %v", err)
+	}
+
+	team := store.Snapshot().Teams[*teamName]
+	if team == nil {
+		exitf("team %q not found in config store", *teamName)
+	}
+
+	manifest, err := controlplane.BuildManagedWarehouseRuntimeArtifacts(team, controlplane.ManagedWarehouseSecretMaterial{
+		WarehouseDatabasePassword: *warehouseDBPassword,
+		MetadataStorePassword:     *metadataStorePass,
+		S3AccessKey:               *s3AccessKey,
+		S3SecretKey:               *s3SecretKey,
+	}, controlplane.ManagedWarehouseRuntimeArtifactOptions{
+		DefaultNamespace: *namespace,
+		DataDir:          *dataDir,
+		Extensions:       splitCSV(*extensionsCSV),
+	})
+	if err != nil {
+		exitf("render managed warehouse runtime artifacts: %v", err)
+	}
+
+	if *outputPath == "" {
+		if _, err := os.Stdout.Write(manifest); err != nil {
+			exitf("write manifest to stdout: %v", err)
+		}
+		return
+	}
+
+	if err := os.WriteFile(*outputPath, manifest, 0o644); err != nil {
+		exitf("write manifest: %v", err)
+	}
+}
+
+func splitCSV(value string) []string {
+	if value == "" {
+		return nil
+	}
+
+	var parts []string
+	for _, part := range strings.Split(value, ",") {
+		part = strings.TrimSpace(part)
+		if part != "" {
+			parts = append(parts, part)
+		}
+	}
+	return parts
+}
+
+func exitf(format string, args ...any) {
+	_, _ = fmt.Fprintf(os.Stderr, format+"\n", args...)
+	os.Exit(1)
+}

config_resolution.go

@@ -47,7 +47,6 @@ type configCLIInputs struct {
 	K8sControlPlaneID         string
 	K8sWorkerPort             int
 	K8sWorkerSecret           string
-	K8sWorkerConfigMap        string
 	K8sWorkerImagePullPolicy  string
 	K8sWorkerServiceAccount   string
 	QueryLog                  bool
@@ -64,7 +63,6 @@ type resolvedConfig struct {
 	K8sControlPlaneID    string
 	K8sWorkerPort        int
 	K8sWorkerSecret      string
-	K8sWorkerConfigMap   string
 	K8sWorkerImagePullPolicy string
 	K8sWorkerServiceAccount  string
 	ConfigStoreConn      string
@@ -117,7 +115,7 @@ func resolveEffectiveConfig(fileCfg *FileConfig, cli configCLIInputs, getenv fun
 	var workerBackend string
 	var k8sWorkerImage, k8sWorkerNamespace, k8sControlPlaneID string
 	var k8sWorkerPort int
-	var k8sWorkerSecret, k8sWorkerConfigMap, k8sWorkerImagePullPolicy, k8sWorkerServiceAccount string
+	var k8sWorkerSecret, k8sWorkerImagePullPolicy, k8sWorkerServiceAccount string
 	var configStoreConn string
 	var configPollInterval time.Duration
 	var adminToken string
@@ -349,9 +347,6 @@ func resolveEffectiveConfig(fileCfg *FileConfig, cli configCLIInputs, getenv fun
 		if fileCfg.K8s.WorkerSecret != "" {
 			k8sWorkerSecret = fileCfg.K8s.WorkerSecret
 		}
-		if fileCfg.K8s.WorkerConfigMap != "" {
-			k8sWorkerConfigMap = fileCfg.K8s.WorkerConfigMap
-		}
 		if fileCfg.K8s.WorkerImagePullPolicy != "" {
 			k8sWorkerImagePullPolicy = fileCfg.K8s.WorkerImagePullPolicy
 		}
@@ -575,9 +570,6 @@ func resolveEffectiveConfig(fileCfg *FileConfig, cli configCLIInputs, getenv fun
 	if v := getenv("DUCKGRES_K8S_WORKER_SECRET"); v != "" {
 		k8sWorkerSecret = v
 	}
-	if v := getenv("DUCKGRES_K8S_WORKER_CONFIGMAP"); v != "" {
-		k8sWorkerConfigMap = v
-	}
 	if v := getenv("DUCKGRES_K8S_WORKER_IMAGE_PULL_POLICY"); v != "" {
 		k8sWorkerImagePullPolicy = v
 	}
@@ -766,9 +758,6 @@ func resolveEffectiveConfig(fileCfg *FileConfig, cli configCLIInputs, getenv fun
 	if cli.Set["k8s-worker-secret"] {
 		k8sWorkerSecret = cli.K8sWorkerSecret
 	}
-	if cli.Set["k8s-worker-configmap"] {
-		k8sWorkerConfigMap = cli.K8sWorkerConfigMap
-	}
 	if cli.Set["k8s-worker-image-pull-policy"] {
 		k8sWorkerImagePullPolicy = cli.K8sWorkerImagePullPolicy
 	}
@@ -837,7 +826,6 @@ func resolveEffectiveConfig(fileCfg *FileConfig, cli configCLIInputs, getenv fun
 		K8sControlPlaneID:        k8sControlPlaneID,
 		K8sWorkerPort:            k8sWorkerPort,
 		K8sWorkerSecret:          k8sWorkerSecret,
-		K8sWorkerConfigMap:       k8sWorkerConfigMap,
 		K8sWorkerImagePullPolicy: k8sWorkerImagePullPolicy,
 		K8sWorkerServiceAccount:  k8sWorkerServiceAccount,
 		ConfigStoreConn:          configStoreConn,