-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support for local kind cluster (#190)
Add local kind support. [See Diagram](https://raw.githubusercontent.com/substratusai/substratus/5c813780ccc3d139dd12586869dd85e3e9898fab/docs/diagrams/arch-kind.excalidraw.png) Fixes #152
- Loading branch information
Showing
45 changed files
with
2,691 additions
and
1,723 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| # Start from the latest go base image | ||
| FROM golang:1.19 AS builder | ||
| ARG TARGETOS=linux | ||
| ARG TARGETARCH=amd64 | ||
|
|
||
| WORKDIR /workspace | ||
| COPY go.mod go.sum ./ | ||
| RUN go mod download | ||
|
|
||
| COPY cmd/sci-kind/main.go cmd/sci-kind/main.go | ||
| COPY internal/ internal/ | ||
|
|
||
| # Build the app | ||
| RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ | ||
| go build -a -o main cmd/sci-kind/main.go | ||
|
|
||
| FROM gcr.io/distroless/static:nonroot | ||
| WORKDIR / | ||
|
|
||
| # Copy the Pre-built binary file from the previous stage | ||
| COPY --from=builder /workspace/main . | ||
| USER root | ||
| EXPOSE 10080 | ||
| EXPOSE 8080 | ||
|
|
||
| # run the executable | ||
| CMD ["/main"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "flag" | ||
| "fmt" | ||
| "log" | ||
| "net" | ||
| "net/http" | ||
|
|
||
| "github.com/substratusai/substratus/internal/sci" | ||
| scikind "github.com/substratusai/substratus/internal/sci/kind" | ||
| "google.golang.org/grpc" | ||
| "google.golang.org/grpc/health" | ||
| hv1 "google.golang.org/grpc/health/grpc_health_v1" | ||
| ) | ||
|
|
||
| func main() { | ||
| var cfg struct { | ||
| port int | ||
| signedURLPort int | ||
| hostSignedURLAddress string | ||
| } | ||
| flag.IntVar(&cfg.port, "port", 10080, "port number to listen on") | ||
| flag.IntVar(&cfg.signedURLPort, "signed-url-port", 8080, "port to listen for signed url traffic") | ||
| flag.StringVar(&cfg.hostSignedURLAddress, "host-signed-url-address", "http://localhost:30080", | ||
| "host address that port forwards to the signed url port within the cluster. this should be set in kind config.yaml.") | ||
| flag.Parse() | ||
|
|
||
| s := &scikind.Server{ | ||
| SignedURLAddress: cfg.hostSignedURLAddress, | ||
| } | ||
| signedURLServer := &http.Server{ | ||
| Addr: fmt.Sprintf(":%v", cfg.signedURLPort), | ||
| Handler: s, | ||
| } | ||
| go func() { | ||
| log.Printf("Listening for signed URL traffic on address: %v", cfg.signedURLPort) | ||
| log.Fatal(signedURLServer.ListenAndServe()) | ||
| }() | ||
|
|
||
| gs := grpc.NewServer() | ||
| sci.RegisterControllerServer(gs, s) | ||
|
|
||
| // Setup Health Check | ||
| hs := health.NewServer() | ||
| hs.SetServingStatus("", hv1.HealthCheckResponse_SERVING) | ||
| hv1.RegisterHealthServer(gs, hs) | ||
|
|
||
| addr := fmt.Sprintf(":%v", cfg.port) | ||
| log.Printf("Listening for gRPC traffic on address: %v", addr) | ||
| lis, err := net.Listen("tcp", addr) | ||
| if err != nil { | ||
| log.Fatalf("failed to listen: %v", err) | ||
| } | ||
|
|
||
| if err := gs.Serve(lis); err != nil { | ||
| log.Fatalf("failed to serve: %v", err) | ||
| } | ||
| } |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| # This patch inject a sidecar container which is a HTTP proxy for the | ||
| # controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. | ||
| apiVersion: apps/v1 | ||
| kind: Deployment | ||
| metadata: | ||
| name: controller-manager | ||
| namespace: system | ||
| spec: | ||
| template: | ||
| spec: | ||
| affinity: | ||
| nodeAffinity: | ||
| requiredDuringSchedulingIgnoredDuringExecution: | ||
| nodeSelectorTerms: | ||
| - matchExpressions: | ||
| - key: kubernetes.io/arch | ||
| operator: In | ||
| values: | ||
| - amd64 | ||
| - arm64 | ||
| - ppc64le | ||
| - s390x | ||
| - key: kubernetes.io/os | ||
| operator: In | ||
| values: | ||
| - linux | ||
| containers: | ||
| - name: kube-rbac-proxy | ||
| securityContext: | ||
| allowPrivilegeEscalation: false | ||
| capabilities: | ||
| drop: | ||
| - "ALL" | ||
| image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1 | ||
| args: | ||
| - "--secure-listen-address=0.0.0.0:8443" | ||
| - "--upstream=http://127.0.0.1:8080/" | ||
| - "--logtostderr=true" | ||
| - "--v=0" | ||
| ports: | ||
| - containerPort: 8443 | ||
| protocol: TCP | ||
| name: https | ||
| resources: | ||
| limits: | ||
| cpu: 500m | ||
| memory: 128Mi | ||
| requests: | ||
| cpu: 5m | ||
| memory: 64Mi | ||
| - name: manager | ||
| envFrom: | ||
| - configMapRef: | ||
| name: system | ||
| args: | ||
| - "--health-probe-bind-address=:8081" | ||
| - "--metrics-bind-address=127.0.0.1:8080" | ||
| - "--leader-elect" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| apiVersion: v1 | ||
| kind: ConfigMap | ||
| metadata: | ||
| name: system | ||
| namespace: substratus | ||
| data: | ||
| CLOUD: kind | ||
| CLUSTER_NAME: substratus | ||
| PRINCIPAL: unused |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| # Adds namespace to all resources. | ||
| namespace: substratus | ||
|
|
||
| # Labels to add to all resources and selectors. | ||
| #labels: | ||
| #- includeSelectors: true | ||
| # pairs: | ||
| # someName: someValue | ||
|
|
||
| resources: | ||
| - ./namespace.yaml | ||
| - ./config.yaml | ||
| - ../crd | ||
| - ../rbac | ||
| - ../manager | ||
| - ../registry-kind | ||
| - ../sci-kind | ||
| # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in | ||
| # crd/kustomization.yaml | ||
| #- ../webhook | ||
| # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. | ||
| #- ../certmanager | ||
| # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. | ||
| #- ../prometheus | ||
|
|
||
| # Protect the /metrics endpoint by putting it behind auth. | ||
| # If you want your controller-manager to expose the /metrics | ||
| # endpoint w/o any authn/z, please comment the following line. | ||
| patches: | ||
| - path: manager_patch.yaml | ||
| apiVersion: kustomize.config.k8s.io/v1beta1 | ||
| kind: Kustomization |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| apiVersion: v1 | ||
| kind: Namespace | ||
| metadata: | ||
| name: substratus | ||
| labels: | ||
| control-plane: controller-manager | ||
| app.kubernetes.io/name: namespace | ||
| app.kubernetes.io/instance: system | ||
| app.kubernetes.io/component: manager | ||
| app.kubernetes.io/created-by: substratus | ||
| app.kubernetes.io/part-of: substratus | ||
| app.kubernetes.io/managed-by: kustomize |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| apiVersion: v1 | ||
| kind: ConfigMap | ||
| metadata: | ||
| name: configure-cri | ||
| data: | ||
| configure-cri.sh: | | ||
| #!/usr/bin/env bash | ||
| set -x | ||
| # Exit on non-existant variable. | ||
| set -u | ||
| # Exit on error. | ||
| set -e | ||
| export IMAGE_REGISTRY=$REGISTRY_PORT_5000_TCP_ADDR:5000 | ||
| if ! grep -q $IMAGE_REGISTRY /mnt/etc/containerd/config.toml; then | ||
| containerd_version=$(nsenter --target 1 --mount bash -c "containerd --version | awk '{ print substr(\$3,0,4) }'") | ||
| if [ "$containerd_version" = "1.3." ] || [ "$containerd_version" = "1.4." ]; then | ||
| cat <<EOF >> /mnt/etc/containerd/config.toml | ||
| [plugins.cri.registry.configs."$IMAGE_REGISTRY"] | ||
| endpoint = ["http://$IMAGE_REGISTRY"] | ||
| EOF | ||
| else | ||
| # Correct config for containerd 1.5 and above | ||
| cat <<EOF >> /mnt/etc/containerd/config.toml | ||
| [plugins."io.containerd.grpc.v1.cri".registry.mirrors."$IMAGE_REGISTRY"] | ||
| endpoint = ["http://$IMAGE_REGISTRY"] | ||
| EOF | ||
| fi | ||
| nsenter --target 1 --mount bash -c "systemctl is-active --quiet containerd && echo 'Restarting containerd' && systemctl restart containerd" | ||
| # Wait for containerd to be ready so that skaffold doesn't fail. | ||
| nsenter --target 1 --mount bash -c "while ! ctr -n k8s.io containers ls; do sleep 1; done" | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| apiVersion: apps/v1 | ||
| kind: DaemonSet | ||
| metadata: | ||
| name: configure-cri | ||
| labels: | ||
| app: configure-cri | ||
| spec: | ||
| selector: | ||
| matchLabels: | ||
| app: configure-cri | ||
| template: | ||
| metadata: | ||
| labels: | ||
| app: configure-cri | ||
| spec: | ||
| hostPID: true | ||
| initContainers: | ||
| - name: configure-cri | ||
| image: ubuntu:22.04 | ||
| command: ["/scripts/configure-cri.sh"] | ||
| volumeMounts: | ||
| - name: etc | ||
| mountPath: "/mnt/etc" | ||
| - mountPath: /scripts | ||
| name: scripts | ||
| securityContext: | ||
| privileged: true | ||
| volumes: | ||
| - name: etc | ||
| hostPath: | ||
| path: /etc | ||
| - name: scripts | ||
| configMap: | ||
| name: configure-cri | ||
| defaultMode: 0744 | ||
| containers: | ||
| - name: pause | ||
| image: gcr.io/google_containers/pause | ||
| tolerations: | ||
| - effect: NoSchedule | ||
| operator: Exists |
Oops, something went wrong.