From c7456bd977583545dace6ea65b56dbb0a48be71c Mon Sep 17 00:00:00 2001 From: Nick Cao Date: Fri, 7 Mar 2025 13:40:47 -0500 Subject: [PATCH 1/2] Generate certificates with easy-rsa --- cmd/main.go | 3 +- .../templates/_endpoints.tpl | 4 +-- .../templates/controller-deployment.yaml | 24 ++++++-------- .../templates/secrets-job.yaml | 31 +++++++++++++++++-- internal/service/controller_service.go | 7 +---- internal/service/endpoints.go | 22 ------------- internal/service/router_service.go | 7 +---- internal/service/selfsigned.go | 28 +++++++++++++---- 8 files changed, 65 insertions(+), 61 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index d05f2920..914dcfad 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -21,7 +21,6 @@ import ( "crypto/tls" "encoding/pem" "flag" - "net" "os" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -135,7 +134,7 @@ func main() { os.Exit(1) } - oidcCert, err := service.NewSelfSignedCertificate("jumpstarter oidc", []string{"localhost"}, []net.IP{}) + oidcCert, err := service.NewSelfSignedLocalhostCertificate() if err != nil { setupLog.Error(err, "unable to generate certificate for internal oidc provider") os.Exit(1) diff --git a/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/_endpoints.tpl b/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/_endpoints.tpl index a925e71c..d3223588 100644 --- a/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/_endpoints.tpl +++ b/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/_endpoints.tpl @@ -1,2 +1,2 @@ -{{- define "router.endpoint" }}{{ if .Values.grpc.routerHostname }}{{ .Values.grpc.routerHostname }}{{ else }}router.{{ .Values.global.baseDomain | required "grpc.routerHostname or global.baseDomain must be set"}}{{ end }}{{- end }} -{{- define "controller.endpoint" }}{{ if .Values.grpc.hostname }}{{ .Values.grpc.hostname }}{{ else }}grpc.{{ .Values.global.baseDomain | required "grpc.hostname or global.baseDomain must be set"}}{{ end }}{{- end }} \ No newline at end of file +{{- define "controller.endpoint" }}{{ if .Values.grpc.endpoint }}{{ .Values.grpc.endpoint }}{{ else if .Values.hostname }}{{ .Values.hostname }}:{{ .Values.grpc.tls.port }}{{ else }}grpc.{{ .Values.global.baseDomain | required "grpc.endpoint or hostname or global.baseDomain must be set" }}:{{ .Values.grpc.tls.port }}{{ end }}{{- end }} +{{- define "router.endpoint" }}{{ if .Values.grpc.routerEndpoint }}{{ .Values.grpc.routerEndpoint }}{{ else if .Values.routerHostname }}{{ .Values.routerHostname }}:{{ .Values.grpc.tls.port }}{{ else }}router.{{ .Values.global.baseDomain | required "grpc.routerEndpoint or routerHostname or global.baseDomain must be set" }}:{{ .Values.grpc.tls.port }}{{ end }}{{- end }} diff --git a/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/controller-deployment.yaml b/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/controller-deployment.yaml index 6fb8d932..7b17b8df 100644 --- a/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/controller-deployment.yaml +++ b/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/controller-deployment.yaml @@ -48,6 +48,10 @@ spec: runAsNonRoot: true seccompProfile: type: RuntimeDefault + volumes: + - name: tls + secret: + secretName: jumpstarter-tls containers: - args: - --leader-elect @@ -55,21 +59,9 @@ spec: - -metrics-bind-address=:8080 env: - name: GRPC_ENDPOINT - {{ if .Values.grpc.endpoint }} - value : {{ .Values.grpc.endpoint }} - {{ else if .Values.hostname }} - value: {{ .Values.hostname }}:{{ .Values.grpc.tls.port }} - {{ else }} - value: grpc.{{ .Values.global.baseDomain }}:{{ .Values.grpc.tls.port }} - {{ end }} + value: {{ template "controller.endpoint" . }} - name: GRPC_ROUTER_ENDPOINT - {{ if .Values.grpc.routerEndpoint }} - value: {{ .Values.grpc.routerEndpoint }} - {{ else if .Values.routerHostname }} - value: {{ .Values.routerHostname }}:{{ .Values.grpc.tls.port }} - {{ else }} - value: router.{{ .Values.global.baseDomain }}:{{ .Values.grpc.tls.port }} - {{ end }} + value: {{ template "router.endpoint" . }} - name: CONTROLLER_KEY valueFrom: secretKeyRef: @@ -84,6 +76,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + volumeMounts: + - name: tls + mountPath: "/etc/jumpstarter/tls" + readOnly: true image: {{ .Values.image }}:{{ default .Chart.AppVersion .Values.tag }} imagePullPolicy: {{ .Values.imagePullPolicy }} diff --git a/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/secrets-job.yaml b/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/secrets-job.yaml index a860ee58..864cabea 100644 --- a/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/secrets-job.yaml +++ b/deploy/helm/jumpstarter/charts/jumpstarter-controller/templates/secrets-job.yaml @@ -12,18 +12,43 @@ spec: metadata: name: jumpstarter-secrets spec: + ttlSecondsAfterFinished: 600 serviceAccountName: controller-manager containers: - name: jumpstarter-secrets - image: quay.io/openshift/origin-cli + image: quay.io/jumpstarter-dev/jumpstarter-utils:latest command: - /bin/sh - -c - | set -e + + CONTROLLER_HOSTNAME=$(trurl {{ template "controller.endpoint" . }} --get "{host}") + ROUTER_HOSTNAME=$(trurl {{ template "router.endpoint" . }} --get "{host}") + + export EASYRSA_BATCH=1 + export EASYRSA_PKI=pki + /usr/share/easy-rsa/3/easyrsa init-pki + /usr/share/easy-rsa/3/easyrsa --no-pass build-ca + /usr/share/easy-rsa/3/easyrsa --no-pass \ + --subject-alt-name=DNS:"${CONTROLLER_HOSTNAME}" \ + --subject-alt-name=DNS:"${ROUTER_HOSTNAME}" \ + build-server-full "${CONTROLLER_HOSTNAME}" + + if ! kubectl get secret jumpstarter-tls-ca -n {{ $namespace }} >/dev/null 2>&1; then + kubectl create secret tls jumpstarter-tls-ca -n={{ $namespace }} \ + --cert=pki/ca.crt --key=pki/private/ca.key + fi + + if ! kubectl get secret jumpstarter-tls -n {{ $namespace }} >/dev/null 2>&1; then + kubectl create secret tls jumpstarter-tls -n={{ $namespace }} \ + --cert=pki/issued/"${CONTROLLER_HOSTNAME}".crt \ + --key=pki/private/"${CONTROLLER_HOSTNAME}".key + fi + {{- range $name := tuple "jumpstarter-router-secret" "jumpstarter-controller-secret" }} - if ! oc get secret {{ $name }} -n {{ $namespace }} >/dev/null 2>&1; then - oc create secret generic {{ $name }} -n={{ $namespace }} \ + if ! kubectl get secret {{ $name }} -n {{ $namespace }} >/dev/null 2>&1; then + kubectl create secret generic {{ $name }} -n={{ $namespace }} \ --from-literal=key="$(openssl rand -hex 32)" fi {{- end }} diff --git a/internal/service/controller_service.go b/internal/service/controller_service.go index 3ba44ef4..be609e20 100644 --- a/internal/service/controller_service.go +++ b/internal/service/controller_service.go @@ -699,12 +699,7 @@ func (s *ControllerService) ListLeases( func (s *ControllerService) Start(ctx context.Context) error { logger := log.FromContext(ctx) - dnsnames, ipaddresses, err := endpointToSAN(controllerEndpoint()) - if err != nil { - return err - } - - cert, err := NewSelfSignedCertificate("jumpstarter controller", dnsnames, ipaddresses) + cert, err := LoadCertificate("/etc/jumpstarter/tls/") if err != nil { return err } diff --git a/internal/service/endpoints.go b/internal/service/endpoints.go index 93d0cc37..bc111a36 100644 --- a/internal/service/endpoints.go +++ b/internal/service/endpoints.go @@ -1,18 +1,9 @@ package service import ( - "net" "os" ) -func controllerEndpoint() string { - ep := os.Getenv("GRPC_ENDPOINT") - if ep == "" { - return "localhost:8082" - } - return ep -} - func routerEndpoint() string { ep := os.Getenv("GRPC_ROUTER_ENDPOINT") if ep == "" { @@ -20,16 +11,3 @@ func routerEndpoint() string { } return ep } - -func endpointToSAN(endpoint string) ([]string, []net.IP, error) { - host, _, err := net.SplitHostPort(endpoint) - if err != nil { - return nil, nil, err - } - ip := net.ParseIP(host) - if ip != nil { - return []string{}, []net.IP{ip}, nil - } else { - return []string{host}, []net.IP{}, nil - } -} diff --git a/internal/service/router_service.go b/internal/service/router_service.go index 344b4374..6d1aeaad 100644 --- a/internal/service/router_service.go +++ b/internal/service/router_service.go @@ -112,12 +112,7 @@ func (s *RouterService) Stream(stream pb.RouterService_StreamServer) error { func (s *RouterService) Start(ctx context.Context) error { log := log.FromContext(ctx) - dnsnames, ipaddresses, err := endpointToSAN(routerEndpoint()) - if err != nil { - return err - } - - cert, err := NewSelfSignedCertificate("jumpstarter router", dnsnames, ipaddresses) + cert, err := LoadCertificate("/etc/jumpstarter/tls/") if err != nil { return err } diff --git a/internal/service/selfsigned.go b/internal/service/selfsigned.go index 7dd926bb..0e868768 100644 --- a/internal/service/selfsigned.go +++ b/internal/service/selfsigned.go @@ -7,20 +7,36 @@ import ( "crypto/x509" "crypto/x509/pkix" "math/big" - "net" + "os" + "path" "time" ) -func NewSelfSignedCertificate(commonName string, dnsnames []string, ipaddresses []net.IP) (*tls.Certificate, error) { +func LoadCertificate(base string) (*tls.Certificate, error) { + crt, err := os.ReadFile(path.Join(base, "tls.crt")) + if err != nil { + return nil, err + } + key, err := os.ReadFile(path.Join(base, "tls.key")) + if err != nil { + return nil, err + } + cert, err := tls.X509KeyPair(crt, key) + if err != nil { + return nil, err + } + return &cert, nil +} + +func NewSelfSignedLocalhostCertificate() (*tls.Certificate, error) { template := x509.Certificate{ SerialNumber: big.NewInt(1), - Subject: pkix.Name{CommonName: commonName}, - Issuer: pkix.Name{CommonName: commonName}, + Subject: pkix.Name{CommonName: "localhost"}, + Issuer: pkix.Name{CommonName: "localhost"}, NotBefore: time.Now(), NotAfter: time.Now().Add(365 * 24 * time.Hour), BasicConstraintsValid: true, - DNSNames: dnsnames, - IPAddresses: ipaddresses, + DNSNames: []string{"localhost"}, } priv, err := rsa.GenerateKey(rand.Reader, 2048) From bd8d3b6fa518cbfc360aebb72e729ab105fb9d21 Mon Sep 17 00:00:00 2001 From: Nick Cao Date: Fri, 7 Mar 2025 15:45:05 -0500 Subject: [PATCH 2/2] Run kind and e2e ci on arm --- .github/workflows/e2e.yaml | 7 ++++++- .github/workflows/pr-kind.yaml | 25 +++++++++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index e57de758..2415df49 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -7,7 +7,12 @@ on: jobs: e2e-tests: - runs-on: ubuntu-latest + strategy: + matrix: + os: + - ubuntu-24.04 + - ubuntu-24.04-arm + runs-on: ${{ matrix.os }} steps: - uses: jumpstarter-dev/jumpstarter-e2e@main with: diff --git a/.github/workflows/pr-kind.yaml b/.github/workflows/pr-kind.yaml index 2d1e06d8..8f930b8e 100644 --- a/.github/workflows/pr-kind.yaml +++ b/.github/workflows/pr-kind.yaml @@ -4,15 +4,32 @@ on: pull_request: branches: - main - jobs: - deploy-kind: - runs-on: ubuntu-latest + deploy-kind-matrix: + strategy: + matrix: + os: + - ubuntu-24.04 + - ubuntu-24.04-arm + runs-on: ${{ matrix.os }} steps: - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Run make deploy run: make deploy + # https://github.com/orgs/community/discussions/26822 + deploy-kind: + runs-on: ubuntu-latest + needs: + - deploy-kind-matrix + if: ${{ always() }} + steps: + - run: exit 1 + if: >- + ${{ + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + || contains(needs.*.result, 'skipped') + }}