WIP: one DNS per workspace

kcp-dev · Nov 1, 2022 · afde7e9 · afde7e9
1 parent 3583793
commit afde7e9
Show file tree

Hide file tree

Showing 10 changed files with 349 additions and 104 deletions.
diff --git a/pkg/cliplugins/workload/plugin/syncer.yaml b/pkg/cliplugins/workload/plugin/syncer.yaml
@@ -11,12 +11,6 @@ metadata:
   namespace: {{.Namespace}}
 ---
 apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{.DNSServiceAccount}}
-  namespace: {{.Namespace}}
----
-apiVersion: v1
 kind: Secret
 metadata:
   name: {{.ServiceAccount}}-token
@@ -39,6 +33,38 @@ rules:
   - "list"
   - "watch"
   - "delete"
+- apiGroups:  # TODO: use Role and RoleBinding
+    - ""
+  resources:
+    - serviceaccounts
+    - services
+  verbs:
+    - "create"
+    - "get"
+    - "list"
+    - "update"
+    - "delete"
+- apiGroups:
+    - "apps"
+  resources:
+    - deployments
+  verbs:
+    - "create"
+    - "get"
+    - "list"
+    - "update"
+    - "delete"
+- apiGroups:
+    - "rbac"
+  resources:
+    - roles
+    - rolebindings
+  verbs:
+    - "create"
+    - "get"
+    - "list"
+    - "update"
+    - "delete"
 - apiGroups:
   - "apiextensions.k8s.io"
   resources:
@@ -59,20 +85,6 @@ rules:
 {{- end}}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: {{.DNSClusterRole}}
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - configmaps
-    verbs:
-      - "get"
-      - "list"
-      - "watch"
----
-apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: {{.ClusterRoleBinding}}
@@ -85,19 +97,6 @@ subjects:
   name: {{.ServiceAccount}}
   namespace: {{.Namespace}}
 ---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: {{.DNSClusterRoleBinding}}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: {{.DNSClusterRole}}
-subjects:
-  - kind: ServiceAccount
-    name: {{.DNSServiceAccount}}
-    namespace: {{.Namespace}}
----
 apiVersion: v1
 kind: Secret
 metadata:
@@ -178,59 +177,3 @@ spec:
           secret:
             secretName: {{.Secret}}
             optional: false
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{.DNSAppName}}
-  namespace: {{.Namespace}}
-spec:
-  replicas: {{.Replicas}}
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: {{.DNSAppName}}
-  template:
-    metadata:
-      labels:
-        app: {{.DNSAppName}}
-    spec:
-      containers:
-      - name: kcp-dns
-        command:
-        - /ko-app/syncer
-        args:
-        - dns
-        - start
-        env:
-        - name: NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        image: {{.Image}}
-        imagePullPolicy: IfNotPresent
-        terminationMessagePolicy: FallbackToLogsOnError
-      serviceAccountName: {{.DNSServiceAccount}}
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{.DNSAppName}}
-  namespace: {{.Namespace}}
-  labels:
-    app: {{.DNSAppName}}
-spec:
-  type: ClusterIP
-  selector:
-    app: {{.DNSAppName}}
-  ports:
-    - name: dns
-      port: 53
-      protocol: UDP
-      targetPort: 5353
-    - name: dns-tcp
-      port: 53
-      protocol: TCP
-      targetPort: 5353
-
diff --git a/pkg/syncer/namespace/deployment_dns.yaml b/pkg/syncer/namespace/deployment_dns.yaml
@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: Name
+  namespace: Namespace
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: Name
+  template:
+    metadata:
+      labels:
+        app: Name
+    spec:
+      containers:
+        - name: kcp-dns
+          command:
+            - /ko-app/syncer
+          args:
+            - dns
+            - start
+          env:
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CONFIGMAP_NAME
+              value: ConfigMapName
+          image: Image
+          imagePullPolicy: IfNotPresent
+          terminationMessagePolicy: FallbackToLogsOnError
+      serviceAccountName: Name
diff --git a/pkg/syncer/namespace/namespace_downstream_controller.go b/pkg/syncer/namespace/namespace_downstream_controller.go
@@ -60,6 +60,11 @@ type DownstreamController struct {
 	listDownstreamNamespaces  func() ([]runtime.Object, error)
 	createConfigMap           func(ctx context.Context, configMap *corev1.ConfigMap) (*corev1.ConfigMap, error)
 	updateConfigMap           func(ctx context.Context, configMap *corev1.ConfigMap) (*corev1.ConfigMap, error)
+	createServiceAccount      func(ctx context.Context, name, namespace string) error
+	createRole                func(ctx context.Context, name, namespace string) error
+	createRoleBinding         func(ctx context.Context, name, namespace string) error
+	createDeployment          func(ctx context.Context, name, namespace string) error
+	createService             func(ctx context.Context, name, namespace string) error
 
 	syncTargetName      string
 	syncTargetWorkspace logicalcluster.Name
@@ -108,6 +113,11 @@ func NewDownstreamController(
 		updateConfigMap: func(ctx context.Context, configMap *corev1.ConfigMap) (*corev1.ConfigMap, error) {
 			return kubeClient.CoreV1().ConfigMaps(configMap.Namespace).Update(ctx, configMap, metav1.UpdateOptions{})
 		},
+		createServiceAccount: createServiceAccountFn(kubeClient),
+		createRole:           createRoleFn(kubeClient),
+		createRoleBinding:    createRoleBindingFn(kubeClient),
+		createDeployment:     createDeploymentFn(kubeClient),
+		createService:        createServiceFn(kubeClient),
 
 		syncTargetName:      syncTargetName,
 		syncTargetWorkspace: syncTargetWorkspace,

diff --git a/pkg/syncer/namespace/namespace_downstream_process.go b/pkg/syncer/namespace/namespace_downstream_process.go
@@ -18,8 +18,13 @@ package namespace
 
 import (
 	"context"
+	"crypto/sha256"
 	"encoding/json"
+	"fmt"
+	"strings"
 
+	"github.com/kcp-dev/logicalcluster/v2"
+	"github.com/martinlindhe/base36"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -43,7 +48,7 @@ func (c *DownstreamController) process(ctx context.Context, key string) error {
 
 	logger = logger.WithValues(DownstreamNamespace, namespaceName)
 
-	// Always refresh the DNS ConfigMap
+	// Always refresh the DNS ConfigMap (even when the namespace has been deleted or is being deleted)
 	err = c.updateDNSConfigMap(ctx)
 	if err != nil {
 		return err
@@ -61,6 +66,11 @@ func (c *DownstreamController) process(ctx context.Context, key string) error {
 	downstreamNamespace := downstreamNamespaceObj.(*unstructured.Unstructured)
 	logger = logging.WithObject(logger, downstreamNamespace)
 
+	if !downstreamNamespace.GetDeletionTimestamp().IsZero() {
+		logger.V(4).Info("downstream namespace is being deleted, ignoring key")
+		return nil
+	}
+
 	namespaceLocatorJSON := downstreamNamespace.GetAnnotations()[shared.NamespaceLocatorAnnotation]
 	if namespaceLocatorJSON == "" {
 		logger.Error(nil, "downstream namespace has no namespaceLocator annotation")
@@ -90,7 +100,14 @@ func (c *DownstreamController) process(ctx context.Context, key string) error {
 		logger.Info("deleting downstream namespace because the upstream namespace doesn't exist")
 		return c.deleteDownstreamNamespace(ctx, namespaceName)
 	}
-	// The upstream namespace still exists, nothing to do.
+
+	// The upstream namespace still exists, make sure the DNS nameserver for the
+	// namespace workspace is up and running
+	if err := c.ensureDNSExists(ctx, nsLocator.Workspace); err != nil {
+		logger.Error(err, "failed to check if DNS nameserver exists (retrying)")
+		return err
+	}
+
 	return nil
 }
 
@@ -158,3 +175,42 @@ func (c *DownstreamController) updateDNSConfigMap(ctx context.Context) error {
 	}
 	return nil
 }
+
+// ensureDNSExists checks all DNS objects are created and up-to-date.
+func (c *DownstreamController) ensureDNSExists(ctx context.Context, workspace logicalcluster.Name) error {
+	logger := klog.FromContext(ctx)
+	logger.WithName("dns")
+
+	workspaceID := c.getWorkspaceID(workspace)
+	logger.Info("ensuring dns deployment exist", "workspace", workspaceID)
+
+	err := c.createServiceAccount(ctx, workspaceID, c.dnsNamespace)
+	if err != nil {
+		logger.Error(err, "failed to create DNS service account")
+	}
+	err = c.createRole(ctx, workspaceID, c.dnsNamespace)
+	if err != nil {
+		logger.Error(err, "failed to create DNS service account")
+	}
+	err = c.createRoleBinding(ctx, workspaceID, c.dnsNamespace)
+	if err != nil {
+		logger.Error(err, "failed to create DNS service account")
+	}
+	err = c.createDeployment(ctx, workspaceID, c.dnsNamespace)
+	if err != nil {
+		logger.Error(err, "failed to create DNS service account")
+	}
+	err = c.createService(ctx, workspaceID, c.dnsNamespace)
+	if err != nil {
+		logger.Error(err, "failed to create DNS service account")
+	}
+	return nil
+}
+
+// getWorkspaceID returns a unique ID for the workspace name. It's
+// a valid DNS segment and can be used as namespace or object names.
+func (c *DownstreamController) getWorkspaceID(workspace logicalcluster.Name) string {
+	syncerHash := sha256.Sum224([]byte(workspace.String()))
+	base36hash := strings.ToLower(base36.EncodeBytes(syncerHash[:]))
+	return fmt.Sprintf("kcp-dns-%s-%s", c.syncTargetName, base36hash[:8])
+}