Make mock provider async [WIP]

.circleci/config.yml

@@ -109,12 +109,18 @@ jobs:
       - run:
           name: Run the end-to-end test suite
           command: |
-            make e2e
+            make e2e.clean/vkubelet-mock-0 e2e.clean/vkubelet-mock-1
+            make e2e/vkubelet-mock-0 e2e/vkubelet-mock-1
       - run:
           name: Collect logs on failure from vkubelet-mock-0
           command: |
             kubectl logs vkubelet-mock-0
           when: on_fail
+      - run:
+          name: Collect logs on failure from vkubelet-mock-1
+          command: |
+            kubectl logs vkubelet-mock-1
+          when: on_fail
 
 workflows:
   version: 2

Makefile.e2e

@@ -15,40 +15,37 @@ skaffold: skaffold/$(MODE)
 
 .PHONY: skaffold/%
 skaffold/%: PROFILE := local
-skaffold/%: skaffold.validate
+skaffold/%: skaffold.validate | bin/e2e/virtual-kubelet
 	skaffold $(*) \
 		-f $(PWD)/hack/skaffold/virtual-kubelet/skaffold.yml \
 		-p $(PROFILE)
 
 bin/e2e:
 	@mkdir -p bin/e2e
 
+.PHONY: bin/e2e/virtual-kubelet
 bin/e2e/virtual-kubelet: bin/e2e
 	GOOS=linux GOARCH=amd64 $(MAKE) OUTPUT_DIR=$(@D) build
 
 # e2e runs the end-to-end test suite against the Kubernetes cluster targeted by the current kubeconfig.
 # It automatically deploys the virtual-kubelet with the mock provider by running "make skaffold MODE=run".
 # It is the caller's responsibility to cleanup the deployment after running this target (e.g. by running "make skaffold MODE=delete").
 .PHONY: e2e
-e2e: KUBECONFIG ?= $(HOME)/.kube/config
-e2e: NAMESPACE := default
-e2e: NODE_NAME := vkubelet-mock-0
-e2e: TAINT_KEY := virtual-kubelet.io/provider
-e2e: TAINT_VALUE := mock
-e2e: TAINT_EFFECT := NoSchedule
-e2e: export VK_BUILD_TAGS += mock_provider
-e2e: e2e.clean bin/e2e/virtual-kubelet skaffold/run
+e2e: e2e/vkubelet-mock-0
+.PHONY: e2e/%
+e2e/%: KUBECONFIG ?= $(HOME)/.kube/config
+e2e/%: NAMESPACE := default
+e2e/%: export VK_BUILD_TAGS += mock_provider
+e2e/%: bin/e2e/virtual-kubelet skaffold/run
 	@echo Running tests...
 	cd $(PWD)/test/e2e && go test -v -timeout 5m -tags e2e ./... \
 		-kubeconfig=$(KUBECONFIG) \
 		-namespace=$(NAMESPACE) \
-		-node-name=$(NODE_NAME) \
-		-taint-key=$(TAINT_KEY) \
-		-taint-value=$(TAINT_VALUE) \
-		-taint-effect=$(TAINT_EFFECT)
-
+		-node-name=$(*)
 .PHONY: e2e.clean
-e2e.clean: NODE_NAME ?= vkubelet-mock-0
 e2e.clean: skaffold/delete
-	kubectl delete --ignore-not-found node $(NODE_NAME); \
 	if [ -f bin/e2e/virtual-kubelet ]; then  rm bin/e2e/virtual-kubelet; fi
+
+.PHONY: e2e.clean/%
+e2e.clean/%: e2e.clean
+	kubectl delete --ignore-not-found node $(*)

hack/skaffold/virtual-kubelet/Dockerfile

@@ -1,16 +1,16 @@
 FROM gcr.io/distroless/base
 
-ENV APISERVER_CERT_LOCATION /vkubelet-mock-0-crt.pem
-ENV APISERVER_KEY_LOCATION /vkubelet-mock-0-key.pem
+ENV APISERVER_CERT_LOCATION /vkubelet-mock-crt.pem
+ENV APISERVER_KEY_LOCATION /vkubelet-mock-key.pem
 ENV KUBELET_PORT 10250
 
 # Use the pre-built binary in "bin/virtual-kubelet".
 COPY bin/e2e/virtual-kubelet /virtual-kubelet
 # Copy the configuration file for the mock provider.
-COPY hack/skaffold/virtual-kubelet/vkubelet-mock-0-cfg.json /vkubelet-mock-0-cfg.json
+COPY hack/skaffold/virtual-kubelet/vkubelet-mock-cfg.json /vkubelet-mock-cfg.json
 # Copy the certificate for the HTTPS server.
-COPY hack/skaffold/virtual-kubelet/vkubelet-mock-0-crt.pem /vkubelet-mock-0-crt.pem
+COPY hack/skaffold/virtual-kubelet/vkubelet-mock-crt.pem /vkubelet-mock-crt.pem
 # Copy the private key for the HTTPS server.
-COPY hack/skaffold/virtual-kubelet/vkubelet-mock-0-key.pem /vkubelet-mock-0-key.pem
+COPY hack/skaffold/virtual-kubelet/vkubelet-mock-key.pem /vkubelet-mock-key.pem
 
 CMD ["/virtual-kubelet"]

hack/skaffold/virtual-kubelet/base.yml

@@ -13,6 +13,7 @@ rules:
   resources:
   - configmaps
   - secrets
+  - services
   verbs:
   - get
   - list
@@ -26,6 +27,7 @@ rules:
   - get
   - list
   - watch
+  - patch
 - apiGroups:
   - ""
   resources:
@@ -39,12 +41,14 @@ rules:
   - nodes/status
   verbs:
   - update
+  - patch
 - apiGroups:
   - ""
   resources:
   - pods/status
   verbs:
   - update
+  - patch
 - apiGroups:
   - ""
   resources:

hack/skaffold/virtual-kubelet/pod.yml → hack/skaffold/virtual-kubelet/pod1.yml

@@ -13,16 +13,23 @@ spec:
     - --nodename
     - vkubelet-mock-0
     - --provider
-    - mock
+    - mocklegacy
     - --provider-config
-    - /vkubelet-mock-0-cfg.json
+    - /vkubelet-mock-cfg.json
     - --startup-timeout
     - 10s
     - --klog.v
     - "2"
     - --klog.logtostderr
     - --log-level
     - debug
+    env:
+    - name: KUBELET_PORT
+      value: "10250"
+    - name: VKUBELET_POD_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
     ports:
     - name: metrics
       containerPort: 10255

hack/skaffold/virtual-kubelet/pod2.yml

@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: vkubelet-mock-1
+spec:
+  containers:
+  - name: vkubelet-mock-1
+    image: virtual-kubelet
+    # "IfNotPresent" is used to prevent Minikube from trying to pull from the registry (and failing) in the first place.
+    imagePullPolicy: IfNotPresent
+    args:
+    - /virtual-kubelet
+    - --nodename
+    - vkubelet-mock-1
+    - --provider
+    - mock
+    - --provider-config
+    - /vkubelet-mock-cfg.json
+    - --startup-timeout
+    - 10s
+    - --klog.v
+    - "2"
+    - --klog.logtostderr
+    - --log-level
+    - debug
+    - --metrics-addr
+    - :10256
+    env:
+    - name: KUBELET_PORT
+      value: "10350"
+    - name: VKUBELET_POD_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
+    ports:
+    - name: metrics
+      containerPort: 10256
+    readinessProbe:
+      httpGet:
+        path: /stats/summary
+        port: metrics
+  serviceAccountName: virtual-kubelet

hack/skaffold/virtual-kubelet/skaffold.yml

@@ -10,7 +10,8 @@ deploy:
   kubectl:
     manifests:
     - hack/skaffold/virtual-kubelet/base.yml
-    - hack/skaffold/virtual-kubelet/pod.yml
+    - hack/skaffold/virtual-kubelet/pod1.yml
+    - hack/skaffold/virtual-kubelet/pod2.yml
 profiles:
 - name: local
   build:

hack/skaffold/virtual-kubelet/vkubelet-mock-0-cfg.json → hack/skaffold/virtual-kubelet/vkubelet-mock-cfg.json

@@ -3,5 +3,10 @@
     "cpu": "2",
     "memory": "32Gi",
     "pods": "128"
+  },
+  "vkubelet-mock-1": {
+    "cpu": "2",
+    "memory": "32Gi",
+    "pods": "128"
   }
-}
+}