Add e2e test (#9)

* Add e2e test

Signed-off-by: zeroalphat <taichi-takemura@cybozu.co.jp>

---------

Signed-off-by: zeroalphat <taichi-takemura@cybozu.co.jp>

.github/workflows/ci.yaml

@@ -4,15 +4,27 @@ on:
   push:
     branches:
     - 'main'
-env:
-  go-version: "1.21"
 jobs:
   test:
     name: Small tests
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-go@v3
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v4
       with:
-        go-version: ${{ env.go-version }}
+        go-version-file: go.mod
     - run: make test
+  e2e:
+    name: End-to-end tests
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v4
+      with:
+        go-version-file: go.mod
+    - run: make setup
+      working-directory: e2e
+    - run: make start
+      working-directory: e2e
+    - run: make test
+      working-directory: e2e

Dockerfile.cli

@@ -0,0 +1,18 @@
+FROM ghcr.io/cybozu/golang:1.21-jammy as builder
+WORKDIR /work
+COPY go.mod go.mod
+COPY go.sum go.sum
+
+COPY cmd/necoperf-cli cmd/necoperf-cli
+COPY internal internal
+RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o necoperf-cli ./cmd/necoperf-cli
+
+FROM quay.io/cybozu/pause:3.9 as pause
+FROM ghcr.io/cybozu/ubuntu:22.04
+LABEL org.opencontainers.image.source https://github.com/cybozu-go/necoperf
+
+COPY --from=pause /pause /usr/local/bin/pause
+COPY --from=builder /work/necoperf-cli /usr/local/bin/necoperf-cli
+
+USER 1000:1000
+CMD ["pause"]

Dockerfile.daemon

@@ -0,0 +1,37 @@
+ARG FLATCAR_VERSION
+FROM ghcr.io/cybozu/golang:1.21-jammy as builder
+
+WORKDIR /work
+COPY go.mod go.mod
+COPY go.sum go.sum
+
+COPY cmd/necoperf-daemon cmd/necoperf-daemon
+COPY internal internal
+RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o necoperf-daemon ./cmd/necoperf-daemon
+
+FROM ghcr.io/flatcar/flatcar-sdk-amd64:${FLATCAR_VERSION} as flatcar
+RUN ldd /usr/bin/perf | cut -d" " -f3 | xargs tar --dereference -cf libs.tar
+
+FROM scratch
+LABEL org.opencontainers.image.source https://github.com/cybozu-go/necoperf
+ENV HOME=/home/necoperf
+COPY --from=flatcar /bin/ /bin/
+COPY --from=flatcar /sbin/setcap /sbin/setcap
+COPY --from=flatcar /lib64 /lib64
+COPY --from=flatcar /usr/bin/perf /usr/bin/sleep /usr/bin/
+
+# install perf dependency
+COPY --from=flatcar /libs.tar /libs.tar
+RUN mkdir -p /usr/lib64 \
+    && tar -xf libs.tar \
+    && rm libs.tar
+
+COPY --from=builder /work/necoperf-daemon /usr/local/bin/necoperf-daemon
+
+RUN setcap "cap_perfmon,cap_sys_ptrace,cap_syslog,cap_sys_admin,cap_sys_chroot=ep" /usr/bin/perf \
+    && mkdir -p ${HOME} \
+    && chown -R 1000:1000 ${HOME}
+WORKDIR ${HOME}
+
+USER 1000:1000
+ENTRYPOINT ["/usr/local/bin/necoperf-daemon", "start"]

Makefile

@@ -54,6 +54,15 @@ internal/rpc/necoperf_grpc.pb.go: internal/rpc/necoperf.proto
 docs/necoperf-grpc.md: internal/rpc/necoperf.proto
 	$(PROTOC) --doc_out=docs --doc_opt=markdown,$@ $<
 
+.PHONY: docker-build
+docker-build: build
+	docker build -t necoperf-daemon:dev --build-arg="FLATCAR_VERSION=$(FLATCAR_VERSION)" -f Dockerfile.daemon .
+	docker build -t necoperf-cli:dev -f Dockerfile.cli .
+
+.PHONY: e2e
+e2e:
+	$(MAKE) -C e2e
+
 ##@ Tools
 
 .PHONY: setup

Makefile.versions

@@ -1 +1,3 @@
 E2ETEST_K8S_VERSION  := 1.27.1
+# ref to https://github.com/orgs/flatcar/packages/container/package/flatcar-sdk-amd64
+FLATCAR_VERSION := 3602.0.0

config/namespace/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - namespace.yaml

config/namespace/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: necoperf

config/rbac/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+    - serviceaccount.yaml
+    - role.yaml
+    - rolebinding.yaml

config/rbac/role.yaml

@@ -0,0 +1,19 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: necoperf-daemon-role
+  namespace: necoperf
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get","list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: necoperf-cli-role
+  namespace: default
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get","list"]

config/rbac/rolebinding.yaml

@@ -0,0 +1,27 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: necoperf-cli-rolebinding
+  namespace: default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: necoperf-cli-role
+subjects:
+- kind: ServiceAccount
+  name: necoperf-sa
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: necoperf-daemon-rolebinding
+  namespace: necoperf
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: necoperf-daemon-role
+subjects:
+- kind: ServiceAccount
+  name: necoperf-sa
+  namespace: default

config/rbac/serviceaccount.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: necoperf-sa
+  namespace: default

e2e/Makefile

@@ -0,0 +1,86 @@
+include Makefile.versions
+
+ARCH ?= amd64
+OS ?= linux
+
+PROJECT_DIR := $(CURDIR)/../
+BIN_DIR := $(PROJECT_DIR)/bin
+
+CURL := curl -sSLf
+KUBECTL := $(BIN_DIR)/kubectl
+KUSTOMIZE := $(BIN_DIR)/kustomize
+
+KIND := $(BIN_DIR)/kind
+KIND_CLUSTER_NAME := necoperf
+KIND_CONFIG := kind-config.yaml
+
+export KUBECONFIG
+
+.PHONY: help
+help:
+	@echo "Choose one of the following target"
+	@echo
+	@echo "setup      Setup tools"
+	@echo "start      Start kind cluster and install accurate"
+	@echo "test       Run e2e tests"
+	@echo "logs       Save logs as logs.tar.gz"
+	@echo "stop       Stop the kind cluster"
+
+.PHONY: setup
+setup: kubectl kustomize kind
+
+.PHONY: start
+start:
+	$(KIND) create cluster --name=$(KIND_CLUSTER_NAME) --config=$(KIND_CONFIG) --image=kindest/node:v$(E2ETEST_K8S_VERSION) --wait 1m
+	$(MAKE) -C ../ docker-build
+	$(KIND) load docker-image necoperf-daemon:dev --name=$(KIND_CLUSTER_NAME)
+	$(KIND) load docker-image necoperf-cli:dev --name=$(KIND_CLUSTER_NAME)
+	$(KUSTOMIZE) build ../config/namespace | $(KUBECTL) apply -f -
+	$(KUSTOMIZE) build ../config/rbac | $(KUBECTL) apply -f -
+
+.PHONY: test
+test:
+	env RUN_E2E=1 \
+		go test -v -race . -ginkgo.v -ginkgo.fail-fast
+
+.PHONY: stop
+stop:
+	$(KIND) delete cluster --name=$(KIND_CLUSTER_NAME)
+	-docker image rm necoperf-daemon:dev
+	-docker image rm necoperf-cli:dev
+	-docker image prune -f
+
+.PHONY: kustomize
+kustomize: $(KUSTOMIZE)
+$(KUSTOMIZE): $(KUSTOMIZE)-$(KUSTOMIZE_VERSION)
+	ln -sf $(notdir $<) $@
+
+$(KUSTOMIZE)-$(KUSTOMIZE_VERSION):
+	mkdir -p $(dir $@)
+	curl -fsL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv$(KUSTOMIZE_VERSION)/kustomize_v$(KUSTOMIZE_VERSION)_linux_amd64.tar.gz | \
+	tar -xzf - -O > $@
+	chmod a+x $@
+
+.PHONY: kind
+kind: $(KIND)
+$(KIND): $(KIND)-$(KIND_VERSION)
+	ln -sf $(notdir $<) $@
+
+$(KIND)-$(KIND_VERSION):
+	mkdir -p $(dir $@)
+	$(CURL) -o $@ https://github.com/kubernetes-sigs/kind/releases/download/v$(KIND_VERSION)/kind-$(OS)-$(ARCH)
+	chmod a+x $@
+
+.PHONY: kubectl
+kubectl: $(KUBECTL)
+$(KUBECTL): $(KUBECTL)-$(E2ETEST_K8S_VERSION)
+	ln -sf $(notdir $<) $@
+
+$(KUBECTL)-$(E2ETEST_K8S_VERSION):
+	mkdir -p $(dir $@)
+	$(CURL) -o $@ https://storage.googleapis.com/kubernetes-release/release/v$(E2ETEST_K8S_VERSION)/bin/$(OS)/$(ARCH)/kubectl
+	chmod a+x $@
+
+.PHONY: clean
+clean:
+	rm -rf $(BIN_DIR)

e2e/Makefile.versions

@@ -0,0 +1,3 @@
+E2ETEST_K8S_VERSION  := 1.27.1
+KIND_VERSION := 0.20.0
+KUSTOMIZE_VERSION := 5.1.0

e2e/e2e_test.go

@@ -0,0 +1,21 @@
+package e2e
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("necoperf e2e test", func() {
+	It("should be able to run necoperf-cli", func() {
+		By("executing necoperf-cli")
+		_, err := kubectl(nil, "exec", "necoperf-client", "--",
+			"necoperf-cli", "profile", "profiled-pod",
+			"--timeout", "100s")
+		Expect(err).NotTo(HaveOccurred())
+
+		By("checking if profiling result is created")
+		out, err := kubectl(nil, "exec", "necoperf-client", "--", "cat", "/tmp/profiled-pod.script")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(out).To(ContainSubstring("yes"))
+	})
+})

e2e/env_test.go

@@ -0,0 +1,7 @@
+package e2e
+
+import "os"
+
+var (
+	runE2E = os.Getenv("RUN_E2E") != ""
+)

e2e/kind-config.yaml

@@ -0,0 +1,5 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker

e2e/manifests/necoperf-client.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: necoperf-client
+  namespace: default
+spec:
+  containers:
+  - name: necoperf-client
+    image: necoperf-cli:dev
+    imagePullPolicy: IfNotPresent
+    command: ["pause"]
+  securityContext:
+    runAsUser: 10000
+    runAsGroup: 10000
+  serviceAccountName: necoperf-sa

e2e/manifests/necoperf-daemonset.yaml

@@ -0,0 +1,61 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: necoperf-daemon
+  namespace: necoperf
+  labels:
+    k8s-app: necoperf-daemon
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: necoperf-daemon
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: necoperf-daemon
+    spec:
+      hostPID: true
+      terminationGracePeriodSeconds: 30
+      containers:
+        - name: necoperf-daemon
+          image: necoperf-daemon:dev
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            capabilities:
+              add:
+                - SYSLOG
+                - SYS_PTRACE
+                - PERFMON
+                - SYS_CHROOT
+                - SYS_ADMIN
+              drop:
+                - ALL
+          volumeMounts:
+            - name: necoperf-workdir
+              mountPath: /var/necoperf
+            - name: containerd-sock
+              mountPath: /run/containerd/containerd.sock
+            - name: sys-kernel-tracing
+              mountPath: /sys/kernel/tracing
+          ports:
+            - name: grpc
+              containerPort: 6543
+            - name: metrics
+              containerPort: 6541
+          livenessProbe:
+            grpc:
+              port: 6543
+            initialDelaySeconds: 10
+          readinessProbe:
+            grpc:
+              port: 6543
+            initialDelaySeconds: 5
+      volumes:
+        - name: necoperf-workdir
+          emptyDir: {}
+        - name: containerd-sock
+          hostPath:
+            path: /run/containerd/containerd.sock
+        - name: sys-kernel-tracing
+          hostPath:
+            path: /sys/kernel/tracing

e2e/manifests/profiled-pod.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: profiled-pod
+spec:
+  securityContext:
+    runAsUser: 10000
+    runAsGroup: 10000
+  containers:
+  - name: profiled-pod
+    image: ghcr.io/cybozu/ubuntu:22.04
+    command: ["yes"]