From e94930179ef3c3c34fab58df7c0092cfd9c9829f Mon Sep 17 00:00:00 2001 From: zeroalphat Date: Mon, 2 Oct 2023 06:14:25 +0000 Subject: [PATCH 01/13] Implement necoperf-cli Signed-off-by: zeroalphat --- Makefile | 18 +++- Makefile.versions | 1 + cmd/necoperf-cli/cmd/profile.go | 98 ++++++++++++++++++ cmd/necoperf-cli/cmd/root.go | 26 +++++ cmd/necoperf-cli/main.go | 7 ++ go.mod | 12 ++- go.sum | 26 +++++ internal/client/client.go | 145 +++++++++++++++++++++++++++ internal/resource/discovery.go | 89 +++++++++++++++++ internal/resource/discovery_test.go | 38 +++++++ internal/resource/suite_test.go | 149 ++++++++++++++++++++++++++++ 11 files changed, 600 insertions(+), 9 deletions(-) create mode 100644 Makefile.versions create mode 100644 cmd/necoperf-cli/cmd/profile.go create mode 100644 cmd/necoperf-cli/cmd/root.go create mode 100644 cmd/necoperf-cli/main.go create mode 100644 internal/client/client.go create mode 100644 internal/resource/discovery.go create mode 100644 internal/resource/discovery_test.go create mode 100644 internal/resource/suite_test.go diff --git a/Makefile b/Makefile index 17903bd..1ece7d1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,7 @@ +include Makefile.versions + BIN_DIR := $(shell pwd)/bin +ENVTEST ?= $(BIN_DIR)/setup-envtest # Tool versions MDBOOK_VERSION = 0.4.27 @@ -27,10 +30,8 @@ build: GOBIN=$(BIN_DIR) go install ./cmd/... .PHONY: test -test: - if find . -name go.mod | grep -q go.mod; then \ - $(MAKE) test-go; \ - fi +test: envtest + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(BIN_DIR) -p path)" go test ./... -coverprofile cover.out -v .PHONY: test-go test-go: test-tools @@ -56,7 +57,8 @@ docs/necoperf-grpc.md: internal/rpc/necoperf.proto ##@ Tools .PHONY: setup -setup: +setup: envtest + mkdir -p bin curl -sfL -o protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-linux-x86_64.zip unzip -o protoc.zip bin/protoc 'include/*' rm -f protoc.zip @@ -64,6 +66,12 @@ setup: GOBIN=$(PWD)/bin go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v$(PROTOC_GEN_GO_GRPC_VERSION) GOBIN=$(PWD)/bin go install github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc@v$(PROTOC_GEN_DOC_VERSION) +.PHONY: envtest +envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. +$(ENVTEST): + mkdir -p bin + test -s $(BIN_DIR)/setup-envtest || GOBIN=$(BIN_DIR) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + $(MDBOOK): mkdir -p bin curl -fsL https://github.com/rust-lang/mdBook/releases/download/v$(MDBOOK_VERSION)/mdbook-v$(MDBOOK_VERSION)-x86_64-unknown-linux-gnu.tar.gz | tar -C bin -xzf - diff --git a/Makefile.versions b/Makefile.versions new file mode 100644 index 0000000..06a16d4 --- /dev/null +++ b/Makefile.versions @@ -0,0 +1 @@ +E2ETEST_K8S_VERSION := 1.27.1 diff --git a/cmd/necoperf-cli/cmd/profile.go b/cmd/necoperf-cli/cmd/profile.go new file mode 100644 index 0000000..4e51c49 --- /dev/null +++ b/cmd/necoperf-cli/cmd/profile.go @@ -0,0 +1,98 @@ +package cmd + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "time" + + "github.com/cybozu-go/necoperf/internal/client" + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" +) + +var config struct { + namespace string + podName string + outputDir string + containerName string + necoperfNS string + timeout time.Duration +} + +func isPodReady(pod *corev1.Pod) bool { + for _, cond := range pod.Status.Conditions { + if cond.Type != corev1.PodReady { + continue + } + return cond.Status == corev1.ConditionTrue + } + return false +} + +func NewProfileCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "profile", + Short: "Exec perf profile of the target container", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + config.podName = args[0] + handler := slog.NewTextHandler(os.Stderr, nil) + logger := slog.New(handler) + + client, err := client.New(logger, config.timeout) + if err != nil { + return err + } + if err := client.SetupDiscovery(); err != nil { + return err + } + + ctx := cmd.Context() + pod, err := client.Discovery.GetPod(ctx, config.namespace, config.podName) + if err != nil { + return err + } + if !isPodReady(pod) { + return fmt.Errorf("pod %s is not ready", pod.Name) + } + containerID, err := client.Discovery.GetContainerID(pod, config.containerName) + if err != nil { + return err + } + logger.Info("get container id", "podName", config.podName, "containerID", containerID) + + pods, err := client.Discovery.GetPodList(ctx, config.necoperfNS) + if err != nil { + return err + } + addr, err := client.Discovery.DiscoveryServerAddr(pods, pod.Status.HostIP) + if err != nil { + return err + } + err = client.SetupGrpcClient(addr) + if err != nil { + return err + } + logger.Info("connect grpc server", "addr", addr) + + err = client.Profile(ctx, config.podName, containerID, config.outputDir) + if err != nil { + return err + } + logger.Info("profile is finished", "output directory", filepath.Join(config.outputDir, config.podName+".script")) + + return nil + }, + SilenceUsage: true, + SilenceErrors: true, + } + cmd.Flags().StringVar(&config.namespace, "namespace", "default", "kubernetes namespace") + cmd.Flags().StringVar(&config.outputDir, "outputDir", "/tmp", "output data directory") + cmd.Flags().StringVar(&config.containerName, "container", "", "container name") + cmd.Flags().StringVar(&config.necoperfNS, "necoperf-namespace", "kube-system", "necoperf namespace") + cmd.Flags().DurationVar(&config.timeout, "timeout", time.Second*30, "timeout seconds") + + return cmd +} diff --git a/cmd/necoperf-cli/cmd/root.go b/cmd/necoperf-cli/cmd/root.go new file mode 100644 index 0000000..f6fbd93 --- /dev/null +++ b/cmd/necoperf-cli/cmd/root.go @@ -0,0 +1,26 @@ +package cmd + +import ( + "log" + + "github.com/spf13/cobra" +) + +func NewRootCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "necoperf-cli", + RunE: func(cmd *cobra.Command, args []string) error { + return cmd.Help() + }, + } + return cmd +} + +func Execute() { + rootCmd := NewRootCommand() + rootCmd.AddCommand(NewProfileCommand()) + + if err := rootCmd.Execute(); err != nil { + log.Fatal(err) + } +} diff --git a/cmd/necoperf-cli/main.go b/cmd/necoperf-cli/main.go new file mode 100644 index 0000000..5493b82 --- /dev/null +++ b/cmd/necoperf-cli/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/cybozu-go/necoperf/cmd/necoperf-cli/cmd" + +func main() { + cmd.Execute() +} diff --git a/go.mod b/go.mod index 923c9d5..b6e17a4 100644 --- a/go.mod +++ b/go.mod @@ -8,13 +8,19 @@ require ( github.com/google/uuid v1.3.0 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0 github.com/oklog/run v1.1.0 + github.com/onsi/ginkgo/v2 v2.11.0 + github.com/onsi/gomega v1.27.10 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 golang.org/x/sync v0.3.0 google.golang.org/grpc v1.58.1 google.golang.org/protobuf v1.31.0 + k8s.io/api v0.28.2 + k8s.io/apimachinery v0.28.2 + k8s.io/client-go v0.28.1 k8s.io/cri-api v0.28.2 k8s.io/kubernetes v1.28.2 + sigs.k8s.io/controller-runtime v0.16.2 ) require ( @@ -32,12 +38,13 @@ require ( github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -80,11 +87,8 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.28.2 // indirect k8s.io/apiextensions-apiserver v0.28.0 // indirect - k8s.io/apimachinery v0.28.2 // indirect k8s.io/apiserver v0.28.1 // indirect - k8s.io/client-go v0.28.1 // indirect k8s.io/component-base v0.28.1 // indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect diff --git a/go.sum b/go.sum index bdc77d3..e13813f 100644 --- a/go.sum +++ b/go.sum @@ -82,8 +82,12 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -94,6 +98,8 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= +github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= @@ -111,6 +117,8 @@ github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -179,8 +187,12 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4Zs github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -214,6 +226,9 @@ github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= @@ -241,6 +256,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -279,6 +295,10 @@ go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJP go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= +go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -294,6 +314,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -460,6 +482,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -601,6 +625,8 @@ k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= +sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= diff --git a/internal/client/client.go b/internal/client/client.go new file mode 100644 index 0000000..2cbd52e --- /dev/null +++ b/internal/client/client.go @@ -0,0 +1,145 @@ +package client + +import ( + "context" + "io" + "log/slog" + "os" + "path/filepath" + "time" + + "github.com/cybozu-go/necoperf/internal/resource" + "github.com/cybozu-go/necoperf/internal/rpc" + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/keepalive" + "google.golang.org/protobuf/types/known/durationpb" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +type Client struct { + logger *slog.Logger + client rpc.NecoPerfClient + Discovery *resource.Discovery + Timeout time.Duration +} + +// https://github.com/grpc-ecosystem/go-grpc-middleware/blob/main/interceptors/logging/examples/slog/example_test.go +func InterceptorLogger(l *slog.Logger) logging.Logger { + return logging.LoggerFunc(func(ctx context.Context, lvl logging.Level, msg string, fields ...any) { + l.Log(ctx, slog.Level(lvl), "msg", msg, fields) + }) +} + +func New(logger *slog.Logger, timeout time.Duration) (*Client, error) { + return &Client{ + logger: logger, + Timeout: timeout, + }, nil +} + +func (c *Client) Save(dataDir, podName string) (*os.File, error) { + err := os.MkdirAll(dataDir, 0755) + if err != nil { + return nil, err + } + + profileFilePath := filepath.Join(dataDir + "/" + podName + ".script") + f, err := os.Create(profileFilePath) + if err != nil { + return nil, err + } + + return f, nil +} + +func (c *Client) Profile(ctx context.Context, podName, containerID, DataDir string) error { + t := durationpb.New(c.Timeout) + req := &rpc.PerfProfileRequest{ + ContainerId: containerID, + Timeout: t, + } + + stream, err := c.client.Profile(ctx, req) + if err != nil { + return err + } + + f, err := c.Save(DataDir, podName) + if err != nil { + return err + } + defer f.Close() + + for { + resp, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + os.Remove(f.Name()) + return err + } + + _, err = f.Write(resp.Data) + if err != nil { + os.Remove(f.Name()) + return err + } + } + + return nil +} + +func (c *Client) SetupDiscovery() error { + config, err := config.GetConfig() + if err != nil { + return err + } + + k8sClient, err := client.New(config, client.Options{}) + if err != nil { + return err + } + + d, err := resource.NewDiscovery(c.logger, k8sClient) + if err != nil { + return err + } + c.Discovery = d + + return nil +} + +func (c *Client) SetupGrpcClient(addr string) error { + kp := keepalive.ClientParameters{ + Time: c.Timeout * 3, + } + opts := []logging.Option{ + logging.WithLogOnEvents(logging.StartCall, logging.FinishCall), + } + + conn, err := grpc.Dial( + addr, + grpc.WithChainUnaryInterceptor( + logging.UnaryClientInterceptor(InterceptorLogger(c.logger), opts...), + ), + grpc.WithChainStreamInterceptor( + logging.StreamClientInterceptor(InterceptorLogger(c.logger), opts...), + ), + grpc.WithTransportCredentials( + insecure.NewCredentials(), + ), + grpc.WithKeepaliveParams( + kp, + ), + ) + if err != nil { + return err + } + c.client = rpc.NewNecoPerfClient(conn) + + return nil +} diff --git a/internal/resource/discovery.go b/internal/resource/discovery.go new file mode 100644 index 0000000..7d44d8a --- /dev/null +++ b/internal/resource/discovery.go @@ -0,0 +1,89 @@ +package resource + +import ( + "context" + "errors" + "fmt" + "log/slog" + "regexp" + "strconv" + + "github.com/cybozu-go/necoperf/internal/constants" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + port = 6543 +) + +type Discovery struct { + logger *slog.Logger + client client.Client +} + +func NewDiscovery(logger *slog.Logger, client client.Client) (*Discovery, error) { + return &Discovery{ + logger: logger, + client: client, + }, nil +} + +func (d *Discovery) GetPod(ctx context.Context, namespace, podName string) (*corev1.Pod, error) { + pod := &corev1.Pod{} + + err := d.client.Get(ctx, client.ObjectKey{ + Namespace: namespace, + Name: podName, + }, pod) + if err != nil { + return nil, err + } + + return pod, nil +} + +func (d *Discovery) GetPodList(ctx context.Context, necoperfNS string) (*corev1.PodList, error) { + pods := &corev1.PodList{} + err := d.client.List(ctx, pods, client.InNamespace(necoperfNS), client.MatchingLabels{ + constants.LabelAppName: constants.AppNameNecoPerf, + }) + if err != nil { + return nil, err + } + + return pods, nil +} + +func (d *Discovery) GetContainerID(pod *corev1.Pod, containerName string) (string, error) { + if len(containerName) == 0 && len(pod.Status.Conditions) >= 1 { + containerName = pod.Spec.Containers[0].Name + } + + for i := range pod.Status.ContainerStatuses { + if pod.Status.ContainerStatuses[i].Name == containerName { + regex := regexp.MustCompile("[a-z]*://") + containerID := regex.ReplaceAllString(pod.Status.ContainerStatuses[i].ContainerID, "") + return containerID, nil + } + } + + return "", errors.New("failed to get container ID") +} + +func (d *Discovery) DiscoveryServerAddr(pods *corev1.PodList, hostIP string) (string, error) { + var podIP string + for _, pod := range pods.Items { + if pod.Status.HostIP == hostIP { + podIP = pod.Status.PodIP + } + } + if len(podIP) == 0 { + return "", errors.New("failed to get pod IP") + } + + addr := fmt.Sprintf("%s:%s", podIP, strconv.Itoa(port)) + d.logger.Info("found the pod IP of necoperf", "hostIP", hostIP, "podIP", podIP) + + return addr, nil +} diff --git a/internal/resource/discovery_test.go b/internal/resource/discovery_test.go new file mode 100644 index 0000000..24dddd3 --- /dev/null +++ b/internal/resource/discovery_test.go @@ -0,0 +1,38 @@ +package resource + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Test Discovery", func() { + ctx := context.Background() + + It("should get container id", func() { + pod, err := d.GetPod(ctx, "test", "test-pod") + Expect(err).NotTo(HaveOccurred()) + Expect(pod.Status.ContainerStatuses[0].ContainerID).NotTo(BeEmpty()) + containerID, err := d.GetContainerID(pod, "t1") + Expect(err).NotTo(HaveOccurred()) + Expect(containerID).NotTo(BeEmpty()) + }) + + It("should discovery server addr", func() { + By("get test pod") + pod, err := d.GetPod(ctx, "test", "test-pod") + Expect(err).NotTo(HaveOccurred()) + Expect(pod.Status.HostIP).NotTo(BeEmpty()) + + By("get necoperf daemonset") + podList, err := d.GetPodList(ctx, "test-for-necoperf") + Expect(err).NotTo(HaveOccurred()) + Expect(len(podList.Items)).To(Equal(1)) + + By("discovery server addr") + addr, err := d.DiscoveryServerAddr(podList, pod.Status.HostIP) + Expect(err).NotTo(HaveOccurred()) + Expect(addr).NotTo(BeEmpty()) + }) +}) diff --git a/internal/resource/suite_test.go b/internal/resource/suite_test.go new file mode 100644 index 0000000..fb7f509 --- /dev/null +++ b/internal/resource/suite_test.go @@ -0,0 +1,149 @@ +package resource + +import ( + "context" + "log/slog" + "testing" + "time" + + "github.com/cybozu-go/necoperf/internal/constants" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +var cfg *rest.Config +var cancelCluster context.CancelFunc +var testEnv *envtest.Environment +var scheme = runtime.NewScheme() +var k8sClient client.Client +var testThreshold time.Duration +var d *Discovery + +const ( + HostIP = "10.69.0.197" +) + +func TestDiscovery(t *testing.T) { + RegisterFailHandler(Fail) + SetDefaultEventuallyTimeout(1 * time.Minute) + SetDefaultEventuallyPollingInterval(1 * time.Second) + RunSpecs(t, "Test discovery") +} + +var _ = BeforeSuite(func() { + var err error + + testThreshold, err = time.ParseDuration("5s") + Expect(err).NotTo(HaveOccurred()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{} + + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + err = clientgoscheme.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + var ctx context.Context + ctx, cancelCluster = context.WithCancel(context.Background()) + testNamespace := corev1.Namespace{} + testNamespace.Name = "test" + + testPod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "test-pod", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "t1", + Image: "necoperf", + }, + }, + }, + } + + necoperfNamespace := corev1.Namespace{} + necoperfNamespace.Name = "test-for-necoperf" + testDaemonset := appsv1.DaemonSet{} + testDaemonset.Namespace = "test-for-necoperf" + testDaemonset.ObjectMeta.Name = "necoperf-daemon" + testDaemonset.ObjectMeta.Labels = map[string]string{constants.LabelAppName: constants.AppNameNecoPerf} + testDaemonset.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{constants.LabelAppName: constants.AppNameNecoPerf}} + testDaemonset.Spec.Template.ObjectMeta.Labels = map[string]string{constants.LabelAppName: constants.AppNameNecoPerf} + testDaemonset.Spec.Template.Spec.Containers = []corev1.Container{{Name: "n1", Image: "necoperf"}} + + err = k8sClient.Create(ctx, &testNamespace) + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Create(ctx, &testPod) + Expect(err).NotTo(HaveOccurred()) + updatePodStatus(ctx, &testPod, "containerd://necoperf", "t1", "10.244.1.2", + corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}) + + err = k8sClient.Create(ctx, &necoperfNamespace) + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Create(ctx, &testDaemonset) + Expect(err).NotTo(HaveOccurred()) + updateDaemonSetStatus(ctx, &testDaemonset) + + d, err = NewDiscovery(slog.Default(), k8sClient) + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + cancelCluster() + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +func updatePodStatus(ctx context.Context, pod *corev1.Pod, containerID, name, podIP string, state corev1.ContainerState) { + pod.Status.ContainerStatuses = []corev1.ContainerStatus{ + { + ContainerID: containerID, + Name: name, + State: state, + }, + } + pod.Status.HostIP = HostIP + pod.Status.PodIP = podIP + err := k8sClient.Status().Update(ctx, pod) + Expect(err).NotTo(HaveOccurred()) +} + +func updateDaemonSetStatus(ctx context.Context, ds *appsv1.DaemonSet) { + ds.Status.DesiredNumberScheduled = 1 + ds.Status.NumberAvailable = 1 + ds.Status.NumberReady = 1 + ds.Status.NumberUnavailable = 0 + err := k8sClient.Status().Update(ctx, ds) + Expect(err).NotTo(HaveOccurred()) + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: ds.GetObjectMeta().GetName() + "1", Namespace: ds.GetNamespace(), Labels: ds.Spec.Template.GetObjectMeta().GetLabels()}, + Spec: corev1.PodSpec{Containers: []corev1.Container{ + {Name: "n1", Image: "necoperf"}, + }, + }, + } + pod.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(ds, appsv1.SchemeGroupVersion.WithKind("DaemonSet"))} + err = k8sClient.Create(ctx, pod) + Expect(err).NotTo(HaveOccurred()) + + updatePodStatus(ctx, pod, "containerd://necoperf", "necoperf-daemon", "10.244.1.3", + corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}) +} From 735f667c694f2a0a5166cd9447e98aa86e4f2412 Mon Sep 17 00:00:00 2001 From: zeroalphat Date: Tue, 17 Oct 2023 00:12:32 +0000 Subject: [PATCH 02/13] Fix typo in discovery.go Signed-off-by: zeroalphat --- internal/resource/discovery.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/resource/discovery.go b/internal/resource/discovery.go index 7d44d8a..c386342 100644 --- a/internal/resource/discovery.go +++ b/internal/resource/discovery.go @@ -56,7 +56,7 @@ func (d *Discovery) GetPodList(ctx context.Context, necoperfNS string) (*corev1. } func (d *Discovery) GetContainerID(pod *corev1.Pod, containerName string) (string, error) { - if len(containerName) == 0 && len(pod.Status.Conditions) >= 1 { + if len(containerName) == 0 && len(pod.Spec.Containers) >= 1 { containerName = pod.Spec.Containers[0].Name } From abc8ad62271ad2a30975c65cbb7dcbfe9f4e2365 Mon Sep 17 00:00:00 2001 From: zeroalphat Date: Tue, 17 Oct 2023 00:28:18 +0000 Subject: [PATCH 03/13] Add necoperf-cli command reference Signed-off-by: zeroalphat --- cmd/necoperf-cli/cmd/profile.go | 12 ++++++------ docs/necoperf-cli.md | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 docs/necoperf-cli.md diff --git a/cmd/necoperf-cli/cmd/profile.go b/cmd/necoperf-cli/cmd/profile.go index 4e51c49..a7be637 100644 --- a/cmd/necoperf-cli/cmd/profile.go +++ b/cmd/necoperf-cli/cmd/profile.go @@ -34,7 +34,7 @@ func isPodReady(pod *corev1.Pod) bool { func NewProfileCommand() *cobra.Command { cmd := &cobra.Command{ Use: "profile", - Short: "Exec perf profile of the target container", + Short: "Perform CPU profiling on the target container", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { config.podName = args[0] @@ -88,11 +88,11 @@ func NewProfileCommand() *cobra.Command { SilenceUsage: true, SilenceErrors: true, } - cmd.Flags().StringVar(&config.namespace, "namespace", "default", "kubernetes namespace") - cmd.Flags().StringVar(&config.outputDir, "outputDir", "/tmp", "output data directory") - cmd.Flags().StringVar(&config.containerName, "container", "", "container name") - cmd.Flags().StringVar(&config.necoperfNS, "necoperf-namespace", "kube-system", "necoperf namespace") - cmd.Flags().DurationVar(&config.timeout, "timeout", time.Second*30, "timeout seconds") + cmd.Flags().StringVar(&config.necoperfNS, "necoperf-namespace", "necoperf", "Namespace in which necoperf-daemon is running") + cmd.Flags().StringVarP(&config.namespace, "namespace", "n", "default", "Namespace in pod being profiled is running") + cmd.Flags().StringVar(&config.containerName, "container", "", "Specify the container name to profile") + cmd.Flags().DurationVar(&config.timeout, "timeout", 30*time.Second, "Time to run cpu profiling on server") + cmd.Flags().StringVar(&config.outputDir, "outputDir", "/tmp", "Directory to output profiling result") return cmd } diff --git a/docs/necoperf-cli.md b/docs/necoperf-cli.md new file mode 100644 index 0000000..1a75e91 --- /dev/null +++ b/docs/necoperf-cli.md @@ -0,0 +1,19 @@ +# necoperf-cli command reference + +```console +necoperf-cli args... +``` + +- [`necoperf-cli profile PODNAME`](#necoperf-cli-profile-podname) + +## `necoperf-cli profile PODNAME` + +Perform profiling for container on pod. + +| Option | Default value |Description | +|:-------|:--------------|:-----------| +| `--necoperf-namespace`|`necoperf`| Namespace in which necoperf-daemon is running| +| `-n`,`--namespace` | `default` | Namespace in which the pod being profiled is running | +| `--container` ||Specify the container name to profile. If no container name is specified, the first container of the pod is set as the target of profiling.| +| `--timeout` |`30s`| Time to run cpu profiling on server| +| `--outputDir` |`/tmp`|Directory for output of profiling results| From 2eece1529940707295536672b5dd94ea96684b18 Mon Sep 17 00:00:00 2001 From: zeroalphat Date: Tue, 17 Oct 2023 00:32:05 +0000 Subject: [PATCH 04/13] Change context used to background Signed-off-by: zeroalphat --- cmd/necoperf-cli/cmd/profile.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/necoperf-cli/cmd/profile.go b/cmd/necoperf-cli/cmd/profile.go index a7be637..63206ff 100644 --- a/cmd/necoperf-cli/cmd/profile.go +++ b/cmd/necoperf-cli/cmd/profile.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "log/slog" "os" @@ -49,7 +50,7 @@ func NewProfileCommand() *cobra.Command { return err } - ctx := cmd.Context() + ctx := context.Background() pod, err := client.Discovery.GetPod(ctx, config.namespace, config.podName) if err != nil { return err From 849fd37f2492c6a30380b5228d9f811c4ecf61ee Mon Sep 17 00:00:00 2001 From: zeroalphat Date: Tue, 17 Oct 2023 01:57:33 +0000 Subject: [PATCH 05/13] Change to return discovery object directly Signed-off-by: zeroalphat --- cmd/necoperf-cli/cmd/profile.go | 11 ++++++----- internal/client/client.go | 18 ++++++++---------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/cmd/necoperf-cli/cmd/profile.go b/cmd/necoperf-cli/cmd/profile.go index 63206ff..4d5bb40 100644 --- a/cmd/necoperf-cli/cmd/profile.go +++ b/cmd/necoperf-cli/cmd/profile.go @@ -46,29 +46,30 @@ func NewProfileCommand() *cobra.Command { if err != nil { return err } - if err := client.SetupDiscovery(); err != nil { + ds, err := client.SetupDiscovery() + if err != nil { return err } ctx := context.Background() - pod, err := client.Discovery.GetPod(ctx, config.namespace, config.podName) + pod, err := ds.GetPod(ctx, config.namespace, config.podName) if err != nil { return err } if !isPodReady(pod) { return fmt.Errorf("pod %s is not ready", pod.Name) } - containerID, err := client.Discovery.GetContainerID(pod, config.containerName) + containerID, err := ds.GetContainerID(pod, config.containerName) if err != nil { return err } logger.Info("get container id", "podName", config.podName, "containerID", containerID) - pods, err := client.Discovery.GetPodList(ctx, config.necoperfNS) + pods, err := ds.GetPodList(ctx, config.necoperfNS) if err != nil { return err } - addr, err := client.Discovery.DiscoveryServerAddr(pods, pod.Status.HostIP) + addr, err := ds.DiscoveryServerAddr(pods, pod.Status.HostIP) if err != nil { return err } diff --git a/internal/client/client.go b/internal/client/client.go index 2cbd52e..25f7802 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -20,10 +20,9 @@ import ( ) type Client struct { - logger *slog.Logger - client rpc.NecoPerfClient - Discovery *resource.Discovery - Timeout time.Duration + logger *slog.Logger + client rpc.NecoPerfClient + Timeout time.Duration } // https://github.com/grpc-ecosystem/go-grpc-middleware/blob/main/interceptors/logging/examples/slog/example_test.go @@ -93,24 +92,23 @@ func (c *Client) Profile(ctx context.Context, podName, containerID, DataDir stri return nil } -func (c *Client) SetupDiscovery() error { +func (c *Client) SetupDiscovery() (*resource.Discovery, error) { config, err := config.GetConfig() if err != nil { - return err + return nil, err } k8sClient, err := client.New(config, client.Options{}) if err != nil { - return err + return nil, err } d, err := resource.NewDiscovery(c.logger, k8sClient) if err != nil { - return err + return nil, err } - c.Discovery = d - return nil + return d, nil } func (c *Client) SetupGrpcClient(addr string) error { From 9bff82ad72e1dd0347d56b30b2987953df2e289f Mon Sep 17 00:00:00 2001 From: zeroalphat Date: Tue, 17 Oct 2023 05:51:25 +0000 Subject: [PATCH 06/13] Change port number to be configurable Signed-off-by: zeroalphat --- cmd/necoperf-daemon/cmd/daemon.go | 3 +- internal/constants/constants.go | 5 ++ internal/resource/discovery.go | 29 ++++++---- internal/resource/discovery_test.go | 27 ++++++++-- internal/resource/suite_test.go | 82 +++++++++++++++++++---------- 5 files changed, 104 insertions(+), 42 deletions(-) diff --git a/cmd/necoperf-daemon/cmd/daemon.go b/cmd/necoperf-daemon/cmd/daemon.go index c4588a7..7e71d86 100644 --- a/cmd/necoperf-daemon/cmd/daemon.go +++ b/cmd/necoperf-daemon/cmd/daemon.go @@ -4,6 +4,7 @@ import ( "log/slog" "os" + "github.com/cybozu-go/necoperf/internal/constants" "github.com/cybozu-go/necoperf/internal/daemon" "github.com/spf13/cobra" ) @@ -29,7 +30,7 @@ func NewDaemonCommand() *cobra.Command { return daemon.Start() }, } - cmd.Flags().IntVar(&port, "port", 6543, "Set server port number") + cmd.Flags().IntVar(&port, "port", constants.NecoPerfGrpcServerPort, "Set server port number") cmd.Flags().StringVar(&runtimeEndpoint, "runtime-endpoint", "unix:///run/containerd/containerd.sock", "Set container runtime endpoint") cmd.Flags().StringVar(&workDir, "work-dir", "/var/necoperf", "Set working directory") diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 62196f0..8b8e28f 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -13,3 +13,8 @@ const ( LabelAppName = "app.kubernetes.io/name" AppNameNecoPerf = "necoperf-daemon" ) + +const ( + NecoPerfGrpcServerPort = 6543 + NecoperfGrpcPortName = "necoperf-grpc" +) diff --git a/internal/resource/discovery.go b/internal/resource/discovery.go index c386342..fa26252 100644 --- a/internal/resource/discovery.go +++ b/internal/resource/discovery.go @@ -13,10 +13,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - port = 6543 -) - type Discovery struct { logger *slog.Logger client client.Client @@ -72,18 +68,31 @@ func (d *Discovery) GetContainerID(pod *corev1.Pod, containerName string) (strin } func (d *Discovery) DiscoveryServerAddr(pods *corev1.PodList, hostIP string) (string, error) { - var podIP string - for _, pod := range pods.Items { - if pod.Status.HostIP == hostIP { - podIP = pod.Status.PodIP + var podIP, addr string + var pod *corev1.Pod + + for _, p := range pods.Items { + if p.Status.HostIP == hostIP { + pod = &p + break } } + podIP = pod.Status.PodIP if len(podIP) == 0 { return "", errors.New("failed to get pod IP") } - addr := fmt.Sprintf("%s:%s", podIP, strconv.Itoa(port)) - d.logger.Info("found the pod IP of necoperf", "hostIP", hostIP, "podIP", podIP) + for _, c := range pod.Spec.Containers { + for _, p := range c.Ports { + if p.Name == constants.NecoperfGrpcPortName { + addr = fmt.Sprintf("%s:%s", podIP, strconv.Itoa(int(p.ContainerPort))) + } + } + } + if len(addr) == 0 { + addr = fmt.Sprintf("%s:%s", podIP, strconv.Itoa(constants.NecoPerfGrpcServerPort)) + } + d.logger.Info("found the address of necoperf grpc server", "hostIP", hostIP, "addr", addr) return addr, nil } diff --git a/internal/resource/discovery_test.go b/internal/resource/discovery_test.go index 24dddd3..fbbeac6 100644 --- a/internal/resource/discovery_test.go +++ b/internal/resource/discovery_test.go @@ -2,7 +2,9 @@ package resource import ( "context" + "fmt" + "github.com/cybozu-go/necoperf/internal/constants" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -23,16 +25,33 @@ var _ = Describe("Test Discovery", func() { By("get test pod") pod, err := d.GetPod(ctx, "test", "test-pod") Expect(err).NotTo(HaveOccurred()) - Expect(pod.Status.HostIP).NotTo(BeEmpty()) + Expect(pod.Status.HostIP).To(Equal(HostIP)) - By("get necoperf daemonset") - podList, err := d.GetPodList(ctx, "test-for-necoperf") + By("get test daemonset") + podList, err := d.GetPodList(ctx, "test-default-port") Expect(err).NotTo(HaveOccurred()) Expect(len(podList.Items)).To(Equal(1)) By("discovery server addr") addr, err := d.DiscoveryServerAddr(podList, pod.Status.HostIP) Expect(err).NotTo(HaveOccurred()) - Expect(addr).NotTo(BeEmpty()) + Expect(addr).To(Equal(fmt.Sprintf("%s:%d", daemonsetPodIP, constants.NecoPerfGrpcServerPort))) + }) + + It("should get the port specified by the server when the server specifies a port other than the default port", func() { + By("get test pod") + pod, err := d.GetPod(ctx, "test", "test-pod") + Expect(err).NotTo(HaveOccurred()) + Expect(pod.Status.HostIP).To(Equal(HostIP)) + + By("get specified port daemonset") + podList, err := d.GetPodList(ctx, "test-specified-port") + Expect(err).NotTo(HaveOccurred()) + Expect(len(podList.Items)).To(Equal(1)) + + By("discovery server addr") + addr, err := d.DiscoveryServerAddr(podList, pod.Status.HostIP) + Expect(err).NotTo(HaveOccurred()) + Expect(addr).To(Equal(fmt.Sprintf("%s:%d", daemonsetPodIP, 8080))) }) }) diff --git a/internal/resource/suite_test.go b/internal/resource/suite_test.go index fb7f509..df36771 100644 --- a/internal/resource/suite_test.go +++ b/internal/resource/suite_test.go @@ -2,6 +2,7 @@ package resource import ( "context" + "fmt" "log/slog" "testing" "time" @@ -28,7 +29,8 @@ var testThreshold time.Duration var d *Discovery const ( - HostIP = "10.69.0.197" + HostIP = "10.69.0.197" + daemonsetPodIP = "10.224.1.3" ) func TestDiscovery(t *testing.T) { @@ -71,34 +73,61 @@ var _ = BeforeSuite(func() { Containers: []corev1.Container{ { Name: "t1", - Image: "necoperf", + Image: "necoperf-cli", }, }, }, } + testPodContainerStatus := corev1.ContainerStatus{ + Name: "t1", + ContainerID: "containerd://t1", + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + } + + //create test namespace and pod + err = k8sClient.Create(ctx, &testNamespace) + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Create(ctx, &testPod) + Expect(err).NotTo(HaveOccurred()) + updatePodStatus(ctx, &testPod, "10.244.1.200", testPodContainerStatus) necoperfNamespace := corev1.Namespace{} - necoperfNamespace.Name = "test-for-necoperf" + necoperfNamespace.Name = "test-default-port" testDaemonset := appsv1.DaemonSet{} - testDaemonset.Namespace = "test-for-necoperf" - testDaemonset.ObjectMeta.Name = "necoperf-daemon" + testDaemonset.Namespace = necoperfNamespace.Name + testDaemonset.ObjectMeta.Name = "necoperf-daemon-default-port" testDaemonset.ObjectMeta.Labels = map[string]string{constants.LabelAppName: constants.AppNameNecoPerf} testDaemonset.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{constants.LabelAppName: constants.AppNameNecoPerf}} testDaemonset.Spec.Template.ObjectMeta.Labels = map[string]string{constants.LabelAppName: constants.AppNameNecoPerf} - testDaemonset.Spec.Template.Spec.Containers = []corev1.Container{{Name: "n1", Image: "necoperf"}} - - err = k8sClient.Create(ctx, &testNamespace) - Expect(err).NotTo(HaveOccurred()) - err = k8sClient.Create(ctx, &testPod) - Expect(err).NotTo(HaveOccurred()) - updatePodStatus(ctx, &testPod, "containerd://necoperf", "t1", "10.244.1.2", - corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}) + testDaemonsetContainer := corev1.Container{Name: "n1", Image: "necoperf-daemon", Ports: []corev1.ContainerPort{{ + Name: "default-port", ContainerPort: constants.NecoPerfGrpcServerPort}}} + testDaemonset.Spec.Template.Spec.Containers = []corev1.Container{testDaemonsetContainer} + //create test daemonset err = k8sClient.Create(ctx, &necoperfNamespace) Expect(err).NotTo(HaveOccurred()) err = k8sClient.Create(ctx, &testDaemonset) Expect(err).NotTo(HaveOccurred()) - updateDaemonSetStatus(ctx, &testDaemonset) + updateDaemonSetStatus(ctx, &testDaemonset, testDaemonsetContainer) + + specifiedPortNamespace := corev1.Namespace{} + specifiedPortNamespace.Name = "test-specified-port" + specifiedPortDaemonset := appsv1.DaemonSet{} + specifiedPortDaemonset.Namespace = specifiedPortNamespace.Name + specifiedPortDaemonset.ObjectMeta.Name = "necoperf-daemon-specified-port" + specifiedPortDaemonset.ObjectMeta.Labels = map[string]string{constants.LabelAppName: constants.AppNameNecoPerf} + specifiedPortDaemonset.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{constants.LabelAppName: constants.AppNameNecoPerf}} + specifiedPortDaemonset.Spec.Template.ObjectMeta.Labels = map[string]string{constants.LabelAppName: constants.AppNameNecoPerf} + specifiedPortDaemonsetContainer := corev1.Container{Name: "n1", Image: "necoperf-daemon", Ports: []corev1.ContainerPort{{ + Name: constants.NecoperfGrpcPortName, ContainerPort: 8080}}} + specifiedPortDaemonset.Spec.Template.Spec.Containers = []corev1.Container{specifiedPortDaemonsetContainer} + + //create necoperf namespace and daemonset + err = k8sClient.Create(ctx, &specifiedPortNamespace) + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Create(ctx, &specifiedPortDaemonset) + Expect(err).NotTo(HaveOccurred()) + updateDaemonSetStatus(ctx, &specifiedPortDaemonset, specifiedPortDaemonsetContainer) d, err = NewDiscovery(slog.Default(), k8sClient) Expect(err).NotTo(HaveOccurred()) @@ -111,13 +140,9 @@ var _ = AfterSuite(func() { Expect(err).NotTo(HaveOccurred()) }) -func updatePodStatus(ctx context.Context, pod *corev1.Pod, containerID, name, podIP string, state corev1.ContainerState) { +func updatePodStatus(ctx context.Context, pod *corev1.Pod, podIP string, containerStatus corev1.ContainerStatus) { pod.Status.ContainerStatuses = []corev1.ContainerStatus{ - { - ContainerID: containerID, - Name: name, - State: state, - }, + containerStatus, } pod.Status.HostIP = HostIP pod.Status.PodIP = podIP @@ -125,7 +150,7 @@ func updatePodStatus(ctx context.Context, pod *corev1.Pod, containerID, name, po Expect(err).NotTo(HaveOccurred()) } -func updateDaemonSetStatus(ctx context.Context, ds *appsv1.DaemonSet) { +func updateDaemonSetStatus(ctx context.Context, ds *appsv1.DaemonSet, container corev1.Container) { ds.Status.DesiredNumberScheduled = 1 ds.Status.NumberAvailable = 1 ds.Status.NumberReady = 1 @@ -135,15 +160,18 @@ func updateDaemonSetStatus(ctx context.Context, ds *appsv1.DaemonSet) { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: ds.GetObjectMeta().GetName() + "1", Namespace: ds.GetNamespace(), Labels: ds.Spec.Template.GetObjectMeta().GetLabels()}, - Spec: corev1.PodSpec{Containers: []corev1.Container{ - {Name: "n1", Image: "necoperf"}, - }, - }, + Spec: corev1.PodSpec{Containers: []corev1.Container{container}}, } + pod.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(ds, appsv1.SchemeGroupVersion.WithKind("DaemonSet"))} err = k8sClient.Create(ctx, pod) Expect(err).NotTo(HaveOccurred()) - updatePodStatus(ctx, pod, "containerd://necoperf", "necoperf-daemon", "10.244.1.3", - corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}) + containerStatus := corev1.ContainerStatus{ + Name: container.Name, + ContainerID: fmt.Sprintf("containerd://%s", container.Name), + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + } + + updatePodStatus(ctx, pod, daemonsetPodIP, containerStatus) } From ee2e99eba87b206dee928d7ab3a516249d686658 Mon Sep 17 00:00:00 2001 From: zeroalphat Date: Wed, 18 Oct 2023 00:35:10 +0000 Subject: [PATCH 07/13] Fix problem with no help message when wrong arguments were passed Signed-off-by: zeroalphat --- cmd/necoperf-cli/cmd/profile.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/necoperf-cli/cmd/profile.go b/cmd/necoperf-cli/cmd/profile.go index 4d5bb40..5b8b58e 100644 --- a/cmd/necoperf-cli/cmd/profile.go +++ b/cmd/necoperf-cli/cmd/profile.go @@ -38,6 +38,7 @@ func NewProfileCommand() *cobra.Command { Short: "Perform CPU profiling on the target container", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true config.podName = args[0] handler := slog.NewTextHandler(os.Stderr, nil) logger := slog.New(handler) @@ -87,8 +88,6 @@ func NewProfileCommand() *cobra.Command { return nil }, - SilenceUsage: true, - SilenceErrors: true, } cmd.Flags().StringVar(&config.necoperfNS, "necoperf-namespace", "necoperf", "Namespace in which necoperf-daemon is running") cmd.Flags().StringVarP(&config.namespace, "namespace", "n", "default", "Namespace in pod being profiled is running") From 9df1ce096ee144b99e1fbb80e03eaf94bc7e6e2b Mon Sep 17 00:00:00 2001 From: zeroalphat Date: Wed, 18 Oct 2023 00:48:56 +0000 Subject: [PATCH 08/13] Add command descriptions Signed-off-by: zeroalphat --- cmd/necoperf-cli/cmd/profile.go | 1 + cmd/necoperf-cli/cmd/root.go | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/necoperf-cli/cmd/profile.go b/cmd/necoperf-cli/cmd/profile.go index 5b8b58e..a2e2f85 100644 --- a/cmd/necoperf-cli/cmd/profile.go +++ b/cmd/necoperf-cli/cmd/profile.go @@ -36,6 +36,7 @@ func NewProfileCommand() *cobra.Command { cmd := &cobra.Command{ Use: "profile", Short: "Perform CPU profiling on the target container", + Long: "Perform CPU profiling on the target container", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { cmd.SilenceUsage = true diff --git a/cmd/necoperf-cli/cmd/root.go b/cmd/necoperf-cli/cmd/root.go index f6fbd93..be5aa81 100644 --- a/cmd/necoperf-cli/cmd/root.go +++ b/cmd/necoperf-cli/cmd/root.go @@ -8,7 +8,9 @@ import ( func NewRootCommand() *cobra.Command { cmd := &cobra.Command{ - Use: "necoperf-cli", + Use: "necoperf-cli", + Short: "necoperf-cli is a command line tool for necoperf", + Long: "necoperf-cli is a command line tool for necoperf", RunE: func(cmd *cobra.Command, args []string) error { return cmd.Help() }, From 58def590663f9e66c8612b2b4de71078371267a8 Mon Sep 17 00:00:00 2001 From: zeroalphat Date: Thu, 19 Oct 2023 05:29:08 +0000 Subject: [PATCH 09/13] Add a mechanism to determine if an object is nil or not Signed-off-by: zeroalphat --- internal/resource/discovery.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/resource/discovery.go b/internal/resource/discovery.go index fa26252..9f1ccb4 100644 --- a/internal/resource/discovery.go +++ b/internal/resource/discovery.go @@ -77,9 +77,13 @@ func (d *Discovery) DiscoveryServerAddr(pods *corev1.PodList, hostIP string) (st break } } + if pod == nil { + return "", fmt.Errorf("failed to find any necoperf pod on host %q", hostIP) + } + podIP = pod.Status.PodIP if len(podIP) == 0 { - return "", errors.New("failed to get pod IP") + return "", fmt.Errorf("failed to get pod IP from %q", pod.Name) } for _, c := range pod.Spec.Containers { From b6cca1232b4c7fee9deb93e99067b90e0e9edbb5 Mon Sep 17 00:00:00 2001 From: Taichi Takemura Date: Fri, 20 Oct 2023 17:18:02 +0900 Subject: [PATCH 10/13] Remove file existence checks Co-authored-by: morimoto-cybozu --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1ece7d1..7835736 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ setup: envtest envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): mkdir -p bin - test -s $(BIN_DIR)/setup-envtest || GOBIN=$(BIN_DIR) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + GOBIN=$(BIN_DIR) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest $(MDBOOK): mkdir -p bin From d4943ae94d349b52494ed8886f300de5739a0efc Mon Sep 17 00:00:00 2001 From: zeroalphat Date: Fri, 20 Oct 2023 08:22:11 +0000 Subject: [PATCH 11/13] Change outputDir to output-dir Signed-off-by: zeroalphat --- cmd/necoperf-cli/cmd/profile.go | 2 +- docs/necoperf-cli.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/necoperf-cli/cmd/profile.go b/cmd/necoperf-cli/cmd/profile.go index a2e2f85..9926f1d 100644 --- a/cmd/necoperf-cli/cmd/profile.go +++ b/cmd/necoperf-cli/cmd/profile.go @@ -94,7 +94,7 @@ func NewProfileCommand() *cobra.Command { cmd.Flags().StringVarP(&config.namespace, "namespace", "n", "default", "Namespace in pod being profiled is running") cmd.Flags().StringVar(&config.containerName, "container", "", "Specify the container name to profile") cmd.Flags().DurationVar(&config.timeout, "timeout", 30*time.Second, "Time to run cpu profiling on server") - cmd.Flags().StringVar(&config.outputDir, "outputDir", "/tmp", "Directory to output profiling result") + cmd.Flags().StringVar(&config.outputDir, "output-dir", "/tmp", "Directory to output profiling result") return cmd } diff --git a/docs/necoperf-cli.md b/docs/necoperf-cli.md index 1a75e91..6d62621 100644 --- a/docs/necoperf-cli.md +++ b/docs/necoperf-cli.md @@ -16,4 +16,4 @@ Perform profiling for container on pod. | `-n`,`--namespace` | `default` | Namespace in which the pod being profiled is running | | `--container` ||Specify the container name to profile. If no container name is specified, the first container of the pod is set as the target of profiling.| | `--timeout` |`30s`| Time to run cpu profiling on server| -| `--outputDir` |`/tmp`|Directory for output of profiling results| +| `--output-dir` |`/tmp`|Directory for output of profiling results| From bf678355569105ef3c362e1359655ac24193040b Mon Sep 17 00:00:00 2001 From: zeroalphat Date: Fri, 20 Oct 2023 08:23:29 +0000 Subject: [PATCH 12/13] Fix syntax errors in client.go Signed-off-by: zeroalphat --- internal/client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/client/client.go b/internal/client/client.go index 25f7802..7cf2695 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -45,7 +45,7 @@ func (c *Client) Save(dataDir, podName string) (*os.File, error) { return nil, err } - profileFilePath := filepath.Join(dataDir + "/" + podName + ".script") + profileFilePath := filepath.Join(dataDir, podName+".script") f, err := os.Create(profileFilePath) if err != nil { return nil, err From ce09ae582b77c1306a87fdd7d29e16d227e74805 Mon Sep 17 00:00:00 2001 From: zeroalphat Date: Fri, 20 Oct 2023 08:25:05 +0000 Subject: [PATCH 13/13] add short option -c Signed-off-by: zeroalphat --- cmd/necoperf-cli/cmd/profile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/necoperf-cli/cmd/profile.go b/cmd/necoperf-cli/cmd/profile.go index 9926f1d..481bb73 100644 --- a/cmd/necoperf-cli/cmd/profile.go +++ b/cmd/necoperf-cli/cmd/profile.go @@ -92,7 +92,7 @@ func NewProfileCommand() *cobra.Command { } cmd.Flags().StringVar(&config.necoperfNS, "necoperf-namespace", "necoperf", "Namespace in which necoperf-daemon is running") cmd.Flags().StringVarP(&config.namespace, "namespace", "n", "default", "Namespace in pod being profiled is running") - cmd.Flags().StringVar(&config.containerName, "container", "", "Specify the container name to profile") + cmd.Flags().StringVarP(&config.containerName, "container", "c", "", "Specify the container name to profile") cmd.Flags().DurationVar(&config.timeout, "timeout", 30*time.Second, "Time to run cpu profiling on server") cmd.Flags().StringVar(&config.outputDir, "output-dir", "/tmp", "Directory to output profiling result")