From 9aafe95f5e0c63dbe9ee088a826068db2dfab46d Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 30 Oct 2024 14:01:51 -0400 Subject: [PATCH 1/7] chore: add pprof and debug docs (#72) * chore: add pprof and debug docs Signed-off-by: Justin Alvarez * update docs Signed-off-by: Justin Alvarez --------- Signed-off-by: Justin Alvarez --- cmd/finch-daemon/main.go | 30 +++++++++++++++++++++++++--- docs/debug.md | 42 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 docs/debug.md diff --git a/cmd/finch-daemon/main.go b/cmd/finch-daemon/main.go index 9b376596..afdc12dc 100644 --- a/cmd/finch-daemon/main.go +++ b/cmd/finch-daemon/main.go @@ -17,6 +17,10 @@ import ( "syscall" "time" + // #nosec + // register HTTP handler for /debug/pprof on the DefaultServeMux. + _ "net/http/pprof" + "github.com/containerd/containerd" "github.com/containerd/nerdctl/pkg/api/types" "github.com/containerd/nerdctl/pkg/config" @@ -47,9 +51,10 @@ const ( ) type DaemonOptions struct { - debug bool - socketAddr string - socketOwner int + debug bool + socketAddr string + socketOwner int + debugAddress string } var options = new(DaemonOptions) @@ -65,6 +70,7 @@ func main() { rootCmd.Flags().StringVar(&options.socketAddr, "socket-addr", defaultFinchAddr, "server listening Unix socket address") rootCmd.Flags().BoolVar(&options.debug, "debug", false, "turn on debug log level") rootCmd.Flags().IntVar(&options.socketOwner, "socket-owner", -1, "Uid and Gid of the server socket") + rootCmd.Flags().StringVar(&options.debugAddress, "debug-addr", "", "") if err := rootCmd.Execute(); err != nil { log.Printf("got error: %v", err) log.Fatal(err) @@ -99,6 +105,24 @@ func run(options *DaemonOptions) error { if err := os.Chown(options.socketAddr, options.socketOwner, options.socketOwner); err != nil { return fmt.Errorf("failed to chown the finch-daemon socket: %w", err) } + + if options.debugAddress != "" { + logger.Infof("Serving debugging endpoint on %q", options.debugAddress) + go func() { + debugListener, err := net.Listen("tcp", options.debugAddress) + if err != nil { + logger.Fatal(err) + } + debugServer := &http.Server{ + Handler: http.DefaultServeMux, + ReadHeaderTimeout: 5 * time.Second, + } + if err := debugServer.Serve(debugListener); err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.Fatal(err) + } + }() + } + server := &http.Server{ Handler: r, ReadHeaderTimeout: 5 * time.Minute, diff --git a/docs/debug.md b/docs/debug.md new file mode 100644 index 00000000..cb5f9320 --- /dev/null +++ b/docs/debug.md @@ -0,0 +1,42 @@ +# Debugging the finch daemon + +This document outlines where to find/access logs and how to configure profiling tools for finch daemon. + +## Logs + +Logs are the first place to check when you suspect a problem with finch-daemon. If `finch-daemon` was started via `systemd` then you can obtain logs using `journalctl`: + +```shell +sudo journalctl -u finch +``` + +> **Note** +> The command above assumes that you have used the unit file definition [finch.service](../finch.service) we have provided. If you have created your own unit file for `finch-daemon` and replace `finch-daemon` with the one you have made. Amazon Linux distributions of Finch also use the name `finch` for the finch-daemon service. + +If you have started `finch-daemon` manually, logs will either be emitted to stderr/stdout. + +## CPU Profiling + +We can use Golangs `pprof` tool to profile the daemon. To enable profiling you must set the `--debug-addr` CLI parameter when invoking `finch-daemon`: + +```shell +./finch-daemon --debug-addr localhost:6060 +``` + +> **Note** +> Similarly to adding the command line option for a local run of finch-daemon, any systemd service file can also be modified to include the `--debug-addr` option. + + +Once you have configured the debug address you can send a `GET` to the `/debug/pprof/profile` endpoint to receive a CPU profile of the daemon. You can specify an optional argument `seconds` to limit the results to a certain time span: + +```shell +curl http://localhost:6060/debug/pprof/profile?seconds=40 > out.pprof +``` + +You can use the `pprof` tool provided by the Go CLI to visualize the data within a web browser: + +```shell +go tool pprof -http=:8080 out.pprof +``` + +For more information on pprof, [see its documentation here](https://pkg.go.dev/net/http/pprof). From 7d7afddb73be94f4f8f142ce4b4c618d5a474942 Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Wed, 2 Oct 2024 22:33:18 +0000 Subject: [PATCH 2/7] fix: Make release binaries static Signed-off-by: Arjun Raja Yogidas --- .github/workflows/release-automation.yaml | 19 ++++++++++++++----- Makefile | 11 +++++++++-- scripts/create-releases.sh | 9 +++++++++ scripts/verify-release-artifacts.sh | 2 +- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release-automation.yaml b/.github/workflows/release-automation.yaml index fcb0a269..2217b5c3 100644 --- a/.github/workflows/release-automation.yaml +++ b/.github/workflows/release-automation.yaml @@ -22,7 +22,7 @@ jobs: uses: "WyriHaximus/github-action-get-previous-tag@04e8485ecb6487243907e330d522ff60f02283ce" # v1.4.0 generate-artifacts: needs: get-latest-tag - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 env: # Set during setup. RELEASE_TAG: ${{ needs.get-latest-tag.outputs.tag }} @@ -43,22 +43,29 @@ jobs: export release_tag=${{ env.RELEASE_TAG }} export release_version=${release_tag/v/} # Remove v from tag name echo "DYNAMIC_BINARY_NAME=finch-daemon-${release_version}-linux-amd64.tar.gz" >> $GITHUB_ENV + echo "STATIC_BINARY_NAME=finch-daemon-${release_version}-linux-amd64-static.tar.gz" >> $GITHUB_ENV mkdir release - name: Install Go licenses run: go install github.com/google/go-licenses@latest - name: Create Third Party Licences File run: make licenses + - name: setup static dependecies + run: | + sudo apt-get update + sudo apt-get install libc6-dev -f - name: Create release binaries run: make RELEASE_TAG=${{ env.RELEASE_TAG }} release - name: Verify Release version run: | - mkdir output - tar -xzf release/${{ env.DYNAMIC_BINARY_NAME }} -C ./output - BINARY_VERSION=$(./output/finch-daemon --version | grep -oP '\d+\.\d+\.\d+') + mkdir -p output/static output/dynamic + tar -xzf release/${{ env.DYNAMIC_BINARY_NAME }} -C ./output/dynamic + tar -xzf release/${{ env.STATIC_BINARY_NAME }} -C ./output/static + DYNAMIC_BINARY_VERSION=$(./output/dynamic/finch-daemon --version | grep -oP '\d+\.\d+\.\d+') + STATIC_BINARY_VERSION=$(./output/static/finch-daemon --version | grep -oP '\d+\.\d+\.\d+') export release_tag=${{ env.RELEASE_TAG }} export release_version=${release_tag/v/} - if ["$BINARY_VERSION" != "$release_version"]; then + if ["$STATIC_BINARY_VERSION" != "$release_version"] || ["$DYNAMIC_BINARY_VERSION" != "$release_version"]; then echo "Version mismatch" exit 1 fi @@ -98,3 +105,5 @@ jobs: files: | ${{ needs.generate-artifacts.outputs.dynamic_binary_name }} ${{ needs.generate-artifacts.outputs.dynamic_binary_name }}.sha256sum + ${{ needs.generate-artifacts.outputs.static_binary_name }} + ${{ needs.generate-artifacts.outputs.static_binary_name }}.sha256sum diff --git a/Makefile b/Makefile index e81e4146..1e97e7ad 100644 --- a/Makefile +++ b/Makefile @@ -22,8 +22,15 @@ build: $(eval PACKAGE := github.com/runfinch/finch-daemon) $(eval VERSION ?= $(shell git describe --match 'v[0-9]*' --dirty='.modified' --always --tags)) $(eval GITCOMMIT := $(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi)) - $(eval LDFLAGS := "-X $(PACKAGE)/version.Version=$(VERSION) -X $(PACKAGE)/version.GitCommit=$(GITCOMMIT) $(EXTRA_LDFLAGS)") - GOOS=linux go build -ldflags $(LDFLAGS) -v -o $(BINARY) $(PACKAGE)/cmd/finch-daemon +ifneq ($(STATIC),) + $(eval GO_BUILDTAGS := osusergo netgo) + $(eval LDFLAGS := "-X $(PACKAGE)/version.Version=$(VERSION) -X $(PACKAGE)/version.GitCommit=$(GITCOMMIT) -extldflags '-static'") + @echo "Building Static Binary" +else + @echo "Building Dynamic Binary" + $(eval LDFLAGS := "-X $(PACKAGE)/version.Version=$(VERSION) -X $(PACKAGE)/version.GitCommit=$(GITCOMMIT)") +endif + GOOS=linux go build $(if $(GO_BUILDTAGS), -tags "$(GO_BUILDTAGS)") -ldflags $(LDFLAGS) $(if $(STATIC), ) -v -o $(BINARY) $(PACKAGE)/cmd/finch-daemon clean: @rm -f $(BINARIES) diff --git a/scripts/create-releases.sh b/scripts/create-releases.sh index d5d1bf9f..93c164d8 100755 --- a/scripts/create-releases.sh +++ b/scripts/create-releases.sh @@ -54,6 +54,7 @@ fi release_version=${1/v/} # Remove v from tag name dynamic_binary_name=finch-daemon-${release_version}-linux-${ARCH}.tar.gz +static_binary_name=finch-daemon-${release_version}-linux-${ARCH}-static.tar.gz make build cp "$LICENSE_FILE" "${OUT_DIR}" @@ -62,6 +63,14 @@ tar -czvf "$RELEASE_DIR"/"$dynamic_binary_name" -- * popd rm -rf "{$OUT_DIR:?}"/* +STATIC=1 make build +cp "$LICENSE_FILE" "${OUT_DIR}" +pushd "$OUT_DIR" +tar -czvf "$RELEASE_DIR"/"$static_binary_name" -- * +popd +rm -rf "{$OUT_DIR:?}"/* + pushd "$RELEASE_DIR" sha256sum "$dynamic_binary_name" > "$RELEASE_DIR"/"$dynamic_binary_name".sha256sum +sha256sum "$static_binary_name" > "$RELEASE_DIR"/"$static_binary_name".sha256sum popd diff --git a/scripts/verify-release-artifacts.sh b/scripts/verify-release-artifacts.sh index f194e4d4..d90a9d35 100755 --- a/scripts/verify-release-artifacts.sh +++ b/scripts/verify-release-artifacts.sh @@ -44,7 +44,7 @@ release_tag=$1 release_version=${release_tag/v/} pushd "$release_dir" || exit 1 -tarballs=("finch-daemon-${release_version}-linux-${arch}.tar.gz") +tarballs=("finch-daemon-${release_version}-linux-${arch}.tar.gz" "finch-daemon-${release_version}-linux-${arch}-static.tar.gz") expected_contents=("finch-daemon" "THIRD_PARTY_LICENSES") release_is_valid=true From ad3e17ced0a46d6e406e6a592e708b3ca2df58b5 Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Sun, 13 Oct 2024 04:03:57 +0000 Subject: [PATCH 3/7] fix: Update makefile for build target Signed-off-by: Arjun Raja Yogidas --- Makefile | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 1e97e7ad..6f3e42f4 100644 --- a/Makefile +++ b/Makefile @@ -13,24 +13,23 @@ BINDIR ?= $(PREFIX)/bin BINARY = $(addprefix bin/,finch-daemon) -ifndef GODEBUG - EXTRA_LDFLAGS += -s -w +PACKAGE := github.com/runfinch/finch-daemon +VERSION := $(shell git describe --match 'v[0-9]*' --dirty='.modified' --always --tags) +GITCOMMIT := $(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi) + +LDFLAGS := -X $(PACKAGE)/version.Version=$(VERSION) -X $(PACKAGE)/version.GitCommit=$(GITCOMMIT) + +ifeq ($(STATIC),) + GO_BUILDTAGS := osusergo netgo + LDFLAGS += -extldflags '-static' + $(info Building Static Binary) +else + $(info Building Dynamic Binary) endif .PHONY: build build: - $(eval PACKAGE := github.com/runfinch/finch-daemon) - $(eval VERSION ?= $(shell git describe --match 'v[0-9]*' --dirty='.modified' --always --tags)) - $(eval GITCOMMIT := $(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi)) -ifneq ($(STATIC),) - $(eval GO_BUILDTAGS := osusergo netgo) - $(eval LDFLAGS := "-X $(PACKAGE)/version.Version=$(VERSION) -X $(PACKAGE)/version.GitCommit=$(GITCOMMIT) -extldflags '-static'") - @echo "Building Static Binary" -else - @echo "Building Dynamic Binary" - $(eval LDFLAGS := "-X $(PACKAGE)/version.Version=$(VERSION) -X $(PACKAGE)/version.GitCommit=$(GITCOMMIT)") -endif - GOOS=linux go build $(if $(GO_BUILDTAGS), -tags "$(GO_BUILDTAGS)") -ldflags $(LDFLAGS) $(if $(STATIC), ) -v -o $(BINARY) $(PACKAGE)/cmd/finch-daemon + GOOS=linux go build $(if $(GO_BUILDTAGS), -tags "$(GO_BUILDTAGS)") -ldflags "$(LDFLAGS)" -v -o $(BINARY) $(PACKAGE)/cmd/finch-daemon clean: @rm -f $(BINARIES) From 5ad86d87150069e8dd0f498422f26687c95b86a5 Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Wed, 23 Oct 2024 18:09:58 +0000 Subject: [PATCH 4/7] ci: update make targets Signed-off-by: Arjun Raja Yogidas --- .github/workflows/release-automation.yaml | 2 +- Makefile | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-automation.yaml b/.github/workflows/release-automation.yaml index 2217b5c3..9c312323 100644 --- a/.github/workflows/release-automation.yaml +++ b/.github/workflows/release-automation.yaml @@ -22,7 +22,7 @@ jobs: uses: "WyriHaximus/github-action-get-previous-tag@04e8485ecb6487243907e330d522ff60f02283ce" # v1.4.0 generate-artifacts: needs: get-latest-tag - runs-on: ubuntu-22.04 + runs-on: ubuntu-20.04 env: # Set during setup. RELEASE_TAG: ${{ needs.get-latest-tag.outputs.tag }} diff --git a/Makefile b/Makefile index 6f3e42f4..6f84bafb 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,14 @@ PACKAGE := github.com/runfinch/finch-daemon VERSION := $(shell git describe --match 'v[0-9]*' --dirty='.modified' --always --tags) GITCOMMIT := $(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi) +ifndef GODEBUG + EXTRA_LDFLAGS += -s -w +endif + LDFLAGS := -X $(PACKAGE)/version.Version=$(VERSION) -X $(PACKAGE)/version.GitCommit=$(GITCOMMIT) +# LDFLAGS += $(EXTRA_LDFLAGS) + ifeq ($(STATIC),) GO_BUILDTAGS := osusergo netgo LDFLAGS += -extldflags '-static' From 429d39f013e88b686762bd34805790ff6da9cf3d Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Thu, 24 Oct 2024 18:20:33 +0000 Subject: [PATCH 5/7] fix: update makefile Signed-off-by: Arjun Raja Yogidas --- Makefile | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 6f84bafb..45cb6070 100644 --- a/Makefile +++ b/Makefile @@ -21,22 +21,22 @@ ifndef GODEBUG EXTRA_LDFLAGS += -s -w endif -LDFLAGS := -X $(PACKAGE)/version.Version=$(VERSION) -X $(PACKAGE)/version.GitCommit=$(GITCOMMIT) - -# LDFLAGS += $(EXTRA_LDFLAGS) +LDFLAGS_BASE := -X $(PACKAGE)/version.Version=$(VERSION) -X $(PACKAGE)/version.GitCommit=$(GITCOMMIT) $(EXTRA_LDFLAGS) +.PHONY: build +build: ifeq ($(STATIC),) - GO_BUILDTAGS := osusergo netgo - LDFLAGS += -extldflags '-static' - $(info Building Static Binary) + @echo "Building Dynamic Binary" + $(eval LDFLAGS := $(LDFLAGS_BASE)) else - $(info Building Dynamic Binary) + @echo "Building Static Binary" + $(eval GO_BUILDTAGS := osusergo netgo) + $(eval LDFLAGS := $(LDFLAGS_BASE) -extldflags '-static') endif - -.PHONY: build -build: - GOOS=linux go build $(if $(GO_BUILDTAGS), -tags "$(GO_BUILDTAGS)") -ldflags "$(LDFLAGS)" -v -o $(BINARY) $(PACKAGE)/cmd/finch-daemon - + GOOS=linux go build \ + $(if $(GO_BUILDTAGS),-tags "$(GO_BUILDTAGS)") \ + -ldflags "$(LDFLAGS)" \ + -v -o $(BINARY) $(PACKAGE)/cmd/finch-daemon clean: @rm -f $(BINARIES) @rm -rf $(BIN) From 3b30ed357cb5fe522f814ad76f37640cf220691e Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Wed, 30 Oct 2024 18:17:40 +0000 Subject: [PATCH 6/7] ci: set CGO_ENABLED=0 Signed-off-by: Arjun Raja Yogidas --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 45cb6070..fa2eeee6 100644 --- a/Makefile +++ b/Makefile @@ -30,10 +30,10 @@ ifeq ($(STATIC),) $(eval LDFLAGS := $(LDFLAGS_BASE)) else @echo "Building Static Binary" - $(eval GO_BUILDTAGS := osusergo netgo) + $(eval GO_BUILDTAGS := netgo) $(eval LDFLAGS := $(LDFLAGS_BASE) -extldflags '-static') endif - GOOS=linux go build \ + CGO_ENABLED=0 GOOS=linux go build \ $(if $(GO_BUILDTAGS),-tags "$(GO_BUILDTAGS)") \ -ldflags "$(LDFLAGS)" \ -v -o $(BINARY) $(PACKAGE)/cmd/finch-daemon From e44eac8890a2b7ded360ab610b649ebc2bc618c8 Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Wed, 30 Oct 2024 18:47:42 +0000 Subject: [PATCH 7/7] ci: simplify build target Signed-off-by: Arjun Raja Yogidas --- Makefile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index fa2eeee6..e6b14b2b 100644 --- a/Makefile +++ b/Makefile @@ -27,16 +27,17 @@ LDFLAGS_BASE := -X $(PACKAGE)/version.Version=$(VERSION) -X $(PACKAGE)/version.G build: ifeq ($(STATIC),) @echo "Building Dynamic Binary" - $(eval LDFLAGS := $(LDFLAGS_BASE)) + CGO_ENABLED=1 GOOS=linux go build \ + -ldflags "$(LDFLAGS_BASE)" \ + -v -o $(BINARY) $(PACKAGE)/cmd/finch-daemon else @echo "Building Static Binary" - $(eval GO_BUILDTAGS := netgo) - $(eval LDFLAGS := $(LDFLAGS_BASE) -extldflags '-static') -endif CGO_ENABLED=0 GOOS=linux go build \ - $(if $(GO_BUILDTAGS),-tags "$(GO_BUILDTAGS)") \ - -ldflags "$(LDFLAGS)" \ + -tags netgo \ + -ldflags "$(LDFLAGS_BASE) -extldflags '-static'" \ -v -o $(BINARY) $(PACKAGE)/cmd/finch-daemon +endif + clean: @rm -f $(BINARIES) @rm -rf $(BIN)