Makefile

export CGO_ENABLED  = 0
export GOTOOLCHAIN ?= local+auto

MODULE   = $(shell $(GO) list -m)
VERSION ?= $(shell git describe --tags --always --dirty --match=v* 2> /dev/null || \
			cat .version 2> /dev/null || echo v0)
PKGS     = $(or $(PKG),$(shell $(GO) list ./...))

GO      = go
PNPM    = $(or $(shell command -v pnpm 2>/dev/null),\
			$(shell command -v corepack 2>/dev/null | sed 's/$$/ pnpm/'),\
			$(error No pnpm command found))
CLANG   = clang
TIMEOUT = 60s
LSFILES = git ls-files -cmo --exclude-standard --
V = 0
Q = $(if $(filter 1,$V),,@)
M = $(shell if [ "$$(tput colors 2> /dev/null || echo 0)" -ge 8 ]; then printf "\033[34;1m▶\033[0m"; else printf "▶"; fi)

GENERATED_JS = \
	console/frontend/node_modules
GENERATED_GO = \
	common/pb/rawflow.pb.go \
	common/pb/rawflow_vtproto.pb.go \
	common/schema/definition_gen.go \
	inlet/flow/input/udp/reuseport_bpfeb.o \
	inlet/flow/input/udp/reuseport_bpfel.o \
	orchestrator/clickhouse/data/asns.csv \
	orchestrator/clickhouse/data/protocols.csv \
	orchestrator/clickhouse/data/tcp.csv \
	orchestrator/clickhouse/data/udp.csv \
	console/filter/parser.go \
	inlet/kafka/loadbalancealgorithm_enumer.go \
	outlet/core/asnprovider_enumer.go \
	outlet/core/netprovider_enumer.go \
	outlet/metadata/provider/snmp/authprotocol_enumer.go \
	outlet/metadata/provider/snmp/privprotocol_enumer.go \
	outlet/metadata/provider/gnmi/ifspeedpathunit_enumer.go \
	console/homepagetopwidget_enumer.go \
	common/kafka/saslmechanism_enumer.go \
	common/remotedatasource/parsertype_enumer.go \
	common/remotedatasource/paginationtype_enumer.go
GENERATED_TEST_GO = \
	common/clickhousedb/mocks/mock_driver.go
GENERATED = \
	$(GENERATED_GO) \
	$(GENERATED_JS) \
	console/data/frontend \
	common/embed/data/embed.zip

.PHONY: all all-indep
BUILD_ARGS =
# For architecture variants, see:
# - https://go.dev/wiki/MinimumRequirements#amd64
# - https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels
# - https://clang.llvm.org/docs/UsersManual.html#x86
# - https://go.dev/wiki/MinimumRequirements#arm64
# - https://github.com/golang/go/issues/60905
# - https://github.com/llvm/llvm-project/blob/main/llvm/unittests/TargetParser/TargetParserTest.cpp#L1077
# - https://en.wikipedia.org/wiki/Comparison_of_ARM_processors
# - https://docs.docker.com/build/building/variables/#pre-defined-build-arguments
# - https://github.com/containerd/platforms/pull/8
all: fmt lint all-indep ; $(info $(M) building executable…) @ ## Build program binary
	$Q env GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) \
         $(if $(filter amd64,$(TARGETARCH)),GOAMD64=$(TARGETVARIANT),\
         $(if $(filter arm64,$(TARGETARCH)),GOARM64=$(if $(findstring .,$(TARGETVARIANT)),$(TARGETVARIANT),$(TARGETVARIANT:%=%.0)),\
         $(if $(filter arm,$(TARGETARCH)),GOARM=$(TARGETVARIANT:v%=%)))) \
	   $(GO) build \
		-tags release,grpcnotrace,validator_novalidatefn \
		-ldflags '-X $(MODULE)/common/helpers.AkvoradoVersion=$(VERSION)' \
		$(BUILD_ARGS) \
		-o bin/$(basename $(MODULE)) ./cmd/akvorado
all-indep: $(GENERATED)

# Tools

BUF         = env GO=$(GO) ./bin/external-tool buf
ENUMER      = go tool enumer
GOIMPORTS   = go tool goimports
GOTESTSUM   = go tool gotestsum
MOCKGEN     = go tool mockgen
PIGEON      = go tool pigeon
REVIVE      = go tool revive
STATICCHECK = go tool staticcheck
WWHRD       = go tool wwhrd

# Generated files

.SUFFIXES:
.DELETE_ON_ERROR:

$(filter %.go, $(GENERATED_GO) $(GENERATED_TEST_GO)): go.mod

common/pb/rawflow.pb.go common/pb/rawflow_vtproto.pb.go &: .buf.gen.yaml common/pb/rawflow.proto ; $(info $(M) compiling protocol buffers $@…)
	$Q $(BUF) generate --template $(PWD)/.buf.gen.yaml --path $(@:.pb.go=.proto)

common/clickhousedb/mocks/mock_driver.go: ; $(info $(M) generate mocks for ClickHouse driver…)
	$Q $(MOCKGEN) -package mocks -build_constraint "!release" -destination $@ \
		github.com/ClickHouse/clickhouse-go/v2/lib/driver Conn,Row,Rows,ColumnType
	$Q touch $@

inlet/flow/input/udp/reuseport_%.o: inlet/flow/input/udp/reuseport_kern.c inlet/flow/input/udp/vmlinux.h ; $(info $(M) generate eBPF program for input/udp ($*)…)
	$Q ! $(CLANG) -print-targets 2> /dev/null | grep -qF $* || \
		 $(CLANG) -O2 -g -Wall -target $* -c $< -o $@

inlet/kafka/loadbalancealgorithm_enumer.go: inlet/kafka/config.go ; $(info $(M) generate enums for LoadBalanceAlgorithm…)
	$Q $(ENUMER) -type=LoadBalanceAlgorithm -text -transform=kebab -trimprefix=LoadBalance inlet/kafka/config.go
outlet/core/asnprovider_enumer.go: outlet/core/config.go ; $(info $(M) generate enums for ASNProvider…)
	$Q $(ENUMER) -type=ASNProvider -text -transform=kebab -trimprefix=ASNProvider outlet/core/config.go
outlet/core/netprovider_enumer.go: outlet/core/config.go ; $(info $(M) generate enums for NetProvider…)
	$Q $(ENUMER) -type=NetProvider -text -transform=kebab -trimprefix=NetProvider outlet/core/config.go
outlet/metadata/provider/snmp/authprotocol_enumer.go: outlet/metadata/provider/snmp/config.go ; $(info $(M) generate enums for AuthProtocol…)
	$Q $(ENUMER) -type=AuthProtocol -text -transform=kebab -trimprefix=AuthProtocol outlet/metadata/provider/snmp/config.go
outlet/metadata/provider/snmp/privprotocol_enumer.go: outlet/metadata/provider/snmp/config.go ; $(info $(M) generate enums for PrivProtocol…)
	$Q $(ENUMER) -type=PrivProtocol -text -transform=kebab -trimprefix=PrivProtocol outlet/metadata/provider/snmp/config.go
outlet/metadata/provider/gnmi/ifspeedpathunit_enumer.go: outlet/metadata/provider/gnmi/config.go ; $(info $(M) generate enums for IfSpeedPathUnit…)
	$Q $(ENUMER) -type=IfSpeedPathUnit -text -transform=kebab -trimprefix=Speed outlet/metadata/provider/gnmi/config.go
console/homepagetopwidget_enumer.go: console/config.go ; $(info $(M) generate enums for HomepageTopWidget…)
	$Q $(ENUMER) -type=HomepageTopWidget -text -json -transform=kebab -trimprefix=HomepageTopWidget console/config.go
common/kafka/saslmechanism_enumer.go: common/kafka/config.go ; $(info $(M) generate enums for SASLMechanism…)
	$Q $(ENUMER) -type=SASLMechanism -text -transform=kebab -trimprefix=SASL common/kafka/config.go
common/remotedatasource/parsertype_enumer.go: common/remotedatasource/config.go ; $(info $(M) generate enums for ParserType…)
	$Q $(ENUMER) -type=ParserType -text -transform=kebab -trimprefix=Parser common/remotedatasource/config.go
common/remotedatasource/paginationtype_enumer.go: common/remotedatasource/config.go ; $(info $(M) generate enums for PaginationType…)
	$Q $(ENUMER) -type=PaginationType -text -transform=kebab -trimprefix=Pagination common/remotedatasource/config.go

common/schema/definition_gen.go: common/schema/definition.go common/schema/definition_gen.sh ; $(info $(M) generate column definitions…)
	$Q ./common/schema/definition_gen.sh > $@
	$Q $(GOIMPORTS) -w $@

console/filter/parser.go: console/filter/parser.peg ; $(info $(M) generate PEG parser for filters…)
	$Q $(PIGEON) -optimize-basic-latin $< > $@

console/frontend/node_modules: console/frontend/package.json console/frontend/pnpm-lock.yaml
console/frontend/node_modules: ; $(info $(M) fetching node modules…)
	$Q (cd console/frontend ; $(PNPM) install --loglevel=error --frozen-lockfile) && touch $@
console/data/frontend: $(GENERATED_JS)
console/data/frontend: $(shell $(LSFILES) console/frontend 2> /dev/null)
console/data/frontend: ; $(info $(M) building console frontend…)
	$Q cd console/frontend && $(PNPM) run --silent build

console/data/docs/98-metrics.md: $(shell git grep -c -l -P 'reporter\.(Counter|Gauge|Summary|Histogram)Opt' '*.go' 2> /dev/null)
console/data/docs/98-metrics.md: $(shell $(LSFILES) cmd/helper)
console/data/docs/98-metrics.md: cmd/helper/data/metrics.tmpl.md ; $(info $(M) generate metric documentation…)
	$Q go run ./cmd/helper metrics --format=markdown > $@

ASNS_URL = https://vincentbernat.github.io/asn2org/asns.csv
PROTOCOLS_URL = http://www.iana.org/assignments/protocol-numbers/protocol-numbers-1.csv
SERVICES_URL = https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.csv
define caturl
$(if $(filter http://% https://%, $(1)),curl --retry 3 --no-progress-meter --location --fail $(1),cat $(1))
endef

orchestrator/clickhouse/data/asns.csv: ; $(info $(M) generate ASN map…)
	$Q $(call caturl,$(ASNS_URL)) | sed 's|,[^,]*$$||' > $@
	$Q test -s $@
orchestrator/clickhouse/data/protocols.csv: ; $(info $(M) generate protocol map…)
	$Q $(call caturl,$(PROTOCOLS_URL)) \
		| sed -nE -e "1 s/.*/proto,name,description/p" -e "2,$$ s/^([0-9]+,[^ ,]+,[^\",]+),.*/\1/p" \
		> $@
	$Q test -s $@
orchestrator/clickhouse/data/udp.csv orchestrator/clickhouse/data/tcp.csv: orchestrator/clickhouse/data/%.csv: ; $(info $(M) generate $* port numbers…)
	$Q $(call caturl,$(SERVICES_URL)) \
		| sed -nE -e "1 s/.*/port,name/p" -e "2,$$ s/^([^,]+),([0-9]+),$*,.*/\2,\1/p" \
		| awk -F',' '!seen[$$1]++' \
		> $@
	$Q test -s $@

changelog.md: docs/99-changelog.md # To be used by GitHub actions only.
	$Q >  $@ < docs/99-changelog.md \
		sed -n '/^## '$${GITHUB_REF##*/v}' -/,/^## /{//!p}'
	$Q >> $@ echo "**Docker image**: \`docker pull quay.io/$${GITHUB_REPOSITORY}:$${GITHUB_REF##*/v}\`"
	$Q >> $@ echo "**Full changelog**: https://github.com/$${GITHUB_REPOSITORY}/compare/v$$(< docs/99-changelog.md sed -n '/^## '$${GITHUB_REF##*/v}' -/,/^## /{s/^## \([0-9.a-z-]*\) -.*/\1/p}' | tail -1)...v$${GITHUB_REF##*/v}"

# Update default.pgo with the locally running "docker compose" instance.
# Use: "make -j default.pgo".
default.pgo: default-inlet.pgo default-outlet.pgo default-console.pgo
	$Q go tool pprof -proto $^ > $@
	$Q rm $^
default-%.pgo:
	$Q container=akvorado-akvorado-$*-1 ; \
	   ip=$$(docker container inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $$container) ; \
	   [ -n $$ip ] ; \
	   curl -so $@ "http://$$ip:8080/debug/pprof/profile?seconds=30"

common/embed/data/embed.zip: console/data/frontend console/authentication/data/avatars console/data/docs
common/embed/data/embed.zip: console/data/docs/98-metrics.md
common/embed/data/embed.zip: orchestrator/clickhouse/data/protocols.csv orchestrator/clickhouse/data/icmp.csv orchestrator/clickhouse/data/asns.csv orchestrator/clickhouse/data/tcp.csv orchestrator/clickhouse/data/udp.csv
common/embed/data/embed.zip: ; $(info $(M) generate embed.zip…)
	$Q mkdir -p common/embed/data
	$Q TMPDIR=$$(mktemp -d) \
	&& trap 'rm -rf "$$TMPDIR"' EXIT \
	&& tar -cf - $^ | tar -C "$$TMPDIR" -xf - \
	&& find "$$TMPDIR"/console/data/docs -name "*.svg" -type f -print0 | xargs -0 sed -i -e 's/ content="[^"]*"//g' \
	&& ( cd "$$TMPDIR" && zip --quiet --recurse-paths - .) > $@

# Tests

.PHONY: check test tests test-race test-short test-bench test-coverage
.PHONY: test-go test-go-units test-go-staticcheck test-js test-coverage-go test-coverage-js
check test tests: test-go test-js ## Run tests
test-coverage: test-coverage-go test-coverage-js ## Run coverage tests

test-go-units test-go-checks test-bench test-race test-coverage-go: .fmt-go~ .lint-go~ $(GENERATED) $(GENERATED_TEST_GO)
test-go: test-go-units test-go-checks ## Run Go tests
test-go-checks: ; $(info $(M) running Go static checks…)
	$Q $(STATICCHECK) -f stylish -checks inherit,-SA1012 $(PKGS)
test-go-units: ; $(info $(M) running Go tests$(GOTEST_MORE)…)
	$Q mkdir -p test/go
	$Q env PATH=$(dir $(abspath $(shell command -v $(GO)))):$(PATH) \
	  $(GOTESTSUM) --junitfile test/go/tests.xml -- \
		-timeout $(TIMEOUT) \
	    -fullpath \
		$(GOTEST_ARGS) $(PKGS)
test-race: CGO_ENABLED=1
test-race: GOTEST_ARGS=-race
test-race: GOTEST_MORE=, with race detector
test-race: test-go-units  ## Run Go tests with race detector
test-short: GOTEST_ARGS=-short
test-short: GOTEST_MORE=, only short tests
test-short: test-go  ## Run only short Go tests
test-bench: ; $(info $(M) running benchmarks…) @ ## Run Go benchmarks
	$Q $(GO) test \
		-fullpath -run=__absolutelynothing__ -bench=. \
	    $(GOTEST_ARGS) $(PKGS)
# GOTEST_ARGS="-benchmem -memprofile test/go/memprofile.out -cpuprofile test/go/cpuprofile.out"
# GOTEST_ARGS="-cpuprofile test/go/cpuprofile.out -bench RIBLookup"
test-coverage-go: ; $(info $(M) running Go coverage tests…) @ ## Run Go coverage tests
	$Q mkdir -p test/go
	$Q env PATH=$(dir $(abspath $(shell command -v $(GO)))):$(PATH) \
	  $(GOTESTSUM) -- \
	    -fullpath \
		-coverpkg=$(shell echo $(PKGS) | tr ' ' ',') \
		-covermode=atomic \
		-coverprofile=test/go/profile.out.tmp \
	    $(GOTEST_ARGS) $(PKGS)
	$Q GENERATED=$$(awk -F: '(NR > 1) {print $$1}' test/go/profile.out.tmp \
			| sort | uniq | sed "s+^$(MODULE)/++" \
			| xargs grep -l "^//.*DO NOT EDIT\.$$" \
			| sed "s+\(.*\)+^$(MODULE)/\1:+" | paste -s -d '|' -) ; \
	   if [ -n "$$GENERATED" ]; then grep -Ev "$$GENERATED" test/go/profile.out.tmp > test/go/profile.out ; \
	   else cp test/go/profile.out.tmp test/go/profile.out ; \
	   fi
	$Q $(GO) tool cover -html=test/go/profile.out -o test/go/coverage.html
	@printf "Code coverage: "; \
		go tool cover -func test/go/profile.out | awk '($$1 == "total:") { print $$NF}'

test-js: .fmt-js~ .lint-js~ $(GENERATED_JS)
test-js: ; $(info $(M) running JS tests…) @ ## Run JS tests
	$Q cd console/frontend && $(PNPM) run --silent type-check && $(PNPM) run --silent test
test-coverage-js: ; $(info $(M) running JS coverage tests…) @ ## Run JS coverage tests
	$Q cd console/frontend && $(PNPM) run --silent type-check && $(PNPM) run --silent test -- --coverage

.PHONY: lint
lint: .lint-go~ .lint-js~ ## Run linting
.lint-go~: .revive.toml $(shell $(LSFILES) '*.go' 2> /dev/null) ; $(info $(M) running golint…)
	$Q $(REVIVE) -config $(PWD)/.revive.toml -formatter stylish -set_exit_status ./...
	$Q touch $@
.lint-js~: $(shell $(LSFILES) '*.js' '*.ts' '*.vue' '*.html' 2> /dev/null)
.lint-js~: $(GENERATED_JS) ; $(info $(M) running jslint…)
	$Q cd console/frontend && $(PNPM) run --silent lint
	$Q touch $@

.PHONY: fmt
fmt: .fmt-go~ .fmt-js~ ## Format all source files
.fmt-go~: $(shell $(LSFILES) '*.go' 2> /dev/null) ; $(info $(M) formatting Go code…)
	$Q $(GOIMPORTS) -local $(MODULE) -w $? < /dev/null
	$Q touch $@
.fmt-js~: $(shell $(LSFILES) '*.js' '*.ts' '*.vue' '*.html' 2> /dev/null)
.fmt-js~: $(GENERATED_JS) ; $(info $(M) formatting JS code…)
	$Q cd console/frontend && $(PNPM) run --silent format
	$Q touch $@

# Misc

.PHONY: licensecheck
licensecheck: console/frontend/node_modules ; $(info $(M) check dependency licenses…) @ ## Check licenses
	$Q ! git grep -L SPDX-License-Identifier: "*.go" "*.ts" "*.js" || \
		(>&2 echo "*** Missing license identifiers!"; false)
	$Q err=0 ; $(GO) mod vendor && $(WWHRD) --quiet check || err=$$? ; rm -rf vendor/ ; exit $$err
	$Q cd console/frontend ; \
		allowed="$$(sed -n '/^allowlist:/,/^[a-z]/p' ../../.wwhrd.yml | sed -n 's/^  - //p' | paste -sd "|")" ; \
		! $(PNPM) licenses ls --json | jq -r --exit-status --arg allowed "$$allowed" \
			'to_entries[] | select(.key | test("^('"$$allowed"')$$") | not) | "Unallowed license: " + .key + " (" + (.value | map(.name) | join(", ")) + ")"'

.PHONY: clean
clean: ; $(info $(M) cleaning…)	@ ## Cleanup everything
	@rm -rf test $(GENERATED) $(GENERATED_TEST_GO) *~ bin/akvorado

.PHONY: help
help:
	@grep -hE '^[ a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
		awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-17s\033[0m %s\n", $$1, $$2}'

.PHONY: version
version:
	@echo $(VERSION)

.PHONY: docker docker-dev
DOCKER_BUILD_ARGS =
docker: ; $(info $(M) build Docker image…) @ ## Build Docker image
	$Q git ls-files | tar -T- -cf- | docker build --pull -f docker/Dockerfile $(DOCKER_BUILD_ARGS) \
		--build-arg VERSION=$(VERSION) -t quay.io/akvorado/akvorado:main -
docker-dev: TARGETOS=linux
docker-dev: all ; $(info $(M) build development Docker image…) @ ## Build development Docker image
	$Q docker build -f docker/Dockerfile.dev $(DOCKER_BUILD_ARGS) \
		--build-arg VERSION=$(VERSION) -t quay.io/akvorado/akvorado:main .

.PHONY: all-coverage docker-dev-coverage
all-coverage: BUILD_ARGS=-cover -covermode=atomic
all-coverage: all
docker-dev-coverage: BUILD_ARGS=-cover -covermode=atomic
docker-dev-coverage: docker-dev

# This requires "skopeo". I fetch it from nix.
.PHONY: docker-upgrade-versions
docker-upgrade-versions: ; $(info $(M) check for Docker image updates…) @ ## Check for Docker image updates
	$Q sed -En 's/^\s*image:\s+(.+):(.+)\s+#\s+(.+)$$/\1 \2 \3/p' docker/versions.yml \
		| while read -r image version regex; do \
			latest=$$(nix run nixpkgs\#skopeo -- list-tags docker://"$$image" \
				| sed -En 's/\s+"(.*)",?/\1/p' \
				| grep -xP "$$regex" \
				| sort -Vr | head -1); \
			[ "$$version" = "$$latest" ] || { \
				>&2 echo "$$image $$version→$$latest"; \
				sed -i -e "s,$$image:$$version,$$image:$$latest," docker/versions.yml; }; \
		done