fix(plugins): makefile refactor (#5224)

* fix(plugin): make publish

Signed-off-by: Yvonnick Esnault <yvonnick.esnault@corp.ovh.com>

.build/core.mk

@@ -0,0 +1,16 @@
+CI                  := $(if ${CI},${CI},0)
+VERSION             := $(if ${CDS_SEMVER},${CDS_SEMVER},snapshot)
+CDS_VERSION         := $(if ${CDS_VERSION},${CDS_VERSION},snapshot)
+GITHASH             := $(if ${GIT_HASH},${GIT_HASH},`git log -1 --format="%H"`)
+BUILDTIME           := `date "+%m/%d/%y-%H:%M:%S"`
+UNAME               := $(shell uname)
+UNAME_LOWERCASE     := $(shell uname -s| tr A-Z a-z)
+
+.PHONY: help
+help:
+	@grep -hE '^[ a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
+		awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-17s\033[0m %s\n", $$1, $$2}'
+
+install-venom: ## install venom, usage: make venom-install venom_version=v0.27.0 venom_path=/usr/bin/ 
+	@curl https://github.com/ovh/venom/releases/download/$(venom_version)/venom.$(UNAME_LOWERCASE)-amd64 -L -o $(venom_path)/venom && \
+	chmod +x $(venom_path)/venom

.build/go.mk

@@ -0,0 +1,175 @@
+GO_BUILD                 = GOPRIVATE="${GO_PRIVATE}" CGO_ENABLED=0 go build -a -installsuffix cgo
+GO_LIST                  = env GO111MODULE=on GOPRIVATE="${GO_PRIVATE}" go list
+TEST_CMD                 = go test -v -timeout 600s -coverprofile=profile.coverprofile
+TEST_C_CMD               = go test -c -coverprofile=profile.coverprofile
+TEST_RUN_ARGS            = -test.v -test.timeout 600s -test.coverprofile=profile.coverprofile
+LDFLAGS                  = -ldflags "-X github.com/ovh/cds/sdk.VERSION=$(VERSION) -X github.com/ovh/cds/sdk.GOOS=$$GOOS -X github.com/ovh/cds/sdk.GOARCH=$$GOARCH -X github.com/ovh/cds/sdk.GITHASH=$(GITHASH) -X github.com/ovh/cds/sdk.BUILDTIME=$(BUILDTIME) -X github.com/ovh/cds/sdk.BINARY=$(TARGET_ENGINE) -X github.com/ovh/cds/sdk.DBMIGRATE=$(DBMIGRATE)"
+CURRENT_PACKAGE          = $(shell $(GO_LIST))
+TARGET_DIST              := ./dist
+TARGET_RESULTS           := ./dist/test-results
+ENABLE_CROSS_COMPILATION := true
+
+
+##### =====> Clean <===== #####
+
+mk_go_clean: # clean target directory
+	@rm -rf $(TARGET_DIST)
+	@rm -rf $(TARGET_RESULTS)
+	@for testfile in `find ./ -name "bin.test"`; do \
+		rm $$testfile; \
+	done;
+	@for TST in `find ./ -name "tests.log"`; do \
+		rm $$TST; \
+	done;
+	@for profile in `find ./ -name "*.coverprofile"`; do \
+		rm $$profile; \
+	done;
+
+##### =====> Compile <===== #####
+
+IS_TEST                    := $(filter test,$(MAKECMDGOALS))
+TARGET_OS                  := $(filter-out $(TARGET_OS_EXCLUDED), $(if ${ENABLE_CROSS_COMPILATION},$(if ${OS},${OS}, $(if $(IS_TEST), $(shell go env GOOS), windows darwin linux openbsd freebsd)),$(shell go env GOOS)))
+TARGET_ARCH                := $(if ${ARCH},${ARCH}, $(if $(IS_TEST), $(shell go env GOARCH),amd64 arm 386 arm64 ppc64le))
+BINARIES                   = $(addprefix $(TARGET_DIST)/, $(addsuffix -$(OS)-$(ARCH)$(if $(IS_WINDOWS),.exe), $(notdir $(TARGET_NAME))))
+OSARCHVALID                = $(shell go tool dist list | grep -v '^darwin/arm'|grep -v '^darwin/386'|grep -v '^windows/386'|grep -v '^windows/arm'|grep -v '^openbsd/arm*'|grep -v '^openbsd/386'|grep -v '^freebsd/arm*'|grep -v '^freebsd/386')
+IS_OS_ARCH_VALID           = $(filter $(OS)/$(ARCH),$(OSARCHVALID))
+CROSS_COMPILED_BINARIES    = $(foreach OS, $(TARGET_OS), $(foreach ARCH, $(TARGET_ARCH), $(if $(IS_OS_ARCH_VALID), $(BINARIES))))
+GOFILES                    = $(call get_recursive_files, '.')
+
+mk_go_build:
+	$(info *** mk_go_build)
+
+$(CROSS_COMPILED_BINARIES): $(GOFILES)
+	$(info *** compiling $@)
+	@mkdir -p $(TARGET_DIST); \
+	GOOS=$(call get_os_from_binary_file,$@) \
+	GOARCH=$(call get_arch_from_binary_file,$@) \
+	$(GO_BUILD) $(LDFLAGS) -o $@;
+
+##### =====> Compile Tests <===== #####
+
+PKGS     = $(or $(PKG),$(shell $(GO_LIST) ./...))
+TESTPKGS = $(shell $(GO_LIST) -f \
+			'{{ if or .TestGoFiles .XTestGoFiles }}{{ .ImportPath }}{{ end }}' \
+			$(PKGS))
+
+TESTPKGS_C_FILE = $(addsuffix /bin.test, $(subst $(CURRENT_PACKAGE),.,$(PKG)))
+TESTPKGS_C = $(foreach PKG, $(TESTPKGS), $(TESTPKGS_C_FILE))
+
+$(TESTPKGS_C): #main_test.go
+	$(info *** compiling test $@)
+	@cd $(dir $@) && $(TEST_C_CMD) -o bin.test .
+
+##### =====> Running Tests <===== #####
+
+TESTPKGS_RESULTS_LOG_FILE = $(addsuffix /tests.log, $(subst $(CURRENT_PACKAGE),.,$(PKG)))
+TESTPKGS_RESULTS = $(foreach PKG, $(TESTPKGS), $(TESTPKGS_RESULTS_LOG_FILE))
+
+$(HOME)/.richstyle.yml:
+	echo "leaveTestPrefix: true" > $(HOME)/.richstyle.yml
+
+GO_RICHGO = ${GOPATH}/bin/richgo
+$(GO_RICHGO): $(HOME)/.richstyle.yml
+	go get -u github.com/kyoh86/richgo
+
+EXIT_TESTS := 0
+$(TESTPKGS_RESULTS): $(GOFILES) $(TESTPKGS_C) $(GO_RICHGO)
+	$(info *** executing tests in $(dir $@))
+	@-cd $(dir $@) && ./bin.test $(TEST_RUN_ARGS) | tee tests.log | richgo testfilter ;
+
+GO_COV_MERGE = ${GOPATH}/bin/gocovmerge
+$(GO_COV_MERGE):
+	go get -u github.com/wadey/gocovmerge
+
+GO_GOJUNIT = ${GOPATH}/bin/go-junit-report
+$(GO_GOJUNIT):
+	go get -u github.com/jstemmer/go-junit-report
+
+GO_COBERTURA = ${GOPATH}/bin/gocover-cobertura
+$(GO_COBERTURA):
+	go get -u github.com/t-yuki/gocover-cobertura
+
+GO_XUTOOLS = ${GOPATH}/bin/xutools
+$(GO_XUTOOLS):
+	go get -u github.com/richardlt/xutools
+
+mk_go_test: $(GO_COV_MERGE) $(GO_XUTOOLS) $(GO_COBERTURA) $(GOFILES) $(TARGET_RESULTS) $(TESTPKGS_RESULTS) # Run tests
+	@echo "Generating unit tests coverage..."
+	@$(GO_COV_MERGE) `find ./ -name "*.coverprofile"` > $(TARGET_RESULTS)/cover.out
+	@$(GO_COBERTURA) < $(TARGET_RESULTS)/cover.out > $(TARGET_RESULTS)/coverage.xml
+	@go tool cover -html=$(TARGET_RESULTS)/cover.out -o=$(TARGET_RESULTS)/cover.html
+	@NB=$$(grep -c "^FAIL" `find . -type f -name "tests.log"`|grep -v ':0'|grep -v '^0'|wc -l); echo "tests failed $$NB" && exit $$NB
+
+mk_go_test-xunit: $(GO_GOJUNIT) $(GOFILES) $(TARGET_RESULTS) # Generate test with xunit report
+	@echo "Generating xUnit Report..."
+	@for TST in `find . -name "tests.log"`; do \
+		if [ -s $$TST ]; then \
+			FAILED=`grep -E '(FAIL)+\s([a-z\.\/]*)\s\[build failed\]' $$TST | wc -l`; \
+			if [ $$FAILED -gt 0 ]; then \
+				echo "Build Failed \t\t\t($$TST)"; \
+				echo "Build Failed \t\t\t($$TST)" >>  $(TARGET_RESULTS)/fail; \
+			else \
+				NO_TESTS=`grep -E '\?+\s+([a-z\.\/]*)\s\[no test files\]' $$TST | wc -l`; \
+				if [ $$NO_TESTS -gt 0 ]; then \
+					echo "No tests found \t\t\t($$TST)"; \
+				else \
+					if [ ! -z "${CDS_VERSION}" ]; then \
+						echo "Sending $$TST to CDS"; \
+						worker upload --tag `echo $$TST | sed 's|./||' | sed 's|./||' | sed 's|/|_|g') | sed 's|_tests.log||'` $(abspath $$TST); \
+					fi; \
+					echo "Generating xUnit report \t$$TST.tests-results.xml"; \
+					cat $$TST | $(GO_GOJUNIT) > $$TST.tests-results.xml; \
+				fi; \
+			fi; \
+		else \
+			echo "Ignoring empty file \t\t$$TST"; \
+		fi; \
+	done; \
+	xutools pretty --show-failures $(TARGET_DIST)/*.xml > $(TARGET_DIST)/report; \
+	xutools sort-duration $(TARGET_DIST)/*.xml > $(TARGET_DIST)/duration; \
+	if [ -e $(TARGET_DIST)/report ]; then \
+		echo "Report:"; \
+		cat $(TARGET_DIST)/report; \
+	fi; \
+	if [ -e $(TARGET_DIST)/duration ]; then \
+		echo "Max duration:"; \
+		cat $(TARGET_DIST)/duration; \
+	fi;\
+	if [ -e $(TARGET_RESULTS)/fail ]; then \
+		echo "#########################"; \
+		echo "ERROR: Test compilation failure"; \
+		cat $(TARGET_RESULTS)/fail; \
+		exit 1; \
+	fi;
+
+##### =====> lint <===== #####
+
+GOLANG_CI_LINT 		:= ${GOPATH}/bin/golangci-lint
+$(GOLANG_CI_LINT):
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.27.0
+
+mk_go_lint: $(GOLANG_CI_LINT) # run golangci lint
+	$(info *** running lint)
+	$(GOLANG_CI_LINT) run
+
+##### =====> Internals <===== #####
+
+$(TARGET_RESULTS):
+	$(info create $(TARGET_RESULTS) directory)
+	@mkdir -p $(TARGET_RESULTS)
+
+$(TARGET_DIST):
+	$(info create $(TARGET_DIST) directory)
+	@mkdir -p $(TARGET_DIST)
+
+define get_os_from_binary_file
+$(strip $(shell echo $(1) | cut -d'_' -f 2))
+endef
+
+define get_arch_from_binary_file
+$(strip $(patsubst %.exe, %,$(shell echo $(1) | cut -d'_' -f 3)))
+endef
+
+define get_recursive_files
+$(subst ./,,$(shell find $(1) -type f -name "*.go" -print))
+endef

.build/plugin.mk

@@ -0,0 +1,67 @@
+
+BINARIES_CONF                   = $(addprefix $(TARGET_DIST)/, $(addsuffix -$(OS)-$(ARCH).yml, $(notdir $(TARGET_NAME))))
+PLUGIN_CONF                     = $(addprefix $(TARGET_DIST)/, $(addsuffix .yml, $(notdir $(TARGET_NAME))))
+CROSS_COMPILED_PLUGIN_CONF      = $(foreach OS, $(TARGET_OS), $(foreach ARCH, $(TARGET_ARCH), $(if $(IS_OS_ARCH_VALID), $(BINARIES_CONF))))
+
+define PLUGIN_MANIFEST_BINARY
+os: %os%
+arch: %arch%
+cmd: ./%filename%
+endef
+export PLUGIN_MANIFEST_BINARY
+
+define get_os_from_binary_file
+$(strip $(shell echo $(1) | awk '{n=split($$1,a,"-");print a[n-1]}'))
+endef
+
+define get_arch_from_binary_file
+$(strip $(patsubst %.exe, %,$(shell echo $(1) | awk '{n=split($$1,a,"-");print a[n]}')))
+endef
+
+define get_arch_from_conf_file
+$(strip $(patsubst %.yml, %,$(shell echo $(1) | awk '{n=split($$1,a,"-");print a[n]}')))
+endef
+
+define get_executor_path_from_binary_file
+$(strip $(patsubst dist/%, %, $(patsubst %-, %, $(shell echo $(1) |awk '{n=split($$1,a,"-");for (i = 2; i < n-1; i++) printf a[i] "-"}'))))
+endef
+
+## Prepare yml file for each os-arch
+$(CROSS_COMPILED_PLUGIN_CONF): $(GOFILES)
+	$(info *** prepare conf $@)
+	@mkdir -p $(TARGET_DIST); \
+	echo "$$PLUGIN_MANIFEST_BINARY" > $@; \
+	OS=$(call get_os_from_binary_file,$@); \
+	ARCH=$(call get_arch_from_conf_file,$@); \
+	perl -pi -e s,%os%,$$OS,g $@ ; \
+	perl -pi -e s,%arch%,$$ARCH,g $@ ; \
+	EXTENSION=""; \
+	if test "$(TARGET_NAME)" == *"windows"* ; then EXTENSION=".exe"; fi; \
+	FILENAME=$(TARGET_NAME)-$$OS-$$ARCH$$EXTENSION; \
+	perl -pi -e s,%filename%,$$FILENAME,g $@
+
+$(PLUGIN_CONF):
+	$(info *** prepare conf $@)
+	@mkdir -p $(TARGET_DIST); \
+	cp $(TARGET_NAME).yml $@;
+
+mk_go_build_plugin: $(CROSS_COMPILED_PLUGIN_CONF) $(PLUGIN_CONF) $(CROSS_COMPILED_BINARIES)
+
+mk_plugin_publish:
+	@echo "Updating plugin $(TARGET_NAME)..."
+	cdsctl admin plugins import $(TARGET_DIST)/$(TARGET_NAME).yml
+	@for GOOS in $(TARGET_OS); do \
+		for GOARCH in $(TARGET_ARCH); do \
+			EXTENSION=""; \
+			for V in $(OSARCHVALID); do \
+				if test "$$GOOS/$$GOARCH" = "$$V"; then \
+					if test "$$GOOS" = "windows" ; then EXTENSION=".exe"; fi; \
+					echo "Updating plugin binary $(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION"; \
+					cdsctl admin plugins binary-add $(TARGET_NAME) $(TARGET_DIST)/$(TARGET_NAME)-$$GOOS-$$GOARCH.yml $(TARGET_DIST)/$(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION; \
+				fi; \
+			done; \
+		done; \
+	done
+
+mk_plugin_package: 
+	tar czf $(TARGET_DIST)/cds-$(TARGET_NAME)-all.tar.gz $(TARGET_DIST)/$(TARGET_NAME)*