diff --git a/.github/release.py b/.github/release.py new file mode 100755 index 0000000..13473a1 --- /dev/null +++ b/.github/release.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import shutil +import subprocess +import sys + + +def _goos(): + yield 'linux' + yield 'freebsd' + + +def _goarch(goos): + yield '386' + yield 'amd64' + yield 'arm' + yield 'arm64' + if goos == 'linux': + yield 'ppc64le' + yield 's390x' + yield 'riscv64' + + +def _goarm(goarch): + if goarch != 'arm': + yield '' + return + yield '6' + yield '7' + + +def _static(): + yield True + yield False + + +def _build_tarball(os): + if os == 'linux': + yield True + yield False + + +def filename_for_entry(prog_name, entry): + arch = entry['goarch'] + if entry['goarch'] == 'arm': + arch += 'v' + entry['goarm'] + ret = f'{prog_name}-{entry["goos"]}-{arch}' + if entry['static']: + ret += '-static' + if entry['build_tarball']: + ret += '.tgz' + return ret + + +def matrix(prog_name): + for goos in _goos(): + for goarch in _goarch(goos): + for goarm in _goarm(goarch): + for static in _static(): + for build_tarball in _build_tarball(goos): + yield { + 'goos': goos, + 'goarch': goarch, + 'goarm': goarm, + 'static': static, + 'build_tarball': build_tarball, + } + + +def print_matrix(prog_name): + j = {'include': list(matrix(prog_name))} + + if os.isatty(sys.stdout.fileno()): + print(json.dumps(j, indent=2)) + else: + print(json.dumps(j)) + + +default_tarball = { + 'goos': 'linux', + 'goarch': 'amd64', + 'goarm': '', + 'static': False, + 'build_tarball': True, +} + +default_binary = { + 'goos': 'linux', + 'goarch': 'amd64', + 'goarm': '', + 'static': False, + 'build_tarball': False, +} + + +def run_build(prog_name): + # call the makefile for each matrix entry + + default_tarball_filename = None + default_binary_filename = None + + for entry in matrix(prog_name): + env = {'GOOS': entry['goos'], 'GOARCH': entry['goarch']} + + if entry['goarm']: + env['GOARM'] = entry['goarm'] + + if entry['static']: + env['BUILD_STATIC'] = 'yes' + + if entry['build_tarball']: + target = 'tarball' + else: + target = 'binary' + + print(f"Running make {target} for {env}") + + subprocess.run(['make', target], env=os.environ | env, check=True) + + want_filename = filename_for_entry(prog_name, entry) + + if entry['build_tarball']: + os.rename(f'{prog_name}.tgz', want_filename) + else: + os.rename(f'{prog_name}', want_filename) + + # if this is the default tarball or binary, save the filename + # we'll use it later to publish a "default" package + + if entry == default_tarball: + default_tarball_filename = want_filename + + if entry == default_binary: + default_binary_filename = want_filename + + # Remove the directory to reuse it + subprocess.run(['make', 'clean-release-dir'], env=os.environ | env, check=True) + + # publish the default tarball and binary + if default_tarball_filename: + shutil.copy(default_tarball_filename, f'{prog_name}.tgz') + + if default_binary_filename: + shutil.copy(default_binary_filename, f'{prog_name}') + + +def main(): + parser = argparse.ArgumentParser( + description='Build release binaries and tarballs for all supported platforms') + parser.add_argument('action', help='Action to perform (ex. run-build, print-matrix)') + parser.add_argument('prog_name', help='Name of the program (ex. crowdsec-firewall-bouncer)') + + args = parser.parse_args() + + if args.action == 'print-matrix': + print_matrix(args.prog_name) + + if args.action == 'run-build': + run_build(args.prog_name) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/build-binary-package.yml b/.github/workflows/build-binary-package.yml index 3b2ecb1..4c4d1a1 100644 --- a/.github/workflows/build-binary-package.yml +++ b/.github/workflows/build-binary-package.yml @@ -1,69 +1,37 @@ # .github/workflows/build-docker-image.yml name: build-binary-package -on: +on: release: types: prereleased +env: + PROGRAM_NAME: crowdsec-blocklist-mirror + jobs: - build-binary-package: - name: Build and upload binary package + build: + name: Build and upload all platforms runs-on: ubuntu-latest - strategy: - matrix: - static: - - false - - true - goos: - - linux - - freebsd - goarch: - - amd64 - - arm64 + steps: + - name: Set up Go 1.20.1 uses: actions/setup-go@v3 with: go-version: 1.20.1 - id: go - - name: Check out code into the Go module directory + + - name: Check out repository uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Build the binaries - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - run: | - make release - if: ${{ !matrix.static }} - - - name: Build the binaries (static) - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} + - name: Build all versions run: | - make release BUILD_STATIC=yes - mv crowdsec-blocklist-mirror-${{ env.GOOS }}-${{ env.GOARCH }}.tgz crowdsec-blocklist-mirror-${{ env.GOOS }}-${{ env.GOARCH }}-static.tgz - if: ${{ matrix.static }} + make platform-all - name: Upload to release - uses: JasonEtco/upload-to-release@master - with: - args: crowdsec-blocklist-mirror-${{ env.GOOS }}-${{ env.GOARCH }}.tgz application/x-gzip env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - if: ${{ !matrix.static }} - - - name: Upload to release (static) - uses: JasonEtco/upload-to-release@master - with: - args: crowdsec-blocklist-mirror-${{ env.GOOS }}-${{ env.GOARCH }}-static.tgz application/x-gzip - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - if: ${{ matrix.static }} + run: | + tag_name="${GITHUB_REF##*/}" + hub release edit $(find . -name "$PROGRAM_NAME*" -maxdepth 1 -printf "-a %p ") -m "" "$tag_name" diff --git a/.gitignore b/.gitignore index 011d826..53f0520 100644 --- a/.gitignore +++ b/.gitignore @@ -23,5 +23,5 @@ # built binaries /crowdsec-blocklist-mirror +/crowdsec-blocklist-mirror-* /crowdsec-blocklist-mirror.tgz -/crowdsec-blocklist-mirror*/ diff --git a/Makefile b/Makefile index 2c55593..be275ab 100644 --- a/Makefile +++ b/Makefile @@ -1,52 +1,81 @@ -# Go parameters GOCMD=go GOBUILD=$(GOCMD) build -GOCLEAN=$(GOCMD) clean GOTEST=$(GOCMD) test -GOGET=$(GOCMD) get -GOOS ?= linux -GOARCH ?= amd64 +BINARY_NAME=crowdsec-blocklist-mirror +REPO_NAME=cs-blocklist-mirror +TARBALL_NAME=$(BINARY_NAME).tgz # Current versioning information from env -BUILD_VERSION?="$(shell git describe --tags)" -BUILD_TIMESTAMP=$(shell date +%F"_"%T) -BUILD_TAG="$(shell git rev-parse HEAD)" +BUILD_VERSION?=$(shell git describe --tags) +BUILD_TIMESTAMP?=$(shell date +%F"_"%T) +BUILD_TAG?=$(shell git rev-parse HEAD) LD_OPTS_VARS=\ --X 'github.com/crowdsecurity/cs-blocklist-mirror/version.Version=$(BUILD_VERSION)' \ --X 'github.com/crowdsecurity/cs-blocklist-mirror/version.BuildDate=$(BUILD_TIMESTAMP)' \ --X 'github.com/crowdsecurity/cs-blocklist-mirror/version.Tag=$(BUILD_TAG)' +-X 'github.com/crowdsecurity/$(REPO_NAME)/pkg/version.Version=$(BUILD_VERSION)' \ +-X 'github.com/crowdsecurity/$(REPO_NAME)/pkg/version.BuildDate=$(BUILD_TIMESTAMP)' \ +-X 'github.com/crowdsecurity/$(REPO_NAME)/pkg/version.Tag=$(BUILD_TAG)' ifdef BUILD_STATIC - export LD_OPTS=-ldflags "-a -v -s -w -extldflags '-static' $(LD_OPTS_VARS)" -tags netgo + export LD_OPTS=-ldflags "-a -s -w -extldflags '-static' $(LD_OPTS_VARS)" -tags netgo else - export LD_OPTS=-ldflags "-a -v -s -w $(LD_OPTS_VARS)" + export LD_OPTS=-ldflags "-a -s -w $(LD_OPTS_VARS)" endif -PREFIX?="/" -BINARY_NAME=crowdsec-blocklist-mirror +.PHONY: all +all: build + +# Remove everything including all platform binaries and tarballs +.PHONY: clean +clean: clean-release-dir + @$(RM) $(BINARY_NAME) + @$(RM) $(TARBALL_NAME) + @$(RM) -r $(BINARY_NAME)-* # platform binary name and leftover release dir + @$(RM) $(BINARY_NAME)-*.tgz # platform release file + +# +# Build binaries +# + +.PHONY: binary +binary: goversion + $(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME) -RELDIR = "crowdsec-blocklist-mirror-${BUILD_VERSION}" +.PHONY: build +build: goversion clean binary -all: clean build +# +# Unit and integration tests +# -build: goversion clean - $(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME) +.PHONY: lint +lint: + golangci-lint run .PHONY: test test: - @$(GOTEST) ./... + @$(GOTEST) $(LD_OPTS) ./... -clean: - @rm -f $(BINARY_NAME) - @rm -rf ${RELDIR} - @rm -f crowdsec-blocklist-mirror-*.tgz || "" +.PHONY: func-tests +func-tests: build + pipenv install --dev + pipenv run pytest -v -.PHONY: release -release: build - @if [ -z ${BUILD_VERSION} ] ; then BUILD_VERSION="local" ; fi - @if [ -d $(RELDIR) ]; then echo "$(RELDIR) already exists, clean" ; exit 1 ; fi +# +# Build release tarballs +# + +RELDIR = $(BINARY_NAME)-$(BUILD_VERSION) + +# Called during release, to reuse the directory for other platforms +.PHONY: clean-release-dir +clean-release-dir: + @$(RM) -r $(RELDIR) + +.PHONY: tarball +tarball: binary + @if [ -z $(BUILD_VERSION) ]; then BUILD_VERSION="local" ; fi + @if [ -d $(RELDIR) ]; then echo "$(RELDIR) already exists, please run 'make clean' and retry" ; exit 1 ; fi @echo Building Release to dir $(RELDIR) @mkdir $(RELDIR)/ @cp $(BINARY_NAME) $(RELDIR)/ @@ -57,11 +86,18 @@ release: build @chmod +x $(RELDIR)/install.sh @chmod +x $(RELDIR)/uninstall.sh @chmod +x $(RELDIR)/upgrade.sh - @tar cvzf crowdsec-blocklist-mirror-$(GOOS)-$(GOARCH).tgz $(RELDIR) + @tar cvzf $(TARBALL_NAME) $(RELDIR) -.PHONY: func-tests -func-tests: build - pipenv install --dev - pipenv run pytest -v +.PHONY: release +release: clean tarball + +# +# Build binaries and release tarballs for all platforms +# + +.PHONY: platform-all +platform-all: goversion clean + python3 .github/release.py run-build $(BINARY_NAME) +# Check if go is the right version include mk/goversion.mk diff --git a/main.go b/main.go index a4ecc5a..b765b02 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/crowdsecurity/cs-blocklist-mirror/version" + "github.com/crowdsecurity/cs-blocklist-mirror/pkg/version" csbouncer "github.com/crowdsecurity/go-cs-bouncer" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" diff --git a/version/version.go b/pkg/version/version.go similarity index 100% rename from version/version.go rename to pkg/version/version.go