CLI-1054 Replace Sarama with a Confluent Kafka Library or API for Pro…

…duce and Consume (#978) * replace sarama with ckgo in produce and consume. Todo: srClient and Ctx in consume. * remove redundant print statements in produce and consume * Update command_topic.go * Update command_topic.go * add group handler in consume() * Update go.mod * run go mod tidy * Update command_topic.go * update azure-pipelines.yml specify gcc version to be 10.3 * Update azure-pipelines.yml * install gcc 11.2 in azure pipelines setup * Update setup-go.yml * update azure-pipelines.yml * Update setup-go.yml * run gcc -version * Update azure-pipelines.yml * install gcc 11.2 32 and 64-bit * show where gcc * set gcc path * Update setup-go.yml * Display GCC version (Windows) * show RAM info and install Go 1.15.15 * Update setup-go.yml * install Go 1.16.3 * install Go 1.17 * delete -race flag in Windows test * delete -race flag in unit tests * Update Makefile * delete -race flag in unit tests if Windows remove RAM info in ap. * Update standard_build_layout.patch * Update command_topic.go * chore: minor version bump v1.39.0 [ci skip] * chore: minor version bump v1.40.0 [ci skip] * add windows dockerfile * Update Makefile * update windows dockerfile * Update Dockerfile_windows * Update Dockerfile_windows * chore: minor version bump v1.41.0 [ci skip] * added git-credential-netrc.perl * clean up go.sum and delete Sarama error catchers * Update standard_build_layout.patch * disable release in windows goreleaser * use prebuilt windows image hosted on ECR * update go.sum * address comments * refactor produce method and update comments * build ckgo for all platforms * Use one unified makefile for building confluent * CLI-1359, fix AP running issues. * Revert bad merges in release file. * address comments Add log tracer in produce/consume. Add retry in writing schema file. Refactor makefile targets. * resolve conflict with main * Trigger Build * address comments part i * Use one goreleaser file for linux building * set empty TAGS var for OS other than musl linux * address comments part ii update install.sh and Readme. Cleanup make build targets in makefile. delete build-glibc file. * update ccloud sdk version to 0.0.91 fix conflict with main branch. * delete duplicate consumer.close() call * cleanup make targets and update readme * Set RDKAFKA_PATH properly based on GOPATH (and restrict search path for faster speed) * address comments iii Revert retry in confluent_kafka.go. narrow down rdkafka search path. * Update release.mk * Update .goreleaser-fake.yml * update switch/restore librdkafka targets * capture stderr from cgo and timeout * remove non-windows-compatible package Co-authored-by: David Hyde <DABH@users.noreply.github.com> Co-authored-by: Brian Strauch <bstrauch@confluent.io>
confluentinc · Nov 3, 2021 · ffb8bd5 · ffb8bd5
1 parent 737e267
commit ffb8bd5
Show file tree

Hide file tree

Showing 24 changed files with 639 additions and 565 deletions.
diff --git a/.goreleaser-build.yml b/.goreleaser-build.yml
@@ -7,6 +7,7 @@ builds:
     main: cmd/confluent/main.go
     flags:
       - -mod=readonly
+      - -tags={{.Env.TAGS}}
     ldflags:
       - -s -w -X main.version={{.Env.VERSION}} -X main.commit={{.ShortCommit}} -X main.date={{.Date}} -X main.host={{.Env.HOSTNAME}}
     gcflags:

diff --git a/.goreleaser-fake.yml b/.goreleaser-fake.yml
@@ -38,5 +38,4 @@ release:
   disable: true
 
 archive:
-  format: binary
-
+  format: binary
diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -17,18 +17,40 @@ builds:
     main: cmd/confluent/main.go
     flags:
       - -mod=readonly
+      - -tags=musl
     ldflags:
       - -s -w -X main.version={{.Env.VERSION}} -X main.commit={{.ShortCommit}} -X main.date={{.Date}} -X main.host={{.Env.HOSTNAME}}
     gcflags:
       - all=-trimpath={{.Env.GOPATH}}
     asmflags:
       - all=-trimpath={{.Env.GOPATH}}
+    env:
+      - CGO_ENABLED=1
+      - CC=x86_64-linux-musl-gcc
+      - CXX=x86_64-linux-musl-g++
+      - CGO_LDFLAGS=${CGO_LDFLAGS} -static
     goos:
       - linux
+    goarch:
+      - amd64
+  - binary: confluent
+    main: cmd/confluent/main.go
+    flags:
+      - -mod=readonly
+    ldflags:
+      - -s -w -X main.version={{.Env.VERSION}} -X main.commit={{.ShortCommit}} -X main.date={{.Date}} -X main.host={{.Env.HOSTNAME}} -buildmode=exe  
+    gcflags:
+      - all=-trimpath={{.Env.GOPATH}}
+    asmflags:
+      - all=-trimpath={{.Env.GOPATH}}
+    env:
+      - CGO_ENABLED=1
+      - CC=x86_64-w64-mingw32-gcc
+      - CXX=x86_64-w64-mingw32-g++
+    goos:
       - windows
     goarch:
       - amd64
-      - 386
   - binary: confluent
     id: signed-amd64
     main: cmd/confluent/main.go
@@ -67,11 +89,15 @@ builds:
     goarch:
       - arm64
     hooks:
+      pre:
+        - cmd: make switch-librdkafka-arm64
       post:
+        - cmd: make restore-librdkafka-amd64
         - cmd: make download-licenses
           env:
             - LICENSE_BIN=confluent
             - LICENSE_BIN_PATH=./dist/signed-arm64_darwin_arm64/confluent
+            - CGO_ENABLED=1
         - cmd: gon gon_confluent_arm64.hcl
 
 archives:

diff --git a/Dockerfile_alpine b/Dockerfile_alpine
diff --git a/Makefile b/Makefile
@@ -4,9 +4,49 @@ GIT_REMOTE_NAME ?= origin
 MAIN_BRANCH     ?= main
 RELEASE_BRANCH  ?= main
 
-.PHONY: build
+.PHONY: cross-build # cross-compile from Darwin/amd64 machine to Win64, Linux64 and Darwin/arm64
+cross-build:
+ifeq ($(GOARCH),arm64) # build for darwin/arm64.
+	make build-darwin-arm64
+else # build for amd64 arch
+    ifeq ($(GOOS),windows)
+		CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ make cli-builder
+    else ifeq ($(GOOS),linux) 
+		CGO_ENABLED=1 CC=x86_64-linux-musl-gcc CXX=x86_64-linux-musl-g++ CGO_LDFLAGS="-static" TAGS=musl make cli-builder
+    else # build for Darwin/amd64
+		CGO_ENABLED=1 make cli-builder
+    endif
+endif
+
+.PHONY: build # compile natively based on the system
 build:
-	@GOPRIVATE=github.com/confluentinc VERSION=$(VERSION) HOSTNAME=$(HOSTNAME) goreleaser build -f .goreleaser-build.yml --rm-dist --single-target --snapshot
+ifneq "" "$(findstring NT,$(shell uname))" # build for Windows
+	CC=gcc CXX=g++ make cli-builder
+else ifneq (,$(findstring Linux,$(shell uname)))
+    ifneq (,$(findstring musl,$(shell ldd --version))) # build for musl Linux
+		CC=gcc CXX=g++ TAGS=musl make cli-builder
+    else # build for glibc Linux
+		CC=gcc CXX=g++ make cli-builder
+    endif
+else
+    ifneq (,$(findstring x86_64,$(shell uname -m))) # build for Darwin/amd64
+		make cli-builder
+    else # build for Darwin/arm64
+		make switch-librdkafka-arm64
+		make cli-builder || true 
+		make restore-librdkafka-amd64
+    endif
+endif
+
+.PHONY: build-darwin-arm64
+build-darwin-arm64:
+	make switch-librdkafka-arm64
+	CGO_ENABLED=1 make cli-builder || true 
+	make restore-librdkafka-amd64
+
+.PHONY: cli-builder
+cli-builder:
+	@GOPRIVATE=github.com/confluentinc TAGS=$(TAGS) CGO_ENABLED=$(CGO_ENABLED) CC=$(CC) CXX=$(CXX) CGO_LDFLAGS=$(CGO_LDFLAGS) VERSION=$(VERSION) HOSTNAME=$(HOSTNAME) goreleaser build -f .goreleaser-build.yml --rm-dist --single-target --snapshot
 
 include ./mk-files/dockerhub.mk
 include ./mk-files/semver.mk
@@ -21,6 +61,7 @@ REF := $(shell [ -d .git ] && git rev-parse --short HEAD || echo "none")
 DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
 HOSTNAME := $(shell id -u -n)@$(shell hostname)
 RESOLVED_PATH=github.com/confluentinc/cli/cmd/confluent
+RDKAFKA_PATH := $(shell find $(GOPATH)/pkg/mod/github.com/confluentinc -name librdkafka_vendor)
 
 S3_BUCKET_PATH=s3://confluent.cloud
 S3_STAG_FOLDER_NAME=cli-release-stag
@@ -46,16 +87,19 @@ jenkins-deps:
 	go get github.com/goreleaser/goreleaser@v0.164.0
 
 ifeq ($(shell uname),Darwin)
-SHASUM ?= gsha256sum
+    SHASUM ?= gsha256sum
 else ifneq (,$(findstring NT,$(shell uname)))
 # TODO: I highly doubt this works. Completely untested. The output format is likely very different than expected.
-SHASUM ?= CertUtil SHA256 -hashfile
+    SHASUM ?= CertUtil SHA256 -hashfile
+else ifneq (,$(findstring Windows,$(shell systeminfo)))
+    SHASUM ?= CertUtil SHA256 -hashfile
 else
-SHASUM ?= sha256sum
+    SHASUM ?= sha256sum
 endif
 
 show-args:
 	@echo "VERSION: $(VERSION)"
+	@echo "RDKAFKA_PATH: $(RDKAFKA_PATH)"
 
 #
 # START DEVELOPMENT HELPERS
@@ -171,15 +215,24 @@ lint-licenses: build
 	GITHUB_TOKEN=$(token) golicense $${args} .golicense.hcl ./dist/confluent_$(shell go env GOOS)_$(shell go env GOARCH)/confluent || true
 
 .PHONY: coverage-unit
+## disabled -race flag for Windows build because of 'ThreadSanitizer failed to allocate' error: https://github.com/golang/go/issues/46099. Will renable in the future when this issue is resolved.
 coverage-unit:
-      ifdef CI
+ifdef CI
 	@# Run unit tests with coverage.
-	@GOPRIVATE=github.com/confluentinc go test -v -race -coverpkg=$$(go list ./... | grep -v test | grep -v mock | tr '\n' ',' | sed 's/,$$//g') -coverprofile=unit_coverage.txt $$(go list ./... | grep -v vendor | grep -v test) $(UNIT_TEST_ARGS)
+  ifeq "$(OS)" "Windows_NT"
+	@GOPRIVATE=github.com/confluentinc go test -v -coverpkg=$$(go list ./... | grep -v test | grep -v mock | tr '\n' ',' | sed 's/,$$//g') -coverprofile=unit_coverage.txt $$(go list ./... | grep -v vendor | grep -v test) $(UNIT_TEST_ARGS) -ldflags '-buildmode=exe'
+  else
+	@GOPRIVATE=github.com/confluentinc go test -v -race -coverpkg=$$(go list ./... | grep -v test | grep -v mock | tr '\n' ',' | sed 's/,$$//g') -coverprofile=unit_coverage.txt $$(go list ./... | grep -v vendor | grep -v test) $(UNIT_TEST_ARGS) -ldflags '-buildmode=exe'
+  endif
 	@grep -h -v "mode: atomic" unit_coverage.txt >> coverage.txt
-      else
+else
 	@# Run unit tests.
-	@GOPRIVATE=github.com/confluentinc go test -race -coverpkg=./... $$(go list ./... | grep -v vendor | grep -v test) $(UNIT_TEST_ARGS)
-      endif
+  ifeq "$(OS)" "Windows_NT"
+	@GOPRIVATE=github.com/confluentinc go test -coverpkg=./... $$(go list ./... | grep -v vendor | grep -v test) $(UNIT_TEST_ARGS) -ldflags '-buildmode=exe'
+  else
+	@GOPRIVATE=github.com/confluentinc go test -race -coverpkg=./... $$(go list ./... | grep -v vendor | grep -v test) $(UNIT_TEST_ARGS) -ldflags '-buildmode=exe'
+  endif
+endif
 
 .PHONY: coverage-integ
 coverage-integ:

diff --git a/README.md b/README.md
@@ -42,7 +42,7 @@ The Confluent CLI lets you manage your Confluent Cloud and Confluent Platform de
 
 ## Install
 
-The CLI has pre-built binaries for macOS, Linux, and Windows, on both i386 and x86_64 architectures.
+The CLI has pre-built binaries for macOS, Linux, and Windows.
 
 You can download a tarball with the binaries. These are both on Github releases and in S3.
 
@@ -106,6 +106,8 @@ $ make build
 $ dist/confluent_$(go env GOOS)_$(go env GOARCH)/confluent -h
 ```
 
+Please update your system to MacOS 11.0 or later, if you are developing for Darwin/arm64.
+
 If `make deps` fails with an "unknown revision" error, you probably need to put your username and a
 [github personal access token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token)
 in your ~/.netrc file as outlined [here](https://gist.github.com/technoweenie/1072829). The access token needs to be
@@ -167,13 +169,23 @@ Things under `internal/cmd` are commands, things under `internal/pkg` are packag
 When you add a new command or resource, assuming it's already in the SDK, you generally just need to create
 * `internal/cmd/<command>/<command>.go` (and test)
 
-### Build Other Platforms
+### Cross Compile for Other Platforms
+
+Cross compilation from a Darwin/amd64 machine to Darwin/arm64, Linux/amd64 and Windows/amd64 platforms is supported. To build for Darwin/arm64, run the following:
+
+    GOARCH=arm64 make cross-build
+
+To cross compile for Linux (glibc or musl), install cross compiler `musl-cross` with homebrew:
+
+    brew install FiloSottile/musl-cross/musl-cross
+    GOOS=linux make cross-build
+
+To cross compile for Windows/amd64, install `Mingw-w64` compilers with homebrew:
 
-If you have a need to build a binary for a platform that is not the current one, use the following:
+    brew install mingw-w64
+    GOOS=windows make cross-build
 
-    GOOS=linux make build
-    GOOS=mac make build
-    GOOS=windows make build
+Cross compilation from M1 Macbook (Darwin/arm64) to other platforms is also supported. For detailed documentation, refer to [How to Build CLI with Confluent-Kafka-go for All Platforms](https://confluentinc.atlassian.net/wiki/spaces/Foundations/pages/2610299218/How+to+Build+CLI+with+Confluent-Kafka-go+for+All+Platforms)
 
 ### URLS
 Use the `login` command with the `--url` option to point to a different development environment

diff --git a/azure-pipelines-templates/setup-go.yml b/azure-pipelines-templates/setup-go.yml
@@ -30,6 +30,20 @@ steps:
   condition: eq( variables['Agent.OS'], 'Windows_NT' )
   displayName: Install Go 1.16.3 (Windows)
 
+- powershell: |
+    $ProgressPreference = 'SilentlyContinue'
+    Invoke-WebRequest -Uri "https://github.com/brechtsanders/winlibs_mingw/releases/download/11.2.0-12.0.1-9.0.0-r1/winlibs-x86_64-posix-seh-gcc-11.2.0-llvm-12.0.1-mingw-w64-9.0.0-r1.zip" -OutFile "$(Agent.BuildDirectory)\gcc11.2.zip" -Verbose
+    & ${env:ProgramFiles}\7-Zip\7z.exe x $(Agent.BuildDirectory)/gcc11.2.zip "-o$(Agent.BuildDirectory)" -y
+  condition: eq( variables['Agent.OS'], 'Windows_NT' )
+  displayName: Install gcc 11.2 x86_64 (Windows)
+
+- script: |
+    echo '##vso[task.prependpath]$(Agent.BuildDirectory)\mingw64\bin'
+    echo '##vso[task.setvariable variable=PATH;]$(Agent.BuildDirectory)\mingw64\bin;$(PATH)'
+    echo '$(PATH)'
+  condition: eq( variables['Agent.OS'], 'Windows_NT' )
+  displayName: Set up the GCC workspace (Windows)
+
 # Linux
 - bash: |
     echo "##vso[task.setvariable variable=GOROOT]$(Agent.BuildDirectory)/go"

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -9,6 +9,9 @@ strategy:
 
 pool:
   vmImage: $(imageName)
+  variables:
+    CC: gcc-11.2
+    CXX: g++-11.2
 
 variables:
   # Fixes checksum errors when downloading chrome
@@ -20,6 +23,13 @@ steps:
 
 - template: azure-pipelines-templates/setup-go.yml
 
+- task: CmdLine@2
+  inputs:
+    script: |
+      gcc --version
+  condition: eq( variables['Agent.OS'], 'Windows_NT' )
+  displayName: 'Display GCC version (Windows)'
+
 - script: |
     curl -Ls https://github.com/github/hub/releases/download/v2.11.1/hub-linux-amd64-2.11.1.tgz | sudo tar -xvz -C /usr/bin --strip-components=2 hub-linux-amd64-2.11.1/bin/hub
     sudo chmod 755 /usr/bin/hub

diff --git a/build_alpine.sh b/build_alpine.sh
diff --git a/debian/Makefile b/debian/Makefile
@@ -56,7 +56,7 @@ install: apply-patches
 	chmod 755 $(DESTDIR)$(BINPATH)/confluent
 
 	cd $(DESTDIR)$(LIBPATH) ; \
-	for dir in darwin_amd64 darwin_arm64 linux_386 linux_amd64 windows_386 windows_amd64; do \
+	for dir in darwin_amd64 darwin_arm64 linux_amd64 windows_amd64; do \
 		mkdir -p $${dir} ; \
 		ext=""; if [[ $${dir} =~ windows_.+ ]]; then ext=".exe"; fi ; \
 		filepath=$${dir}/confluent$${ext} ; \