Merge pull request #1379 from LeGEC/1378-makefile-opt-out-libbrotli

Makefile: change compilation and linking of libbrotli to optional (and off by default)

.github/workflows/dockertests.yml

@@ -62,6 +62,7 @@ jobs:
         if: steps.cache-images.outputs.cache-hit != 'true'
         run: make pg_save_image
     env:
+      USE_BROTLI: 1
       USE_LZO: 1
 
   test:
@@ -174,5 +175,6 @@ jobs:
       - name: Run test
         run: ${{ matrix.command }}
     env:
+      USE_BROTLI: 1
       USE_LIBSODIUM: 1
       USE_LZO: 1

.github/workflows/release.yml

@@ -7,6 +7,7 @@ on:
 
 env:
   GO_VERSION: 1.18
+  USE_BROTLI: 1
   USE_LIBSODIUM: 1
   USE_LZO: 1
 
@@ -91,6 +92,7 @@ jobs:
           arch: ${{ matrix.arch }}
           distro: ubuntu20.04
           env: |
+            USE_BROTLI: ${{ env.USE_BROTLI }}
             USE_LIBSODIUM: ${{ env.USE_LIBSODIUM }}
             USE_LZO: ${{ env.USE_LZO }}
           shell: /bin/bash
@@ -117,7 +119,6 @@ jobs:
             make deps
             
             make ${{ matrix.db }}_build
-
       - name: Rename WAL-G binary
         run: mv main/${{ matrix.db }}/wal-g wal-g-${{ matrix.db }}-${{ matrix.distro }}-${{ matrix.arch }}
 

.github/workflows/unittests.yml

@@ -9,6 +9,7 @@ on:
 
 env:
   GO_VERSION: 1.18
+  USE_BROTLI: 1
 
 jobs:
   unittest:
@@ -38,6 +39,7 @@ jobs:
       - name: Test
         run: make all_unittests
         env:
+          USE_BROTLI: 1
           USE_LIBSODIUM: 1
           USE_LZO: 1
           TEST_MODIFIER: -race

Makefile

@@ -19,7 +19,11 @@ GOLANGCI_LINT_VERSION ?= "v1.37.0"
 REDIS_VERSION ?= "5.0.8"
 TOOLS_MOD_DIR := ./internal/tools
 
-BUILD_TAGS:=brotli
+BUILD_TAGS:=
+
+ifdef USE_BROTLI
+	BUILD_TAGS:=$(BUILD_TAGS) brotli
+endif
 
 ifdef USE_LIBSODIUM
 	BUILD_TAGS:=$(BUILD_TAGS) libsodium
@@ -233,7 +237,8 @@ install:
 	@echo "Nothing to be done. Use pg_install/mysql_install/mongo_install/fdb_install/gp_install... instead."
 
 link_brotli:
-	./link_brotli.sh
+	@if [ -n "${USE_BROTLI}" ]; then ./link_brotli.sh; fi
+	@if [ -z "${USE_BROTLI}" ]; then echo "info: USE_BROTLI is not set, skipping 'link_brotli' task"; fi
 
 link_libsodium:
 	@if [ ! -z "${USE_LIBSODIUM}" ]; then\
@@ -242,7 +247,7 @@ link_libsodium:
 
 unlink_brotli:
 	rm -rf vendor/github.com/google/brotli/*
-	mv tmp/brotli/* vendor/github.com/google/brotli/
+	if [ -n "${USE_BROTLI}" ] ; then mv tmp/brotli/* vendor/github.com/google/brotli/; fi
 	rm -rf tmp/brotli
 
 unlink_libsodium:

docker/fdb_tests/Dockerfile

@@ -13,6 +13,7 @@ COPY cmd/ cmd/
 COPY main/ main/
 COPY utility/ utility/
 COPY Makefile Makefile
+ENV USE_BROTLI 1
 
 RUN sed -i 's|#cgo LDFLAGS: -lbrotli.*|&-static -lbrotlicommon-static -lm|' \
         vendor/github.com/google/brotli/go/cbrotli/cgo.go && \

docker/gp_tests/Dockerfile

@@ -13,7 +13,7 @@ COPY cmd/ cmd/
 COPY main/ main/
 COPY utility/ utility/
 COPY Makefile Makefile
-
+ENV USE_BROTLI 1
 
 RUN sed -i 's|#cgo LDFLAGS: -lbrotli.*|&-static -lbrotlicommon-static -lm|' \
         vendor/github.com/google/brotli/go/cbrotli/cgo.go && \

docs/README.md

@@ -229,6 +229,7 @@ The following steps describe how to build WAL-G for PostgreSQL, but the process
 
 Optional:
 
+- To build with brotli compressor and decompressor, set the `USE_BROTLI` environment variable.
 - To build with libsodium, set the `USE_LIBSODIUM` environment variable.
 - To build with lzo decompressor, set the `USE_LZO` environment variable.
 
@@ -254,6 +255,7 @@ main/pg/wal-g --version
 Users can also install WAL-G by using `make pg_install`. Specifying the `GOBIN` environment variable before installing allows the user to specify the installation location. By default, `make pg_install` puts the compiled binary in the root directory (`/`).
 
 ```sh
+export USE_BROTLI=1
 export USE_LIBSODIUM=1
 export USE_LZO=1
 make pg_clean
@@ -266,6 +268,7 @@ GOBIN=/usr/local/bin make pg_install
 ```sh
 # brew command is Homebrew for Mac OS
 brew install cmake
+export USE_BROTLI=1
 export USE_LIBSODIUM="true" # since we're linking libsodium later
 ./link_brotli.sh
 ./link_libsodium.sh
@@ -283,6 +286,7 @@ The compiled binary to run is `main/pg/wal-g`
 
 WAL-G relies heavily on unit tests. These tests do not require S3 configuration as the upload/download parts are tested using mocked objects. Unit tests can be run using
 ```bash
+export USE_BROTLI=1
 make unittest
 ```
 For more information on testing, please consult [test](test), [testtools](testtools) and `unittest` section in [Makefile](Makefile).
@@ -291,6 +295,7 @@ WAL-G will perform a round-trip compression/decompression test that generates a
 
 Test coverage can be obtained using:
 ```bash
+export USE_BROTLI=1
 make coverage
 ```
 This command generates `coverage.out` file and opens HTML representation of the coverage.

tests_func/Dockerfile.mongodb

@@ -11,6 +11,7 @@ ARG MONGO_VERSION=
 ARG MONGO_TOOLS_VERSION=100.4.0
 ENV TMP_DIR /var/tmp/wal-g
 ENV WALG_CONF_DIR /etc/wal-g/
+ENV USE_BROTLI 1
 
 RUN echo "deb http://$MONGO_REPO/apt/ubuntu bionic/${MONGO_PACKAGE}/${MONGO_MAJOR} multiverse" | tee "/etc/apt/sources.list.d/mongodb_${MONGO_MAJOR}.list" && \
     wget -qO - https://www.mongodb.org/static/pgp/server-${MONGO_MAJOR}.asc | apt-key add -
@@ -48,6 +49,7 @@ COPY staging/wal-g ${WALG_REPO}/
 
 WORKDIR ${WALG_REPO}
 
+
 RUN make link_external_deps
 RUN make BUILD_ARGS=-race mongo_build && \
     cp main/mongo/wal-g /usr/bin/wal-g

tests_func/Dockerfile.redis

@@ -12,6 +12,7 @@ ENV USER redis
 ENV GROUP redis
 ENV USERGROUP $USER:$GROUP
 ENV HOME /home/$USER
+ENV USE_BROTLI 1
 
 # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added
 RUN groupadd -r -g 999 $USER && useradd -r -g $GROUP -u 999 $USER