diff --git a/Makefile b/Makefile index 2029fd2bd456..01fcb368f96e 100644 --- a/Makefile +++ b/Makefile @@ -134,30 +134,17 @@ define restore_hex_cache_from_erl_term endef # -------------------------------------------------------------------- -# Distribution. +# Distribution - common variables and generic functions. # -------------------------------------------------------------------- -.PHONY: source-dist clean-source-dist - -SOURCE_DIST_BASE ?= rabbitmq-server -SOURCE_DIST_SUFFIXES ?= tar.xz -SOURCE_DIST ?= $(PACKAGES_DIR)/$(SOURCE_DIST_BASE)-$(PROJECT_VERSION) - -# The first source distribution file is used by packages: if the archive -# type changes, you must update all packages' Makefile. -SOURCE_DIST_FILES = $(addprefix $(SOURCE_DIST).,$(SOURCE_DIST_SUFFIXES)) - -.PHONY: $(SOURCE_DIST_FILES) - -source-dist: $(SOURCE_DIST_FILES) - @: - RSYNC ?= rsync RSYNC_V_0 = RSYNC_V_1 = -v RSYNC_V_2 = -v RSYNC_V = $(RSYNC_V_$(V)) -RSYNC_FLAGS += -a $(RSYNC_V) \ +BASE_RSYNC_FLAGS += -a $(RSYNC_V) \ + --delete \ + --delete-excluded \ --exclude '.sw?' --exclude '.*.sw?' \ --exclude '*.beam' \ --exclude '*.d' \ @@ -188,12 +175,10 @@ RSYNC_FLAGS += -a $(RSYNC_V) \ --exclude '$(notdir $(DEPS_DIR))/' \ --exclude 'hexer*' \ --exclude 'logs/' \ - --exclude 'packaging' \ --exclude 'PKG_*.md' \ --exclude '/plugins/' \ --include 'cli/plugins' \ --exclude '$(notdir $(DIST_DIR))/' \ - --exclude 'test' \ --exclude '/$(notdir $(PACKAGES_DIR))/' \ --exclude '/PACKAGES/' \ --exclude '/amqp_client/doc/' \ @@ -208,9 +193,21 @@ RSYNC_FLAGS += -a $(RSYNC_V) \ --exclude '/ranch/doc/' \ --exclude '/ranch/examples/' \ --exclude '/sockjs/examples/' \ - --exclude '/workflow_sources/' \ - --delete \ - --delete-excluded + --exclude '/workflow_sources/' + +SOURCE_DIST_RSYNC_FLAGS += $(BASE_RSYNC_FLAGS) \ + --exclude 'packaging' \ + --exclude 'test' + +# For source-bundle, explicitly include folders that are needed +# for tests to execute. These are added before excludes from +# the base flags so rsync honors the first match. +SOURCE_BUNDLE_RSYNC_FLAGS += \ + --include 'rabbit_shovel_test/ebin' \ + --include 'rabbit_shovel_test/ebin/*' \ + --include 'rabbitmq_ct_helpers/tools' \ + --include 'rabbitmq_ct_helpers/tools/*' \ + $(BASE_RSYNC_FLAGS) TAR ?= tar TAR_V_0 = @@ -228,65 +225,125 @@ ZIP_V_1 = ZIP_V_2 = ZIP_V = $(ZIP_V_$(V)) -.PHONY: $(SOURCE_DIST) -.PHONY: clean-source-dist distclean-packages clean-unpacked-source-dist - -$(SOURCE_DIST): $(ERLANG_MK_RECURSIVE_DEPS_LIST) - $(verbose) mkdir -p $(dir $@) - $(gen_verbose) $(RSYNC) $(RSYNC_FLAGS) ./ $@/ - $(verbose) echo "$(PROJECT_DESCRIPTION) $(PROJECT_VERSION)" > "$@/git-revisions.txt" - $(verbose) echo "$(PROJECT) $$(git rev-parse HEAD) $$(git describe --tags --exact-match 2>/dev/null || git symbolic-ref -q --short HEAD)" >> "$@/git-revisions.txt" - $(verbose) echo "$$(TZ= git --no-pager log -n 1 --format='%cd' --date='format-local:%Y%m%d%H%M.%S')" > "$@.git-times.txt" - $(verbose) cat packaging/common/LICENSE.head > $@/LICENSE - $(verbose) mkdir -p $@/deps/licensing - $(verbose) set -e; for dep in $$(cat $(ERLANG_MK_RECURSIVE_DEPS_LIST) | LC_COLLATE=C sort); do \ - $(RSYNC) $(RSYNC_FLAGS) \ - $$dep \ - $@/deps; \ +ifeq ($(shell tar --version | grep -c "GNU tar"),0) +# Skip all flags if this is Darwin (a.k.a. macOS, a.k.a. OS X) +ifeq ($(shell uname | grep -c "Darwin"),0) +TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS = --uid 0 \ + --gid 0 \ + --numeric-owner \ + --no-acls \ + --no-fflags \ + --no-xattrs +endif +else +TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS = --owner 0 \ + --group 0 \ + --numeric-owner +endif + +DIST_SUFFIXES ?= tar.xz + +# Function to create distribution targets +# Args: $(1) - Full distribution path +# $(2) - RSYNC flags to use +define create_dist_target +$(1): $(ERLANG_MK_RECURSIVE_DEPS_LIST) + $${verbose} mkdir -p $$(dir $$@) + $${gen_verbose} $${RSYNC} $(2) ./ $$@/ + $${verbose} echo "$(PROJECT_DESCRIPTION) $(PROJECT_VERSION)" > $$@/git-revisions.txt + $${verbose} echo "$(PROJECT) $$$$(git rev-parse HEAD) $$$$(git describe --tags --exact-match 2>/dev/null || git symbolic-ref -q --short HEAD)" >> $$@/git-revisions.txt + $${verbose} echo "$$$$(TZ= git --no-pager log -n 1 --format='%cd' --date='format-local:%Y%m%d%H%M.%S')" > $$@.git-times.txt + $${verbose} cat packaging/common/LICENSE.head > $$@/LICENSE + $${verbose} mkdir -p $$@/deps/licensing + $${verbose} set -e; for dep in $$$$(cat $(ERLANG_MK_RECURSIVE_DEPS_LIST) | LC_COLLATE=C sort); do \ + $${RSYNC} $(2) \ + $$$$dep \ + $$@/deps; \ rm -f \ - $@/deps/rabbit_common/rebar.config \ - $@/deps/rabbit_common/rebar.lock; \ - if test -f $@/deps/$$(basename $$dep)/erlang.mk && \ - test "$$(wc -l $@/deps/$$(basename $$dep)/erlang.mk | awk '{print $$1;}')" = "1" && \ - grep -qs -E "^[[:blank:]]*include[[:blank:]]+(erlang\.mk|.*/erlang\.mk)$$" $@/deps/$$(basename $$dep)/erlang.mk; then \ - echo "include ../../erlang.mk" > $@/deps/$$(basename $$dep)/erlang.mk; \ + $$@/deps/rabbit_common/rebar.config \ + $$@/deps/rabbit_common/rebar.lock; \ + if test -f $$@/deps/$$$$(basename $$$$dep)/erlang.mk && \ + test "$$$$(wc -l $$@/deps/$$$$(basename $$$$dep)/erlang.mk | awk '{print $$$$1;}')" = "1" && \ + grep -qs -E "^[[:blank:]]*include[[:blank:]]+(erlang\.mk|.*/erlang\.mk)$$$$" $$@/deps/$$$$(basename $$$$dep)/erlang.mk; then \ + echo "include ../../erlang.mk" > $$@/deps/$$$$(basename $$$$dep)/erlang.mk; \ fi; \ - sed -E -i.bak "s|^[[:blank:]]*include[[:blank:]]+\.\./.*erlang.mk$$|include ../../erlang.mk|" \ - $@/deps/$$(basename $$dep)/Makefile && \ - rm $@/deps/$$(basename $$dep)/Makefile.bak; \ - mix_exs=$@/deps/$$(basename $$dep)/mix.exs; \ - if test -f $$mix_exs; then \ - (cd $$(dirname "$$mix_exs") && \ - (test -d $@/deps/.hex || env DEPS_DIR=$@/deps MIX_HOME=$@/deps/.mix HEX_HOME=$@/deps/.hex MIX_ENV=prod FILL_HEX_CACHE=yes mix local.hex --force) && \ - env DEPS_DIR=$@/deps MIX_HOME=$@/deps/.mix HEX_HOME=$@/deps/.hex MIX_ENV=prod FILL_HEX_CACHE=yes mix deps.get --only prod && \ + sed -E -i.bak "s|^[[:blank:]]*include[[:blank:]]+\.\./.*erlang.mk$$$$|include ../../erlang.mk|" \ + $$@/deps/$$$$(basename $$$$dep)/Makefile && \ + rm $$@/deps/$$$$(basename $$$$dep)/Makefile.bak; \ + mix_exs=$$@/deps/$$$$(basename $$$$dep)/mix.exs; \ + if test -f $$$$mix_exs; then \ + (cd $$$$(dirname "$$$$mix_exs") && \ + (test -d $$@/deps/.hex || env DEPS_DIR=$$@/deps MIX_HOME=$$@/deps/.mix HEX_HOME=$$@/deps/.hex MIX_ENV=prod FILL_HEX_CACHE=yes mix local.hex --force) && \ + env DEPS_DIR=$$@/deps MIX_HOME=$$@/deps/.mix HEX_HOME=$$@/deps/.hex MIX_ENV=prod FILL_HEX_CACHE=yes mix deps.get --only prod && \ cp $(CURDIR)/mk/rabbitmq-mix.mk . && \ rm -rf _build deps); \ fi; \ - if test -f "$$dep/license_info"; then \ - cp "$$dep/license_info" "$@/deps/licensing/license_info_$$(basename "$$dep")"; \ - cat "$$dep/license_info" >> $@/LICENSE; \ + if test -f "$$$$dep/license_info"; then \ + cp "$$$$dep/license_info" "$$@/deps/licensing/license_info_$$$$(basename $$$$dep)"; \ + cat "$$$$dep/license_info" >> $$@/LICENSE; \ fi; \ - find "$$dep" -maxdepth 1 -name 'LICENSE-*' -exec cp '{}' $@/deps/licensing \; ; \ - (cd $$dep; \ - echo "$$(basename "$$dep") $$(git rev-parse HEAD) $$(git describe --tags --exact-match 2>/dev/null || git symbolic-ref -q --short HEAD)") \ - >> "$@/git-revisions.txt"; \ - ! test -d $$dep/.git || (cd $$dep; \ - echo "$$(env TZ= git --no-pager log -n 1 --format='%cd' --date='format-local:%Y%m%d%H%M.%S')") \ - >> "$@.git-times.txt"; \ + find "$$$$dep" -maxdepth 1 -name 'LICENSE-*' -exec cp '{}' $$@/deps/licensing \; ; \ + (cd $$$$dep; \ + echo "$$$$(basename "$$$$dep") $$$$(git rev-parse HEAD) $$$$(git describe --tags --exact-match 2>/dev/null || git symbolic-ref -q --short HEAD)") \ + >> "$$@/git-revisions.txt"; \ + ! test -d $$$$dep/.git || (cd $$$$dep; \ + echo "$$$$(env TZ= git --no-pager log -n 1 --format='%cd' --date='format-local:%Y%m%d%H%M.%S')") \ + >> "$$@.git-times.txt"; \ done - $(verbose) cat packaging/common/LICENSE.tail >> $@/LICENSE - $(verbose) find $@/deps/licensing -name 'LICENSE-*' -exec cp '{}' $@ \; - $(verbose) rm -rf $@/deps/licensing - $(verbose) for file in $$(find $@ -name '*.app.src'); do \ + $${verbose} cat packaging/common/LICENSE.tail >> $$@/LICENSE + $${verbose} find $$@/deps/licensing -name 'LICENSE-*' -exec cp '{}' $$@ \; + $${verbose} rm -rf $$@/deps/licensing + $${verbose} for file in $$$$(find $$@ -name '*.app.src'); do \ sed -E -i.bak \ -e 's/[{]vsn[[:blank:]]*,[[:blank:]]*(""|"0.0.0")[[:blank:]]*}/{vsn, "$(PROJECT_VERSION)"}/' \ -e 's/[{]broker_version_requirements[[:blank:]]*,[[:blank:]]*\[\][[:blank:]]*}/{broker_version_requirements, ["$(PROJECT_VERSION)"]}/' \ - $$file; \ - rm $$file.bak; \ + $$$$file; \ + rm $$$$file.bak; \ done - $(verbose) echo "PLUGINS := $(PLUGINS)" > $@/plugins.mk -# Remember the latest Git timestamp. - $(verbose) sort -r < "$@.git-times.txt" | head -n 1 > "$@.git-time.txt" + $${verbose} echo "PLUGINS := $(PLUGINS)" > $$@/plugins.mk + $${verbose} sort -r < "$$@.git-times.txt" | head -n 1 > "$$@.git-time.txt" + $${verbose} $$(call erlang,$$(call dump_hex_cache_to_erl_term,$$(call core_native_path,$$@),$$(call core_native_path,$$@.git-time.txt))) + $${verbose} find $$@ -print0 | xargs -0 touch -t "$$$$(cat $$@.git-time.txt)" + $${verbose} rm "$$@.git-times.txt" "$$@.git-time.txt" + +$(1).manifest: $(1) + $${gen_verbose} cd $$(dir $$@) && \ + find $$(notdir $$<) | LC_COLLATE=C sort > $$@ + +$(1).tar.xz: $(1).manifest + $${gen_verbose} cd $$(dir $$@) && \ + $${TAR} $${TAR_V} $${TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS} --no-recursion -T $$(notdir $$<) -cf - | \ + $${XZ} > $$@ + +$(1).tar.gz: $(1).manifest + $${gen_verbose} cd $$(dir $$@) && \ + $${TAR} $${TAR_V} $${TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS} --no-recursion -T $$(notdir $$<) -cf - | \ + $${GZIP} --best > $$@ + +$(1).tar.bz2: $(1).manifest + $${gen_verbose} cd $$(dir $$@) && \ + $${TAR} $${TAR_V} $${TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS} --no-recursion -T $$(notdir $$<) -cf - | \ + $${BZIP2} > $$@ + +$(1).zip: $(1).manifest + $${verbose} rm -f $$@ + $${gen_verbose} cd $$(dir $$@) && \ + $${ZIP} $${ZIP_V} --names-stdin $$@ < $$(notdir $$<) + +endef + +# Function to create clean targets +# Args: $(1) - Base name (e.g. SOURCE_DIST_BASE or BUNDLE_DIST_BASE) +define create_clean_targets +.PHONY: clean-$(1) + +clean-$(1): + $${gen_verbose} rm -rf -- $(1)-* + +# Add each clean target to the clean:: rule +clean:: clean-$(1) +endef + # Mix Hex component requires a cache file, otherwise it refuses to build # offline... That cache is an ETS table with all the applications we # depend on, plus some versioning informations and checksums. There @@ -300,11 +357,6 @@ $(SOURCE_DIST): $(ERLANG_MK_RECURSIVE_DEPS_LIST) # # The ETS file must be recreated before compiling RabbitMQ. See the # `restore-hex-cache-ets-file` Make target. - $(verbose) $(call erlang,$(call dump_hex_cache_to_erl_term,$(call core_native_path,$@),$(call core_native_path,$@.git-time.txt))) -# Fix file timestamps to have reproducible source archives. - $(verbose) find $@ -print0 | xargs -0 touch -t "$$(cat "$@.git-time.txt")" - $(verbose) rm "$@.git-times.txt" "$@.git-time.txt" - define dump_hex_cache_to_erl_term In = "$(1)/deps/.hex/cache.ets", Out = "$(1)/deps/.hex/cache.erl", @@ -333,50 +385,35 @@ define dump_hex_cache_to_erl_term init:stop(). endef -$(SOURCE_DIST).manifest: $(SOURCE_DIST) - $(gen_verbose) cd $(dir $(SOURCE_DIST)) && \ - find $(notdir $(SOURCE_DIST)) | LC_COLLATE=C sort > $@ +# -------------------------------------------------------------------- +# Distribution - public targets +# -------------------------------------------------------------------- -ifeq ($(shell tar --version | grep -c "GNU tar"),0) -# Skip all flags if this is Darwin (a.k.a. macOS, a.k.a. OS X) -ifeq ($(shell uname | grep -c "Darwin"),0) -TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS = --uid 0 \ - --gid 0 \ - --numeric-owner \ - --no-acls \ - --no-fflags \ - --no-xattrs -endif -else -TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS = --owner 0 \ - --group 0 \ - --numeric-owner -endif +SOURCE_DIST_BASE ?= rabbitmq-server +SOURCE_DIST ?= $(PACKAGES_DIR)/$(SOURCE_DIST_BASE)-$(PROJECT_VERSION) +SOURCE_DIST_FILES = $(addprefix $(SOURCE_DIST).,$(DIST_SUFFIXES)) -$(SOURCE_DIST).tar.gz: $(SOURCE_DIST).manifest - $(gen_verbose) cd $(dir $(SOURCE_DIST)) && \ - $(TAR) $(TAR_V) $(TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS) --no-recursion -T $(SOURCE_DIST).manifest -cf - | \ - $(GZIP) --best > $@ +.PHONY: source-dist +source-dist: $(SOURCE_DIST_FILES) + @: + +$(eval $(call create_dist_target,$(SOURCE_DIST),$(SOURCE_DIST_RSYNC_FLAGS))) -$(SOURCE_DIST).tar.bz2: $(SOURCE_DIST).manifest - $(gen_verbose) cd $(dir $(SOURCE_DIST)) && \ - $(TAR) $(TAR_V) $(TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS) --no-recursion -T $(SOURCE_DIST).manifest -cf - | \ - $(BZIP2) > $@ +SOURCE_BUNDLE_BASE ?= rabbitmq-server-bundle +SOURCE_BUNDLE_DIST ?= $(PACKAGES_DIR)/$(SOURCE_BUNDLE_BASE)-$(PROJECT_VERSION) +SOURCE_BUNDLE_FILES = $(addprefix $(SOURCE_BUNDLE_DIST).,$(DIST_SUFFIXES)) -$(SOURCE_DIST).tar.xz: $(SOURCE_DIST).manifest - $(gen_verbose) cd $(dir $(SOURCE_DIST)) && \ - $(TAR) $(TAR_V) $(TAR_FLAGS_FOR_REPRODUCIBLE_BUILDS) --no-recursion -T $(SOURCE_DIST).manifest -cf - | \ - $(XZ) > $@ +.PHONY: source-bundle +source-bundle: $(SOURCE_BUNDLE_FILES) + @: -$(SOURCE_DIST).zip: $(SOURCE_DIST).manifest - $(verbose) rm -f $@ - $(gen_verbose) cd $(dir $(SOURCE_DIST)) && \ - $(ZIP) $(ZIP_V) --names-stdin $@ < $(SOURCE_DIST).manifest +$(eval $(call create_dist_target,$(SOURCE_BUNDLE_DIST),$(SOURCE_BUNDLE_RSYNC_FLAGS))) -clean:: clean-source-dist +# Create the clean targets for both distributions +$(eval $(call create_clean_targets,$(SOURCE_DIST_BASE))) +$(eval $(call create_clean_targets,$(SOURCE_BUNDLE_BASE))) -clean-source-dist: - $(gen_verbose) rm -rf -- $(SOURCE_DIST_BASE)-* +.PHONY: distclean-packages clean-unpacked-source-dist distclean:: distclean-packages