Merge pull request #7133 from zmstone/chore-build-with-rockylinux

build: rockylinux for 5.0
emqx · Feb 28, 2022 · 3bdcf18 · 3bdcf18
2 parents d8cf442 + b68cb96
commit 3bdcf18
Show file tree

Hide file tree

Showing 14 changed files with 284 additions and 203 deletions.
diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
@@ -227,10 +227,9 @@ jobs:
           - ubuntu16.04
           - debian10
           - debian9
-          # - opensuse
+          - rockylinux8
           - centos7
           - raspbian10
-          # - raspbian9
         build_machine:
           - aws-arm64
           - ubuntu-20.04
@@ -313,11 +312,11 @@ jobs:
         fi
         echo "pwd is $PWD"
         PkgTypes="tgz pkg"
-        WithElixir=""
+        IsElixir="no"
         if [ ${{ matrix.build_elixir }} = "with_elixir" ]; then
           PkgTypes="tgz"
           # set Elixir build flag
-          WithElixir="--with-elixir"
+          IsElixir="yes"
         fi
         for PKGTYPE in ${PkgTypes};
         do
@@ -326,7 +325,8 @@ jobs:
             --pkgtype "${PKGTYPE}" \
             --arch "${ARCH}" \
             --otp "${OTP}" \
-            --elixir "${ELIXIR}" "$WithElixir" \
+            --elixir "${IsElixir}" \
+            --elixir-vsn "${ELIXIR}" \
             --system "${SYSTEM}" \
             --builder "ghcr.io/emqx/emqx-builder/5.0-7:${ELIXIR}-${OTP}-${SYSTEM}"
         done

diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
@@ -41,7 +41,7 @@ jobs:
         - 1.13.3
         os:
         - ubuntu20.04
-        - centos7
+        - rockylinux8
 
     container: "ghcr.io/emqx/emqx-builder/5.0-7:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
 
@@ -51,8 +51,6 @@ jobs:
       run: |
         echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
         echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
-        echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh ${{ matrix.profile }})-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV
-        echo "EMQX_ELIXIRPKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh ${{ matrix.profile }})-elixir${{ matrix.elixir }}-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV
     - name: Get deps git refs for cache
       id: deps-refs
       run: |
@@ -75,23 +73,23 @@ jobs:
     - name: build and test tgz package
       run: |
         make ${EMQX_NAME}-tgz
-        ./scripts/pkg-tests.sh "$EMQX_PKG_NAME" tgz amd64
+        ./scripts/pkg-tests.sh ${EMQX_NAME}-tgz
     - name: run static checks
       if: contains(matrix.os, 'ubuntu')
       run: |
         make static_checks
     - name: build and test deb/rpm packages
       run: |
         make ${EMQX_NAME}-pkg
-        ./scripts/pkg-tests.sh "$EMQX_PKG_NAME" pkg amd64
+        ./scripts/pkg-tests.sh ${EMQX_NAME}-pkg
     - name: build and test tgz package (Elixir)
       run: |
         make ${EMQX_NAME}-elixir-tgz
-        ./scripts/pkg-tests.sh "$EMQX_ELIXIRPKG_NAME" tgz amd64
+        ./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-tgz
     - name: build and test deb/rpm packages (Elixir)
       run: |
-        make ${EMQX_NAME}-elixirpkg
-        ./scripts/pkg-tests.sh "$EMQX_ELIXIRPKG_NAME" pkg amd64
+        make ${EMQX_NAME}-elixir-pkg
+        ./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-pkg
     - uses: actions/upload-artifact@v2
       with:
         name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }}

diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml
@@ -12,12 +12,18 @@ jobs:
 
     strategy:
       fail-fast: true
-
+    matrix:
+      profile:
+        - emqx-enterprise
+      otp:
+        - 24.2.1-1
+      os:
+        - ubuntu20.04
     steps:
     - uses: actions/checkout@v2
     - uses: actions/download-artifact@v2
       with:
-        name:  emqx-24.1.5-4-ubuntu20.04
+        name:  ${{ matrix.profile }}-${{ matrix.otp }}-${{ matrix.os }}
         workflow: build_slim_packages.yaml
         path: .
     - name: Run spellcheck

diff --git a/Makefile b/Makefile
@@ -10,7 +10,6 @@ export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
 export EMQX_DASHBOARD_VERSION ?= v0.20.0
 export DOCKERFILE := deploy/docker/Dockerfile
-export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing
 ifeq ($(OS),Windows_NT)
 	export REBAR_COLOR=none
 	FIND=/usr/bin/find
@@ -214,18 +213,6 @@ endef
 ALL_DOCKERS = $(REL_PROFILES) $(REL_PROFILES:%=%-elixir)
 $(foreach zt,$(ALL_DOCKERS),$(eval $(call gen-docker-target,$(zt))))
 
-## emqx-docker-testing
-## emqx-enterprise-docker-testing
-## is to directly copy a extracted tgz-package to a
-## base image such as ubuntu20.04. Mostly for testing
-.PHONY: $(REL_PROFILES:%=%-docker-testing)
-define gen-docker-target-testing
-$1-docker-testing: $(COMMON_DEPS)
-	@$(BUILD) $1 docker-testing
-endef
-ALL_TGZS = $(REL_PROFILES)
-$(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target-testing,$(zt))))
-
 .PHONY:
 conf-segs:
 	@scripts/merge-config.escript
@@ -235,15 +222,15 @@ conf-segs:
 $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) mix-deps-get
 	@$(BUILD) $(subst -elixir,,$(@)) elixir
 
-.PHONY: $(REL_PROFILES:%=%-elixirpkg)
-define gen-elixirpkg-target
+.PHONY: $(REL_PROFILES:%=%-elixir-pkg)
+define gen-elixir-pkg-target
 # the Elixir places the tar in a different path than Rebar3
-$1-elixirpkg: $1-pkg-elixir
+$1-elixir-pkg: $1-pkg-elixir
 	@env TAR_PKG_DIR=_build/$1-pkg \
 	     IS_ELIXIR=yes \
 	     $(BUILD) $1-pkg pkg
 endef
-$(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixirpkg-target,$(pt))))
+$(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixir-pkg-target,$(pt))))
 
 .PHONY: $(REL_PROFILES:%=%-elixir-tgz)
 define gen-elixir-tgz-target

diff --git a/build b/build
@@ -95,13 +95,15 @@ make_relup() {
     local rel_dir="_build/$PROFILE/rel/emqx"
     mkdir -p "${rel_dir}/lib"
     mkdir -p "${rel_dir}/releases"
+    local name_pattern
+    name_pattern="${PROFILE}-$(./pkg-vsn.sh "$PROFILE" --vsn_matcher)"
     local releases=()
     while read -r tgzfile ; do
         local base_vsn
         base_vsn="$(echo "$tgzfile" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta)\.[0-9])?(-[0-9a-f]{8})?" | head -1)"
         tar -C "$rel_dir" -zxf ---keep-old-files "$tgzfile" emqx/releases emqx/lib
         releases+=( "$base_vsn" )
-    done < <("$FIND" _upgrade_base -maxdepth 1 -name "$PROFILE*${SYSTEM}-${ARCH}.tar.gz" -type f)
+    done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.tar.gz" -type f)
     if [ ${#releases[@]} -eq 0 ]; then
         log "No upgrade base found, relup ignored"
         return 0
@@ -141,15 +143,16 @@ make_tgz() {
       ELIXIR_MAKE_TAR=yes make_elixir_rel
 
       local relpath="_build/${PROFILE}"
-      target_name="${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
+      full_vsn="$(./pkg-vsn.sh "$PROFILE" --long --elixir)"
     else
       # build the src_tarball again to ensure relup is included
       # elixir does not have relup yet.
       make_rel
 
       local relpath="_build/${PROFILE}/rel/emqx"
-      target_name="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
+      full_vsn="$(./pkg-vsn.sh "$PROFILE" --long)"
     fi
+    target_name="${PROFILE}-${full_vsn}.tar.gz"
     target="${pkgpath}/${target_name}"
 
     src_tarball="${relpath}/emqx-${PKG_VSN}.tar.gz"
@@ -201,46 +204,6 @@ make_docker() {
        -f "${DOCKERFILE}" .
 }
 
-## This function accepts any base docker image,
-## a emqx tgz-image, and a image tag (for the image to be built),
-## to build a docker image which runs EMQX
-##
-## Export below variables to quickly build an image
-##
-## Name               Default                         Example
-## ---------------------------------------------------------------------
-## EMQX_BASE_IMAGE    current os                      centos:7
-## EMQX_TGZ_packages/<current-tgz-target>  /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.tar.gz
-## EMQX_IMAGE_TAG     emqx/emqx:<current-vns-rel>     emqx/emqx:testing-tag
-##
-make_docker_testing() {
-    if [ -z "${EMQX_BASE_IMAGE:-}" ]; then
-        case "$SYSTEM" in
-            ubuntu20*)
-                EMQX_BASE_IMAGE="ubuntu:20.04"
-                ;;
-            *)
-                echo "Unsupported testing base image for $SYSTEM"
-                exit 1
-                ;;
-        esac
-    fi
-    EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}"
-    local default_tgz
-    default_tgz="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
-    local tgz="${EMQX_TGZ_PACKAGE:-$default_tgz}"
-    if [ ! -f "$tgz" ]; then
-        log "ERROR: $tgz not built?"
-        exit 1
-    fi
-    set -x
-    docker build \
-       --build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \
-       --build-arg EMQX_TGZ_PACKAGE="${tgz}" \
-       --tag "$EMQX_IMAGE_TAG" \
-       -f "${DOCKERFILE_TESTING}" .
-}
-
 function join {
   local IFS="$1"
   shift
@@ -310,9 +273,6 @@ case "$ARTIFACT" in
     docker)
         make_docker
         ;;
-    docker-testing)
-        make_docker_testing
-        ;;
     elixir)
         make_elixir_rel
         ;;

diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile
@@ -6,16 +6,10 @@ BUILT := $(SRCDIR)/BUILT
 
 EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
 
-ifeq ($(IS_ELIXIR), yes)
-  ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN)
-else
-  ELIXIR_PKG_VSN :=
-endif
-
 TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx
 TAR_PKG     := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz
 SOURCE_PKG  := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture)
-TARGET_PKG  := $(EMQX_NAME)-$(PKG_VSN)$(ELIXIR_PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
+TARGET_PKG  := $(EMQX_NAME)-$(shell $(EMQX_REL)/pkg-vsn.sh $(EMQX_NAME) --long)
 
 .PHONY: all
 all: | $(BUILT)
@@ -29,6 +23,9 @@ all: | $(BUILT)
 	cd $(SRCDIR) && dpkg-buildpackage -us -uc
 	mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME)
 	cp $(SRCDIR)/../$(SOURCE_PKG).deb $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb
+	sha256sum $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb | head -c 64 > \
+		$(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb.sha256
+
 
 $(BUILT):
 	mkdir -p $(TOPDIR) $(SRCDIR)

diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile
@@ -16,16 +16,10 @@ endif
 
 EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
 
-ifeq ($(IS_ELIXIR), yes)
-  ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN)
-else
-  ELIXIR_PKG_VSN :=
-endif
-
 TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx
 TAR_PKG     := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz
 SOURCE_PKG  := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m)
-TARGET_PKG  := $(EMQX_NAME)-$(PKG_VSN)$(ELIXIR_PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
+TARGET_PKG  := $(EMQX_NAME)-$(shell $(EMQX_REL)/pkg-vsn.sh $(EMQX_NAME) --long)
 
 SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi)
 # Not $(PWD) as it does not work for make -C
@@ -55,6 +49,8 @@ all: | $(BUILT)
 		emqx.spec
 	mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME)
 	cp $(TOPDIR)/RPMS/$(shell uname -m)/$(SOURCE_PKG).rpm $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm
+	sha256sum $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm | head -c 64 > \
+		$(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm.sha256
 
 $(BUILT):
 	mkdir -p $(TOPDIR) $(SRCDIR) $(SRCDIR)/BUILT

diff --git a/mix.exs b/mix.exs
@@ -511,7 +511,7 @@ defmodule EMQXUmbrella.MixProject do
   # The `:tar` built-in step in Mix Release does not currently add the
   # `etc` directory into the resulting tarball.  The workaround is to
   # add those to the `:overlays` key before running `:tar`.
-  # See: https://hexdocs.pm/mix/1.13.2/Mix.Release.html#__struct__/0
+  # See: https://hexdocs.pm/mix/1.13.3/Mix.Release.html#__struct__/0
   defp prepare_tar_overlays(release) do
     Map.update!(
       release,