Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/release-50' into 1214-sync-maste…
Browse files Browse the repository at this point in the history 
…r-upstreams
  • Loading branch information
@zmstone
zmstone committed Dec 14, 2022
2 parents a3a640d + d2cc7d3 commit 42c58e2
Show file tree
Hide file tree
Showing 75 changed files with 2,169 additions and 566 deletions.
14 changes: 10 additions & 4 deletions .ci/docker-compose-file/Makefile.local
View file
Expand Up @@ -16,7 +16,7 @@ up:
REDIS_TAG=6 \
MONGO_TAG=5 \
PGSQL_TAG=13 \
docker-compose \
docker compose \
-f .ci/docker-compose-file/docker-compose.yaml \
-f .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-mongo-single-tls.yaml \
Expand All @@ -28,10 +28,13 @@ up:
-f .ci/docker-compose-file/docker-compose-redis-single-tls.yaml \
-f .ci/docker-compose-file/docker-compose-redis-sentinel-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml \
up -d --build
-f .ci/docker-compose-file/docker-compose-redis-cluster-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-redis-cluster-tls.yaml \
-f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
up -d --build --remove-orphans

down:
docker-compose \
docker compose \
-f .ci/docker-compose-file/docker-compose.yaml \
-f .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-mongo-single-tls.yaml \
Expand All @@ -43,7 +46,10 @@ down:
-f .ci/docker-compose-file/docker-compose-redis-single-tls.yaml \
-f .ci/docker-compose-file/docker-compose-redis-sentinel-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml \
down
-f .ci/docker-compose-file/docker-compose-redis-cluster-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-redis-cluster-tls.yaml \
-f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
down --remove-orphans

ct:
docker exec -i "$(CONTAINER)" bash -c "rebar3 ct --name 'test@127.0.0.1' -v --suite $(SUITE)"
Expand Down
4 changes: 2 additions & 2 deletions .ci/docker-compose-file/docker-compose-redis-cluster-tcp.yaml
View file
@@ -1,9 +1,9 @@
version: '3.9'

services:
redis_server:
redis_cluster:
image: redis:${REDIS_TAG}
container_name: redis
container_name: redis-cluster
volumes:
- ./redis/:/data/conf
command: bash -c "/bin/bash /data/conf/redis.sh --node cluster && tail -f /var/log/redis-server.log"
Expand Down
4 changes: 2 additions & 2 deletions .ci/docker-compose-file/docker-compose-redis-cluster-tls.yaml
View file
@@ -1,8 +1,8 @@
version: '3.9'

services:
redis_server:
container_name: redis
redis_cluster_tls:
container_name: redis-cluster-tls
image: redis:${REDIS_TAG}
volumes:
- ../../apps/emqx/etc/certs/cacert.pem:/etc/certs/ca.crt
Expand Down
6 changes: 3 additions & 3 deletions .ci/docker-compose-file/redis/.gitignore
View file
@@ -1,3 +1,3 @@
r7000i.log
r7001i.log
r7002i.log
r700?i.log
nodes.700?.conf
*.rdb
7 changes: 4 additions & 3 deletions .ci/docker-compose-file/redis/redis-tls.conf
View file
@@ -1,11 +1,12 @@
daemonize yes
bind 0.0.0.0 ::
logfile /var/log/redis-server.log
protected-mode no
requirepass public
masterauth public

tls-cert-file /etc/certs/redis.crt
tls-key-file /etc/certs/redis.key
tls-ca-cert-file /etc/certs/ca.crt
tls-replication yes
tls-cluster yes
protected-mode no
requirepass public
masterauth public
1 change: 1 addition & 0 deletions .ci/docker-compose-file/redis/redis.conf
View file
@@ -1,5 +1,6 @@
daemonize yes
bind 0.0.0.0 ::
logfile /var/log/redis-server.log
protected-mode no
requirepass public
masterauth public
91 changes: 44 additions & 47 deletions .ci/docker-compose-file/redis/redis.sh
View file
Expand Up @@ -16,13 +16,8 @@ case $key in
shift # past argument
shift # past value
;;
-t)
tls="$2"
shift # past argument
shift # past value
;;
--tls-enabled)
tls=1
tls=true
shift # past argument
;;
*)
Expand All @@ -37,77 +32,79 @@ rm -f \
/data/conf/r7002i.log \
/data/conf/nodes.7000.conf \
/data/conf/nodes.7001.conf \
/data/conf/nodes.7002.conf ;
/data/conf/nodes.7002.conf

if [ "${node}" = "cluster" ] ; then
if $tls ; then
if [ "$node" = "cluster" ]; then
if $tls; then
redis-server /data/conf/redis-tls.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf \
--tls-port 8000 --cluster-enabled yes ;
--tls-port 8000 --cluster-enabled yes
redis-server /data/conf/redis-tls.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf \
--tls-port 8001 --cluster-enabled yes;
--tls-port 8001 --cluster-enabled yes
redis-server /data/conf/redis-tls.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf \
--tls-port 8002 --cluster-enabled yes;
--tls-port 8002 --cluster-enabled yes
else
redis-server /data/conf/redis.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf --cluster-enabled yes;
redis-server /data/conf/redis.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf --cluster-enabled yes;
redis-server /data/conf/redis.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf --cluster-enabled yes;
redis-server /data/conf/redis.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf \
--cluster-enabled yes
redis-server /data/conf/redis.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf \
--cluster-enabled yes
redis-server /data/conf/redis.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf \
--cluster-enabled yes
fi
elif [ "${node}" = "sentinel" ] ; then
if $tls ; then
elif [ "$node" = "sentinel" ]; then
if $tls; then
redis-server /data/conf/redis-tls.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf \
--tls-port 8000 --cluster-enabled no;
--tls-port 8000 --cluster-enabled no
redis-server /data/conf/redis-tls.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf \
--tls-port 8001 --cluster-enabled no --slaveof "$LOCAL_IP" 8000;
--tls-port 8001 --cluster-enabled no --slaveof "$LOCAL_IP" 8000
redis-server /data/conf/redis-tls.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf \
--tls-port 8002 --cluster-enabled no --slaveof "$LOCAL_IP" 8000;
--tls-port 8002 --cluster-enabled no --slaveof "$LOCAL_IP" 8000

else
redis-server /data/conf/redis.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf \
--cluster-enabled no;
--cluster-enabled no
redis-server /data/conf/redis.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf \
--cluster-enabled no --slaveof "$LOCAL_IP" 7000;
--cluster-enabled no --slaveof "$LOCAL_IP" 7000
redis-server /data/conf/redis.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf \
--cluster-enabled no --slaveof "$LOCAL_IP" 7000;
--cluster-enabled no --slaveof "$LOCAL_IP" 7000
fi
fi
REDIS_LOAD_FLG=true;

REDIS_LOAD_FLG=true

while $REDIS_LOAD_FLG;
do
sleep 1;
redis-cli --pass public --no-auth-warning -p 7000 info 1> /data/conf/r7000i.log 2> /dev/null;
if [ -s /data/conf/r7000i.log ]; then
:
else
continue;
sleep 1
redis-cli --pass public --no-auth-warning -p 7000 info 1> /data/conf/r7000i.log 2> /dev/null
if ! [ -s /data/conf/r7000i.log ]; then
continue
fi
redis-cli --pass public --no-auth-warning -p 7001 info 1> /data/conf/r7001i.log 2> /dev/null;
if [ -s /data/conf/r7001i.log ]; then
:
else
continue;
redis-cli --pass public --no-auth-warning -p 7001 info 1> /data/conf/r7001i.log 2> /dev/null
if ! [ -s /data/conf/r7001i.log ]; then
continue
fi
redis-cli --pass public --no-auth-warning -p 7002 info 1> /data/conf/r7002i.log 2> /dev/null;
if [ -s /data/conf/r7002i.log ]; then
:
else
continue;
if ! [ -s /data/conf/r7002i.log ]; then
continue
fi
if [ "${node}" = "cluster" ] ; then
if $tls ; then
yes "yes" | redis-cli --cluster create "$LOCAL_IP:8000" "$LOCAL_IP:8001" "$LOCAL_IP:8002" --pass public --no-auth-warning --tls true --cacert /etc/certs/ca.crt --cert /etc/certs/redis.crt --key /etc/certs/redis.key;
if [ "$node" = "cluster" ] ; then
if $tls; then
yes "yes" | redis-cli --cluster create "$LOCAL_IP:8000" "$LOCAL_IP:8001" "$LOCAL_IP:8002" \
--pass public --no-auth-warning \
--tls true --cacert /etc/certs/ca.crt \
--cert /etc/certs/redis.crt --key /etc/certs/redis.key
else
yes "yes" | redis-cli --cluster create "$LOCAL_IP:7000" "$LOCAL_IP:7001" "$LOCAL_IP:7002" --pass public --no-auth-warning;
yes "yes" | redis-cli --cluster create "$LOCAL_IP:7000" "$LOCAL_IP:7001" "$LOCAL_IP:7002" \
--pass public --no-auth-warning
fi
elif [ "${node}" = "sentinel" ] ; then
elif [ "$node" = "sentinel" ]; then
tee /_sentinel.conf>/dev/null << EOF
port 26379
bind 0.0.0.0 ::
daemonize yes
logfile /var/log/redis-server.log
dir /tmp
EOF
if $tls ; then
if $tls; then
cat >>/_sentinel.conf<<EOF
tls-port 26380
tls-replication yes
Expand All @@ -121,9 +118,9 @@ EOF
sentinel monitor mymaster $LOCAL_IP 7000 1
EOF
fi
redis-server /_sentinel.conf --sentinel;
redis-server /_sentinel.conf --sentinel
fi
REDIS_LOAD_FLG=false;
REDIS_LOAD_FLG=false
done

exit 0;
7 changes: 7 additions & 0 deletions .ci/docker-compose-file/toxiproxy.json
View file
Expand Up @@ -22,5 +22,12 @@
"listen": "0.0.0.0:3307",
"upstream": "mysql-tls:3306",
"enabled": true
},

{
"name": "redis_single_tcp",
"listen": "0.0.0.0:6379",
"upstream": "redis:6379",
"enabled": true
}
]
12 changes: 6 additions & 6 deletions .github/workflows/build_and_push_docker_images.yaml
View file
Expand Up @@ -23,7 +23,7 @@ jobs:
prepare:
runs-on: ubuntu-20.04
# prepare source with any OTP version, no need for a matrix
container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04"
container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04"

outputs:
BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }}
Expand Down Expand Up @@ -167,14 +167,14 @@ jobs:

- uses: docker/build-push-action@v3
with:
push: ${{ needs.prepare.outputs.IS_EXACT_TAG }}
push: ${{ needs.prepare.outputs.IS_EXACT_TAG == 'true' || github.repository_owner != 'emqx' }}
pull: true
no-cache: true
platforms: linux/${{ matrix.arch[0] }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
RUN_FROM=${{ matrix.os[1] }}
EMQX_NAME=${{ steps.meta.outputs.emqx_name }}
file: source/${{ matrix.os[2] }}
Expand Down Expand Up @@ -245,14 +245,14 @@ jobs:

- uses: docker/build-push-action@v3
with:
push: ${{ needs.prepare.outputs.IS_EXACT_TAG }}
push: ${{ needs.prepare.outputs.IS_EXACT_TAG == 'true' || github.repository_owner != 'emqx' }}
pull: true
no-cache: true
platforms: linux/${{ matrix.arch[0] }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
RUN_FROM=${{ matrix.os[1] }}
EMQX_NAME=${{ steps.meta.outputs.emqx_name }}
file: source/${{ matrix.os[2] }}
Expand Down Expand Up @@ -328,7 +328,7 @@ jobs:
docker-elixir-push-multi-arch-manifest:
# note, we only run on amd64
# do not build enterprise elixir images for now
if: needs.prepare.outputs.IS_EXACT_TAG && needs.prepare.outputs.BUILD_PROFILE == 'emqx'
if: needs.prepare.outputs.IS_EXACT_TAG == 'true' && needs.prepare.outputs.BUILD_PROFILE == 'emqx'
needs:
- prepare
- docker-elixir
Expand Down
42 changes: 23 additions & 19 deletions .github/workflows/build_packages.yaml
View file
Expand Up @@ -23,7 +23,7 @@ on:
jobs:
prepare:
runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04
outputs:
BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }}
IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }}
Expand Down Expand Up @@ -173,7 +173,7 @@ jobs:
needs: prepare
runs-on: ${{ matrix.build_machine }}
container:
image: "ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
image: "ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"

strategy:
fail-fast: false
Expand Down Expand Up @@ -203,27 +203,31 @@ jobs:
- debian10
- el8
- el7
- amzn2
build_machine:
- aws-arm64
- ubuntu-20.04
exclude:
- arch: arm64
- arch: arm64
build_machine: ubuntu-20.04
- arch: amd64
build_machine: aws-arm64
include:
- profile: emqx
otp: 24.3.4.2-1
elixir: 1.13.4
build_elixir: with_elixir
arch: amd64
os: ubuntu20.04
build_machine: ubuntu-20.04
- arch: amd64
build_machine: aws-arm64
# elixir: only for opensource edition and only on ubuntu20.04 and el8 on amd64
- build_elixir: with_elixir
profile: emqx-enterprise
- build_elixir: with_elixir
arch: arm64
- build_elixir: with_elixir
os: ubuntu18.04
- build_elixir: with_elixir
os: debian10
- build_elixir: with_elixir
os: debian11
- build_elixir: with_elixir
os: el7
- profile: emqx
otp: 24.3.4.2-1
elixir: 1.13.4
build_elixir: with_elixir
arch: amd64
os: amzn2
build_machine: ubuntu-20.04

defaults:
run:
shell: bash
Expand Down Expand Up @@ -268,7 +272,7 @@ jobs:
--pkgtype "${PKGTYPE}" \
--arch "${ARCH}" \
--elixir "${IsElixir}" \
--builder "ghcr.io/emqx/emqx-builder/5.0-18:${ELIXIR}-${OTP}-${SYSTEM}"
--builder "ghcr.io/emqx/emqx-builder/5.0-24:${ELIXIR}-${OTP}-${SYSTEM}"
done
- uses: actions/upload-artifact@v3
with:
Expand Down

0 comments on commit 42c58e2

Please sign in to comment.