From 9090b3482856224a549d8182b1a20d4635e84eef Mon Sep 17 00:00:00 2001 From: chengshiwen Date: Tue, 20 Jul 2021 14:11:53 +0800 Subject: [PATCH] [Improvement-5858][Docker] Improve docker image support multi-arch like arm64 in docker-compose --- .github/workflows/ci_e2e.yml | 2 +- .github/workflows/ci_ut.yml | 6 +++--- docker/build/Dockerfile | 2 +- docker/build/hooks/build | 16 +++++++++++++++- docker/docker-swarm/docker-compose.yml | 16 ++++++++-------- docker/docker-swarm/docker-stack.yml | 16 ++++++++-------- .../dolphinscheduler/templates/ingress.yaml | 8 ++++---- 7 files changed, 40 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 009b3fb15199..2fd683c166ad 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -45,7 +45,7 @@ jobs: sh ./docker/build/hooks/build - name: Docker Run run: | - export VERSION=$(cat $(pwd)/pom.xml | grep '' -m 1 | awk '{print $1}' | sed 's///' | sed 's/<\/revision>//') + export VERSION=$(cat $(pwd)/pom.xml | grep '' -m 1 | awk '{print $1}' | sed 's///' | sed 's/<\/version>//') sed -i "s/apache\/dolphinscheduler:latest/apache\/dolphinscheduler:${VERSION}/g" $(pwd)/docker/docker-swarm/docker-compose.yml docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d - name: Check Server Status diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index f7f6e1cf573b..0d2aa8941401 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -47,11 +47,11 @@ jobs: ${{ runner.os }}-maven- - name: Bootstrap database run: | - sed -i "/image: bitnami\/postgresql/a\ ports:\n - 5432:5432" $(pwd)/docker/docker-swarm/docker-compose.yml - sed -i "/image: bitnami\/zookeeper/a\ ports:\n - 2181:2181" $(pwd)/docker/docker-swarm/docker-compose.yml + sed -i "/image: postgres/a\ ports:\n - 5432:5432" $(pwd)/docker/docker-swarm/docker-compose.yml + sed -i "/image: zookeeper/a\ ports:\n - 2181:2181" $(pwd)/docker/docker-swarm/docker-compose.yml docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d dolphinscheduler-zookeeper dolphinscheduler-postgresql until docker logs docker-swarm_dolphinscheduler-postgresql_1 2>&1 | grep 'listening on IPv4 address'; do echo "waiting for postgresql ready ..."; sleep 1; done - docker run --rm --network docker-swarm_dolphinscheduler -v $(pwd)/sql/dolphinscheduler_postgre.sql:/docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql bitnami/postgresql:latest bash -c "PGPASSWORD=root psql -h docker-swarm_dolphinscheduler-postgresql_1 -U root -d dolphinscheduler -v ON_ERROR_STOP=1 -f /docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql" + docker run --rm --network docker-swarm_dolphinscheduler -v $(pwd)/sql/dolphinscheduler_postgre.sql:/docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql postgres:11.12 bash -c "PGPASSWORD=root psql -h docker-swarm_dolphinscheduler-postgresql_1 -U root -d dolphinscheduler -v ON_ERROR_STOP=1 -f /docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql" - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index 65ab79cbe32b..cb853847a765 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -62,7 +62,7 @@ RUN sed -i 's/*.conf$/*.ini/' /etc/supervisor/supervisord.conf && \ rm -f /bin/sh && \ ln -s /bin/bash /bin/sh && \ mkdir -p /tmp/xls && \ - echo PS1=\'\\w \\$ \' >> ~/.bashrc && \ + echo PS1=\"\\w \\$ \" >> ~/.bashrc && \ echo "Set disable_coredump false" >> /etc/sudo.conf # 4. expose port diff --git a/docker/build/hooks/build b/docker/build/hooks/build index 0590761eeb91..ef89aaeabadc 100755 --- a/docker/build/hooks/build +++ b/docker/build/hooks/build @@ -47,7 +47,21 @@ echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-${VERSIO mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-${VERSION}-bin.tar.gz $(pwd)/docker/build/ # docker build -BUILD_COMMAND="docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/" +if [ "$1" = "x" ] +then + # build multi-arch images with buildx + if !(docker buildx use dolphinscheduler &> /dev/null) + then + docker buildx create --use --name dolphinscheduler + fi + docker buildx inspect --bootstrap + echo "" + BUILD_PLATFORM=linux/amd64,linux/arm64 + BUILD_COMMAND="docker buildx build --platform $BUILD_PLATFORM --push --build-arg VERSION=$VERSION -t $DOCKER_REPO:$VERSION $(pwd)/docker/build/" +else + BUILD_COMMAND="docker build --build-arg VERSION=$VERSION -t $DOCKER_REPO:$VERSION $(pwd)/docker/build/" +fi + echo -e "$BUILD_COMMAND\n" if (docker info 2> /dev/null | grep -i "ERROR"); then sudo $BUILD_COMMAND diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index faa7abb65ae9..aea9ebbe0eab 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -19,26 +19,26 @@ version: "3.1" services: dolphinscheduler-postgresql: - image: bitnami/postgresql:11.11.0 + image: postgres:11.12 environment: TZ: Asia/Shanghai - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + POSTGRES_USER: root + POSTGRES_PASSWORD: root + POSTGRES_DB: dolphinscheduler volumes: - - dolphinscheduler-postgresql:/bitnami/postgresql + - dolphinscheduler-postgresql:/var/lib/postgresql/data restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-zookeeper: - image: bitnami/zookeeper:3.6.2 + image: zookeeper:3.6.3 environment: TZ: Asia/Shanghai - ALLOW_ANONYMOUS_LOGIN: "yes" + ZOO_DATA_LOG_DIR: /data ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons volumes: - - dolphinscheduler-zookeeper:/bitnami/zookeeper + - dolphinscheduler-zookeeper:/data restart: unless-stopped networks: - dolphinscheduler diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index 52617ca655dc..1999abd491dd 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -19,14 +19,14 @@ version: "3.1" services: dolphinscheduler-postgresql: - image: bitnami/postgresql:11.11.0 + image: postgres:11.12 environment: TZ: Asia/Shanghai - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + POSTGRES_USER: root + POSTGRES_PASSWORD: root + POSTGRES_DB: dolphinscheduler volumes: - - dolphinscheduler-postgresql:/bitnami/postgresql + - dolphinscheduler-postgresql:/var/lib/postgresql/data networks: - dolphinscheduler deploy: @@ -34,13 +34,13 @@ services: replicas: 1 dolphinscheduler-zookeeper: - image: bitnami/zookeeper:3.6.2 + image: zookeeper:3.6.3 environment: TZ: Asia/Shanghai - ALLOW_ANONYMOUS_LOGIN: "yes" + ZOO_DATA_LOG_DIR: /data ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons volumes: - - dolphinscheduler-zookeeper:/bitnami/zookeeper + - dolphinscheduler-zookeeper:/data networks: - dolphinscheduler deploy: diff --git a/docker/kubernetes/dolphinscheduler/templates/ingress.yaml b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml index 0fa1a8b5c932..cc4b554da56f 100644 --- a/docker/kubernetes/dolphinscheduler/templates/ingress.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml @@ -49,8 +49,8 @@ spec: {{- end }} {{- if .Values.ingress.tls.enabled }} tls: - - hosts: - - {{ .Values.ingress.host }} - secretName: {{ .Values.ingress.tls.secretName }} + - hosts: + - {{ .Values.ingress.host }} + secretName: {{ .Values.ingress.tls.secretName }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file