From 29a353e689730cac822165979ce17c5e3ac29835 Mon Sep 17 00:00:00 2001 From: liwenhe1993 Date: Tue, 12 May 2020 20:24:17 +0800 Subject: [PATCH] Using docker/docker-swarm/docker-compose.yml to run pg and zk in ci_ut.yml (#2692) * Using docker/docker-swarm/docker-compose.yml to run pg and zk in ci_ut.yml 1. add initdb volume for pg service in docker-compose.yml 2. modify ci_ut.yml * add sudo in ci_ut.yml * Query the metadata of indexes is not correct in postgresql/dolphinscheduler_ddl.sql 1. replace information_schema.STATISTICS to pg_stat_all_indexes 2. modify ProcessDefinitionMapperTest.java * Remove inner zk and pg * add inner pg --- .github/workflows/ci_ut.yml | 10 ++-- docker/docker-swarm/docker-compose.yml | 6 ++- dockerfile/Dockerfile | 33 ++++-------- dockerfile/startup.sh | 53 ++++--------------- .../mapper/ProcessDefinitionMapperTest.java | 18 ++++++- .../postgresql/dolphinscheduler_ddl.sql | 6 +-- 6 files changed, 51 insertions(+), 75 deletions(-) diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 6d0b2a2a1d5c..55f1259d2b9f 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -21,7 +21,6 @@ on: branches: - dev env: - DOCKER_DIR: ./docker LOG_DIR: /tmp/dolphinscheduler name: Unit Test @@ -47,7 +46,11 @@ jobs: restore-keys: | ${{ runner.os }}-maven- - name: Bootstrap database - run: cd ${DOCKER_DIR} && docker-compose up -d + run: | + sed -i "s/: root/: test/g" $(pwd)/docker/docker-swarm/docker-compose.yml + docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml create --force-recreate dolphinscheduler-zookeeper dolphinscheduler-postgresql + sudo cp $(pwd)/sql/dolphinscheduler-postgre.sql $(docker volume inspect docker-swarm_dolphinscheduler-postgresql-initdb | grep "Mountpoint" | awk -F "\"" '{print $4}') + docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d dolphinscheduler-zookeeper dolphinscheduler-postgresql - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: @@ -82,6 +85,5 @@ jobs: - name: Collect logs run: | mkdir -p ${LOG_DIR} - cd ${DOCKER_DIR} - docker-compose logs db > ${LOG_DIR}/db.txt + docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml logs dolphinscheduler-postgresql > ${LOG_DIR}/db.txt continue-on-error: true diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index b564eda842fc..ee8be2570d92 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -30,6 +30,7 @@ services: POSTGRESQL_DATABASE: dolphinscheduler volumes: - dolphinscheduler-postgresql:/bitnami/postgresql + - dolphinscheduler-postgresql-initdb:/docker-entrypoint-initdb.d networks: - dolphinscheduler @@ -175,10 +176,10 @@ services: image: apache/dolphinscheduler:latest container_name: dolphinscheduler-worker command: ["worker-server"] - ports: + ports: - 1234:1234 - 50051:50051 - environment: + environment: TZ: Asia/Shanghai WORKER_EXEC_THREADS: "100" WORKER_HEARTBEAT_INTERVAL: "10" @@ -221,6 +222,7 @@ networks: volumes: dolphinscheduler-postgresql: + dolphinscheduler-postgresql-initdb: dolphinscheduler-zookeeper: dolphinscheduler-worker-data: dolphinscheduler-logs: diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile index 53e362b6318b..101e0b22196e 100644 --- a/dockerfile/Dockerfile +++ b/dockerfile/Dockerfile @@ -37,32 +37,20 @@ RUN apk add openjdk8 ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk ENV PATH $JAVA_HOME/bin:$PATH -#3. install zk -RUN cd /opt && \ - wget https://downloads.apache.org/zookeeper/zookeeper-3.5.7/apache-zookeeper-3.5.7-bin.tar.gz && \ - tar -zxvf apache-zookeeper-3.5.7-bin.tar.gz && \ - mv apache-zookeeper-3.5.7-bin zookeeper && \ - mkdir -p /tmp/zookeeper && \ - rm -rf ./zookeeper-*tar.gz && \ - rm -rf /opt/zookeeper/conf/zoo_sample.cfg -ADD ./conf/zookeeper/zoo.cfg /opt/zookeeper/conf -ENV ZK_HOME /opt/zookeeper -ENV PATH $ZK_HOME/bin:$PATH - -#4. install pg -RUN apk add postgresql postgresql-contrib - -#5. add dolphinscheduler +#3. add dolphinscheduler ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/ RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/ ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler -#6. modify nginx +#4. install pg +RUN apk add postgresql postgresql-contrib + +#5. modify nginx RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \ rm -rf /etc/nginx/conf.d/* ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d -#7. add configuration and modify permissions and set soft links +#6. add configuration and modify permissions and set soft links ADD ./checkpoint.sh /root/checkpoint.sh ADD ./startup-init-conf.sh /root/startup-init-conf.sh ADD ./startup.sh /root/startup.sh @@ -75,22 +63,21 @@ RUN chmod +x /root/checkpoint.sh && \ chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ chmod +x /opt/dolphinscheduler/script/*.sh && \ chmod +x /opt/dolphinscheduler/bin/*.sh && \ - chmod +x /opt/zookeeper/bin/*.sh && \ dos2unix /root/checkpoint.sh && \ dos2unix /root/startup-init-conf.sh && \ dos2unix /root/startup.sh && \ dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ dos2unix /opt/dolphinscheduler/script/*.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \ - dos2unix /opt/zookeeper/bin/*.sh && \ rm -rf /bin/sh && \ ln -s /bin/bash /bin/sh && \ mkdir -p /tmp/xls -#8. remove apk index cache -RUN rm -rf /var/cache/apk/* +#7. remove apk index cache and disable coredup for sudo +RUN rm -rf /var/cache/apk/* && \ + echo "Set disable_coredump false" >> /etc/sudo.conf -#9. expose port +#8. expose port EXPOSE 2181 2888 3888 5432 5678 1234 12345 50051 8888 ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"] \ No newline at end of file diff --git a/dockerfile/startup.sh b/dockerfile/startup.sh index 71ab50677763..00bb9cdc52aa 100644 --- a/dockerfile/startup.sh +++ b/dockerfile/startup.sh @@ -24,31 +24,6 @@ DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs # start postgresql initPostgreSQL() { - echo "checking postgresql" - if [[ "${POSTGRESQL_HOST}" = "127.0.0.1" || "${POSTGRESQL_HOST}" = "localhost" ]]; then - export PGPORT=${POSTGRESQL_PORT} - - echo "start postgresql service" - rc-service postgresql restart - - # role if not exists, create - flag=$(sudo -u postgres psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='${POSTGRESQL_USERNAME}'") - if [ -z "${flag}" ]; then - echo "create user" - sudo -u postgres psql -tAc "create user ${POSTGRESQL_USERNAME} with password '${POSTGRESQL_PASSWORD}'" - fi - - # database if not exists, create - flag=$(sudo -u postgres psql -tAc "select 1 from pg_database where datname='dolphinscheduler'") - if [ -z "${flag}" ]; then - echo "init db" - sudo -u postgres psql -tAc "create database dolphinscheduler owner ${POSTGRESQL_USERNAME}" - fi - - # grant - sudo -u postgres psql -tAc "grant all privileges on database dolphinscheduler to ${POSTGRESQL_USERNAME}" - fi - echo "test postgresql service" while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do counter=$((counter+1)) @@ -73,24 +48,18 @@ initPostgreSQL() { # start zk initZK() { - echo -e "checking zookeeper" - if [[ "${ZOOKEEPER_QUORUM}" = "127.0.0.1:2181" || "${ZOOKEEPER_QUORUM}" = "localhost:2181" ]]; then - echo "start local zookeeper" - /opt/zookeeper/bin/zkServer.sh restart - else - echo "connect remote zookeeper" - echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do - while ! nc -z ${line%:*} ${line#*:}; do - counter=$((counter+1)) - if [ $counter == 30 ]; then - echo "Error: Couldn't connect to zookeeper." - exit 1 - fi - echo "Trying to connect to zookeeper at ${line}. Attempt $counter." - sleep 5 - done + echo "connect remote zookeeper" + echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do + while ! nc -z ${line%:*} ${line#*:}; do + counter=$((counter+1)) + if [ $counter == 30 ]; then + echo "Error: Couldn't connect to zookeeper." + exit 1 + fi + echo "Trying to connect to zookeeper at ${line}. Attempt $counter." + sleep 5 done - fi + done } # start nginx diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java index 9dafbe138c8f..e92ec5454089 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java @@ -73,6 +73,22 @@ private ProcessDefinition insertOne(){ return processDefinition; } + /** + * insert + * @return ProcessDefinition + */ + private ProcessDefinition insertTwo(){ + //insertOne + ProcessDefinition processDefinition = new ProcessDefinition(); + processDefinition.setName("def 2"); + processDefinition.setProjectId(1010); + processDefinition.setUserId(101); + processDefinition.setUpdateTime(new Date()); + processDefinition.setCreateTime(new Date()); + processDefinitionMapper.insert(processDefinition); + return processDefinition; + } + /** * test update */ @@ -177,7 +193,7 @@ public void testQueryAllDefinitionList() { public void testQueryDefinitionListByIdList() { ProcessDefinition processDefinition = insertOne(); - ProcessDefinition processDefinition1 = insertOne(); + ProcessDefinition processDefinition1 = insertTwo(); Integer[] array = new Integer[2]; array[0] = processDefinition.getId(); diff --git a/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql index 61cbfed4fe4b..16a49a897ee1 100644 --- a/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql @@ -368,9 +368,9 @@ DROP FUNCTION dc_dolphin_T_t_ds_error_command_D_worker_group_id(); delimiter d// CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_process_definition_A_process_definition_unique() RETURNS void AS $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM information_schema.STATISTICS - WHERE TABLE_NAME='t_ds_process_definition' - AND INDEX_NAME ='process_definition_unique') + IF NOT EXISTS (SELECT 1 FROM pg_stat_all_indexes + WHERE relname='t_ds_process_definition' + AND indexrelname ='process_definition_unique') THEN ALTER TABLE t_ds_process_definition ADD CONSTRAINT process_definition_unique UNIQUE (name,project_id); END IF;