optimized cicd

Change-Id: I1f40391b790a329c470d8de39278f834c132fc95
Signed-off-by: Ziming Zhang <zziming@vmware.com>

.travis.yml

@@ -34,7 +34,7 @@ env:
   - CORE_SECRET: tempString
   - KEY_PATH: "/data/secret/keys/secretkey"
   - REDIS_HOST: localhost
-  - REG_VERSION: v2.7.1
+  - REG_VERSION: v2.7.1-patch-2819
   - UI_BUILDER_VERSION: 1.6.0
   - TOKEN_PRIVATE_KEY_PATH: "/home/travis/gopath/src/github.com/goharbor/harbor/tests/private_key.pem"
 addons: 
@@ -52,15 +52,17 @@ before_install:
 - IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'`
 - sudo sed -i '$a DOCKER_OPTS=\"--insecure-registry '$IP':5000\"' /etc/default/docker
 - export IP=$IP
+- sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
+- sudo update-ca-certificates
 - sudo service docker restart
 install:
-- if [ "$UTTEST" == true ]; then bash ./tests/travis/ut_install.sh ; fi
-- if [ "$APITEST_DB" == true ]; then bash ./tests/travis/api_common_install.sh $IP
+- if [ "$UTTEST" == true ]; then bash ./showtime.sh ./tests/travis/ut_install.sh ; fi
+- if [ "$APITEST_DB" == true ]; then bash ./showtime.sh ./tests/travis/api_common_install.sh $IP
   DB; fi
-- if [ "$APITEST_LDAP" == true ]; then bash ./tests/travis/api_common_install.sh $IP
+- if [ "$APITEST_LDAP" == true ]; then bash ./showtime.sh ./tests/travis/api_common_install.sh $IP
   LDAP; fi
 script:
-- if [ "$UTTEST" == true ]; then bash ./tests/travis/ut_run.sh $IP; fi
-- if [ "$APITEST_DB" == true ]; then bash ./tests/travis/api_run.sh DB $IP; fi
-- if [ "$APITEST_LDAP" == true ]; then bash ./tests/travis/api_run.sh LDAP $IP; fi
-- if [ "$OFFLINE" == true ]; then bash ./tests/travis/distro_installer.sh; fi
+- if [ "$UTTEST" == true ]; then bash ./showtime.sh ./tests/travis/ut_run.sh $IP; fi
+- if [ "$APITEST_DB" == true ]; then bash ./showtime.sh ./tests/travis/api_run.sh DB $IP; fi
+- if [ "$APITEST_LDAP" == true ]; then bash ./showtime.sh ./tests/travis/api_run.sh LDAP $IP; fi
+- if [ "$OFFLINE" == true ]; then bash ./showtime.sh ./tests/travis/distro_installer.sh; fi

build_base_docker.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+DOCKERCMD=docker
+CICDHOST=cicd.harbor.bitsf.xin
+DOCKERFILEPATH=make/photon
+
+${DOCKERCMD} login ${CICDHOST} || exit 2
+for name in chartserver clair core db jobservice log nginx portal prepare redis registry registryctl; do
+	echo building $name base docker image
+	$DOCKERCMD build -f $DOCKERFILEPATH/$name/Dockerfile-base -t $CICDHOST/harbor-depend/$name:base . && \
+	$DOCKERCMD push $CICDHOST/harbor-depend/$name:base
+	if [ "$?" != "0" ]; then exit 1; fi
+done

make/photon/Makefile

@@ -97,6 +97,13 @@ CHART_SERVER_CODE_BASE=github.com/helm/chartmuseum
 CHART_SERVER_MAIN_PATH=cmd/chartmuseum
 CHART_SERVER_BIN_NAME=chartm
 
+build_base_docker:
+	for name in chartserver clair common core db jobservice log nginx notary portal prepare redis registry registryctl; do \
+		echo $(name) ; \
+		$(DOCKERBUILD) -f $(DOCKERFILEPATH)/$(name) -t cicd.harbor.bitsf.xin/harbor-depend/$(name):base . ; \
+		$(DOCKERBUILD) push cicd.harbor.bitsf.xin/harbor-depend/$(name):base ; \
+	done
+
 _build_prepare:
 	@echo "building prepare container for photon..."
 	@$(DOCKERBUILD) -f $(DOCKERFILEPATH_PREPARE)/$(DOCKERFILENAME_PREPARE) -t $(DOCKERIMAGENAME_PREPARE):$(VERSIONTAG) .

make/photon/chartserver/Dockerfile

@@ -1,11 +1,5 @@
-FROM photon:2.0
+FROM cicd.harbor.bitsf.xin/harbor-depend/chartserver:base
 
-RUN tdnf install -y shadow sudo >>/dev/null\
-    && tdnf clean all \
-    && mkdir /chartserver/ \
-    && mkdir /harbor/ \
-    && groupadd -r -g 10000 chartuser \
-    && useradd --no-log-init -m -r -g 10000 -u 10000 chartuser
 COPY ./make/photon/chartserver/binary/chartm /chartserver/
 COPY ./make/photon/chartserver/docker-entrypoint.sh /docker-entrypoint.sh
 COPY ./make/photon/common/install_cert.sh /harbor

make/photon/chartserver/Dockerfile-base

@@ -0,0 +1,8 @@
+FROM photon:2.0
+
+RUN tdnf install -y shadow sudo >>/dev/null\
+    && tdnf clean all \
+    && mkdir /chartserver/ \
+    && mkdir /harbor/ \
+    && groupadd -r -g 10000 chartuser \
+    && useradd --no-log-init -m -r -g 10000 -u 10000 chartuser

make/photon/clair/Dockerfile

@@ -1,11 +1,5 @@
-FROM photon:2.0
+FROM cicd.harbor.bitsf.xin/harbor-depend/clair:base
 
-RUN tdnf install -y git shadow sudo rpm xz python-xml >>/dev/null\
-    && tdnf clean all \
-    && mkdir /clair/ \
-    && mkdir /harbor \
-    && groupadd -r -g 10000 clair \
-    && useradd --no-log-init -m -r -g 10000 -u 10000 clair
 COPY ./make/photon/clair/binary/clair /clair/
 COPY ./make/photon/clair/docker-entrypoint.sh /docker-entrypoint.sh
 COPY ./make/photon/clair/dumb-init /dumb-init

make/photon/clair/Dockerfile-base

@@ -0,0 +1,8 @@
+FROM photon:2.0
+
+RUN tdnf install -y git shadow sudo rpm xz python-xml >>/dev/null\
+    && tdnf clean all \
+    && mkdir /clair/ \
+    && mkdir /harbor \
+    && groupadd -r -g 10000 clair \
+    && useradd --no-log-init -m -r -g 10000 -u 10000 clair

make/photon/core/Dockerfile

@@ -1,9 +1,4 @@
-FROM photon:2.0
-
-RUN tdnf install sudo tzdata -y >> /dev/null \
-    && tdnf clean all \
-    && groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor \
-    && mkdir /harbor/
+FROM cicd.harbor.bitsf.xin/harbor-depend/core:base
 
 HEALTHCHECK CMD curl --fail -s http://127.0.0.1:8080/api/ping || exit 1
 COPY ./make/photon/core/harbor_core ./UIVERSION /harbor/

make/photon/core/Dockerfile-base

@@ -0,0 +1,6 @@
+FROM photon:2.0
+
+RUN tdnf install sudo tzdata -y >> /dev/null\
+    && tdnf clean all \
+    && groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor \
+    && mkdir /harbor/

make/photon/db/Dockerfile

@@ -1,22 +1,4 @@
-FROM photon:2.0
-
-ENV PGDATA /var/lib/postgresql/data
-
-RUN tdnf install -y shadow gzip postgresql >> /dev/null\
-    && groupadd -r postgres --gid=999 \
-    && useradd -m -r -g postgres --uid=999 postgres \
-    && mkdir -p /docker-entrypoint-initdb.d \
-    && mkdir -p /run/postgresql \
-    && chown -R postgres:postgres /run/postgresql \
-    && chmod 2777 /run/postgresql \
-    && mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA" \
-    && sed -i "s|#listen_addresses = 'localhost'.*|listen_addresses = '*'|g" /usr/share/postgresql/postgresql.conf.sample \
-    && sed -i "s|#unix_socket_directories = '/tmp'.*|unix_socket_directories = '/run/postgresql'|g" /usr/share/postgresql/postgresql.conf.sample \
-    && tdnf clean all
-
-RUN tdnf erase -y toybox && tdnf install -y util-linux net-tools
-
-VOLUME /var/lib/postgresql/data
+FROM cicd.harbor.bitsf.xin/harbor-depend/db:base
 
 COPY ./make/photon/db/docker-entrypoint.sh /docker-entrypoint.sh
 COPY ./make/photon/db/docker-healthcheck.sh /docker-healthcheck.sh

make/photon/db/Dockerfile-base

@@ -0,0 +1,19 @@
+FROM photon:2.0
+
+ENV PGDATA /var/lib/postgresql/data
+
+RUN tdnf install -y shadow gzip postgresql >> /dev/null\
+    && groupadd -r postgres --gid=999 \
+    && useradd -m -r -g postgres --uid=999 postgres \
+    && mkdir -p /docker-entrypoint-initdb.d \
+    && mkdir -p /run/postgresql \
+    && chown -R postgres:postgres /run/postgresql \
+    && chmod 2777 /run/postgresql \
+    && mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA" \
+    && sed -i "s|#listen_addresses = 'localhost'.*|listen_addresses = '*'|g" /usr/share/postgresql/postgresql.conf.sample \
+    && sed -i "s|#unix_socket_directories = '/tmp'.*|unix_socket_directories = '/run/postgresql'|g" /usr/share/postgresql/postgresql.conf.sample \
+    && tdnf clean all
+
+RUN tdnf erase -y toybox && tdnf install -y util-linux net-tools
+
+VOLUME /var/lib/postgresql/data

make/photon/jobservice/Dockerfile

@@ -1,8 +1,4 @@
-FROM photon:2.0
-
-RUN tdnf install sudo tzdata -y >> /dev/null \
-    && tdnf clean all \
-    && groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor
+FROM cicd.harbor.bitsf.xin/harbor-depend/jobservice:base
 
 COPY ./make/photon/jobservice/harbor_jobservice /harbor/
 

make/photon/jobservice/Dockerfile-base

@@ -0,0 +1,5 @@
+FROM photon:2.0
+
+RUN tdnf install sudo tzdata -y >> /dev/null\
+    && tdnf clean all \
+    && groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor

make/photon/log/Dockerfile

@@ -1,9 +1,4 @@
-FROM photon:2.0
-
-RUN tdnf install -y cronie rsyslog logrotate shadow tar gzip sudo >> /dev/null\
-    && mkdir /var/spool/rsyslog \
-    && groupadd -r -g 10000 syslog && useradd --no-log-init -r -g 10000 -u 10000 syslog \
-    && tdnf clean all
+FROM cicd.harbor.bitsf.xin/harbor-depend/log:base
 
 COPY ./make/photon/log/rsyslog.conf /etc/rsyslog.conf
 

make/photon/log/Dockerfile-base

@@ -0,0 +1,6 @@
+FROM photon:2.0
+
+RUN tdnf install -y cronie rsyslog logrotate shadow tar gzip sudo >> /dev/null\
+    && mkdir /var/spool/rsyslog \
+    && groupadd -r -g 10000 syslog && useradd --no-log-init -r -g 10000 -u 10000 syslog \
+    && tdnf clean all

make/photon/nginx/Dockerfile

@@ -1,12 +1,4 @@
-FROM photon:2.0
-
-RUN tdnf install sudo nginx -y >> /dev/null\
-    && tdnf clean all \
-    && groupadd -r -g 10000 nginx && useradd --no-log-init -r -g 10000 -u 10000 nginx \
-    && ln -sf /dev/stdout /var/log/nginx/access.log \
-    && ln -sf /dev/stderr /var/log/nginx/error.log
-
-VOLUME /var/cache/nginx /var/log/nginx /run
+FROM cicd.harbor.bitsf.xin/harbor-depend/nginx:base
 
 EXPOSE 8080
 

make/photon/nginx/Dockerfile-base

@@ -0,0 +1,9 @@
+FROM photon:2.0
+
+RUN tdnf install sudo nginx -y >> /dev/null\
+    && tdnf clean all \
+    && groupadd -r -g 10000 nginx && useradd --no-log-init -r -g 10000 -u 10000 nginx \
+    && ln -sf /dev/stdout /var/log/nginx/access.log \
+    && ln -sf /dev/stderr /var/log/nginx/error.log
+
+VOLUME /var/cache/nginx /var/log/nginx /run

make/photon/portal/Dockerfile

@@ -17,22 +17,15 @@ RUN cp -r /portal_src/* /build_dir \
     && npm run release
 
 
-FROM photon:2.0
+FROM cicd.harbor.bitsf.xin/harbor-depend/portal:base
+
+COPY make/photon/portal/nginx.conf /etc/nginx/nginx.conf
 
 COPY --from=nodeportal /build_dir/dist /usr/share/nginx/html
 COPY --from=nodeportal /build_dir/swagger.yaml /usr/share/nginx/html
 COPY --from=nodeportal /build_dir/swagger.json /usr/share/nginx/html
 COPY --from=nodeportal /build_dir/LICENSE /usr/share/nginx/html
 
-COPY make/photon/portal/nginx.conf /etc/nginx/nginx.conf
-
-RUN tdnf install -y nginx sudo >> /dev/null \
-    && ln -sf /dev/stdout /var/log/nginx/access.log \
-    && ln -sf /dev/stderr /var/log/nginx/error.log \
-    && groupadd -r -g 10000 nginx && useradd --no-log-init -r -g 10000 -u 10000 nginx \
-    && chown -R nginx:nginx /etc/nginx \
-    && tdnf clean all
-
 EXPOSE 8080
 VOLUME /var/cache/nginx /var/log/nginx /run
 

make/photon/portal/Dockerfile-base

@@ -0,0 +1,8 @@
+FROM photon:2.0
+
+RUN tdnf install -y nginx sudo >> /dev/null \
+    && ln -sf /dev/stdout /var/log/nginx/access.log \
+    && ln -sf /dev/stderr /var/log/nginx/error.log \
+    && groupadd -r -g 10000 nginx && useradd --no-log-init -r -g 10000 -u 10000 nginx \
+    && chown -R nginx:nginx /etc/nginx \
+    && tdnf clean all

make/photon/prepare/Dockerfile

@@ -1,18 +1,16 @@
-FROM photon:2.0
+FROM cicd.harbor.bitsf.xin/harbor-depend/prepare:base
 
 ENV LANG en_US.UTF-8
 
 WORKDIR /usr/src/app
 
 RUN mkdir -p /harbor_make
 
-RUN tdnf install -y python3 \
-    && tdnf install -y python3-pip
-RUN pip3 install pipenv==2018.11.26
-
 COPY make/photon/prepare /usr/src/app
 RUN set -ex && pipenv install --deploy --system
 
 ENTRYPOINT [ "python3", "main.py" ]
 
-VOLUME ["/harbor_make"]
+VOLUME ["/harbor_make"]
+
+COPY make/harbor.yml /input/harbor.yml

make/photon/prepare/Dockerfile-base

@@ -0,0 +1,8 @@
+FROM photon:2.0
+
+ENV LANG en_US.UTF-8
+
+RUN tdnf install -y python3 \
+    && tdnf install -y python3-pip
+RUN pip3 install pipenv==2018.11.26
+

make/photon/prepare/templates/nginx/nginx.https.conf.jinja

@@ -150,6 +150,6 @@ http {
   server {
       listen 8080;
       #server_name harbordomain.com;
-      return 308 https://$host$request_uri;
+      return 308 https://{{https_redirect}}$request_uri;
   } 
 }

make/photon/prepare/utils/docker_compose.py

@@ -6,6 +6,7 @@
 
 docker_compose_template_path = os.path.join(templates_dir, 'docker_compose', 'docker-compose.yml.jinja')
 docker_compose_yml_path = '/compose_location/docker-compose.yml'
+os.makedirs('/compose_location',exist_ok=True)
 
 # render docker-compose
 def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum):

make/photon/prepare/utils/nginx.py

@@ -59,6 +59,7 @@ def render_nginx_template(config_dict):
             nginx_conf,
             uid=DEFAULT_UID,
             gid=DEFAULT_GID,
+            https_redirect='$host' + ('https_port' in config_dict and (":" + str(config_dict['https_port'])) or ""),
             ssl_cert=SSL_CERT_PATH,
             ssl_cert_key=SSL_CERT_KEY_PATH)
         location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTPS

make/photon/redis/Dockerfile

@@ -1,6 +1,4 @@
-FROM photon:2.0
-
-RUN tdnf install -y redis sudo 
+FROM cicd.harbor.bitsf.xin/harbor-depend/redis:base
 
 VOLUME /var/lib/redis
 WORKDIR /var/lib/redis

make/photon/redis/Dockerfile-base

@@ -0,0 +1,3 @@
+FROM photon:2.0
+
+RUN tdnf install -y redis sudo

make/photon/registry/Dockerfile

@@ -1,13 +1,4 @@
-FROM photon:2.0
-
-MAINTAINER wangyan@vmware.com
-
-# The original script in the docker offical registry image.
-RUN tdnf install sudo -y >> /dev/null\
-    && tdnf clean all \
-    && mkdir /harbor/ \
-    && mkdir -p /etc/registry \
-    && groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor  
+FROM cicd.harbor.bitsf.xin/harbor-depend/registry:base
 
 COPY ./make/photon/common/install_cert.sh /harbor
 COPY ./make/photon/registry/entrypoint.sh /

make/photon/registry/Dockerfile-base

@@ -0,0 +1,8 @@
+FROM photon:2.0
+
+# The original script in the docker offical registry image.
+RUN tdnf install sudo -y >> /dev/null\
+    && tdnf clean all \
+    && mkdir /harbor/ \
+    && mkdir -p /etc/registry \
+    && groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor

make/photon/registryctl/Dockerfile

@@ -1,12 +1,4 @@
-FROM photon:2.0
-
-MAINTAINER wangyan@vmware.com
-
-RUN tdnf install sudo -y >> /dev/null \
-    && tdnf clean all \
-    && groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor \
-    && mkdir -p /etc/registry \
-    && mkdir /harbor/
+FROM cicd.harbor.bitsf.xin/harbor-depend/registryctl:base
 
 COPY ./make/photon/common/install_cert.sh /harbor
 COPY ./make/photon/registry/binary/registry /usr/bin

make/photon/registryctl/Dockerfile-base

@@ -0,0 +1,7 @@
+FROM photon:2.0
+
+RUN tdnf install sudo -y >> /dev/null \
+    && tdnf clean all \
+    && groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor \
+    && mkdir -p /etc/registry \
+    && mkdir /harbor/

showtime.sh

@@ -0,0 +1,13 @@
+#/bin/env bash
+
+if [ ! -z "$*" ]; then
+ $@ 2>&1 | while read line;do
+  echo $(date +"%T") $line
+ done
+ exit ${PIPESTATUS[0]}
+else
+ while read line;do
+   echo $(date +"%T") $line
+  done
+  echo ret $?
+fi

tests/cicd/.gitignore

@@ -0,0 +1 @@
+build.*