Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize cicd #8652

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
18 changes: 10 additions & 8 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ env:
- CORE_SECRET: tempString
- KEY_PATH: "/data/secret/keys/secretkey"
- REDIS_HOST: localhost
- REG_VERSION: v2.7.1
- REG_VERSION: v2.7.1-patch-2819
- UI_BUILDER_VERSION: 1.6.0
- TOKEN_PRIVATE_KEY_PATH: "/home/travis/gopath/src/github.com/goharbor/harbor/tests/private_key.pem"
addons:
Expand All @@ -52,15 +52,17 @@ before_install:
- IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'`
- sudo sed -i '$a DOCKER_OPTS=\"--insecure-registry '$IP':5000\"' /etc/default/docker
- export IP=$IP
- sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
- sudo update-ca-certificates
- sudo service docker restart
install:
- if [ "$UTTEST" == true ]; then bash ./tests/travis/ut_install.sh ; fi
- if [ "$APITEST_DB" == true ]; then bash ./tests/travis/api_common_install.sh $IP
- if [ "$UTTEST" == true ]; then bash ./showtime.sh ./tests/travis/ut_install.sh ; fi
- if [ "$APITEST_DB" == true ]; then bash ./showtime.sh ./tests/travis/api_common_install.sh $IP
DB; fi
- if [ "$APITEST_LDAP" == true ]; then bash ./tests/travis/api_common_install.sh $IP
- if [ "$APITEST_LDAP" == true ]; then bash ./showtime.sh ./tests/travis/api_common_install.sh $IP
LDAP; fi
script:
- if [ "$UTTEST" == true ]; then bash ./tests/travis/ut_run.sh $IP; fi
- if [ "$APITEST_DB" == true ]; then bash ./tests/travis/api_run.sh DB $IP; fi
- if [ "$APITEST_LDAP" == true ]; then bash ./tests/travis/api_run.sh LDAP $IP; fi
- if [ "$OFFLINE" == true ]; then bash ./tests/travis/distro_installer.sh; fi
- if [ "$UTTEST" == true ]; then bash ./showtime.sh ./tests/travis/ut_run.sh $IP; fi
- if [ "$APITEST_DB" == true ]; then bash ./showtime.sh ./tests/travis/api_run.sh DB $IP; fi
- if [ "$APITEST_LDAP" == true ]; then bash ./showtime.sh ./tests/travis/api_run.sh LDAP $IP; fi
- if [ "$OFFLINE" == true ]; then bash ./showtime.sh ./tests/travis/distro_installer.sh; fi
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ BUILDBIN=false
MIGRATORFLAG=false
# enable/disable chart repo supporting
CHARTFLAG=false
BUILDTARGET=build

# version prepare
# for docker image tag
Expand Down Expand Up @@ -300,7 +301,7 @@ prepare: update_prepare_version
@$(MAKEPATH)/$(PREPARECMD) $(PREPARECMD_PARA)

build:
make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) \
make -f $(MAKEFILEPATH_PHOTON)/Makefile $(BUILDTARGET) -e DEVFLAG=$(DEVFLAG) \
-e REGISTRYVERSION=$(REGISTRYVERSION) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \
-e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRDBVERSION=$(CLAIRDBVERSION) -e VERSIONTAG=$(VERSIONTAG) \
-e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) \
Expand Down
13 changes: 13 additions & 0 deletions build_base_docker.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#!/usr/bin/env bash

DOCKERCMD=docker
CICDHOST=cicd.harbor.bitsf.xin
DOCKERFILEPATH=make/photon

${DOCKERCMD} login ${CICDHOST} || exit 2
for name in chartserver clair core db jobservice log nginx portal prepare redis registry registryctl; do
echo building $name base docker image
$DOCKERCMD build -f $DOCKERFILEPATH/$name/Dockerfile-base -t $CICDHOST/harbor-depend/$name:base . && \
$DOCKERCMD push $CICDHOST/harbor-depend/$name:base
if [ "$?" != "0" ]; then exit 1; fi
done
7 changes: 7 additions & 0 deletions make/photon/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,13 @@ CHART_SERVER_CODE_BASE=github.com/helm/chartmuseum
CHART_SERVER_MAIN_PATH=cmd/chartmuseum
CHART_SERVER_BIN_NAME=chartm

build_base_docker:
for name in chartserver clair common core db jobservice log nginx notary portal prepare redis registry registryctl; do \
echo $(name) ; \
$(DOCKERBUILD) -f $(DOCKERFILEPATH)/$(name) -t cicd.harbor.bitsf.xin/harbor-depend/$(name):base . ; \
$(DOCKERBUILD) push cicd.harbor.bitsf.xin/harbor-depend/$(name):base ; \
done

_build_prepare:
@echo "building prepare container for photon..."
@$(DOCKERBUILD) -f $(DOCKERFILEPATH_PREPARE)/$(DOCKERFILENAME_PREPARE) -t $(DOCKERIMAGENAME_PREPARE):$(VERSIONTAG) .
Expand Down
8 changes: 1 addition & 7 deletions make/photon/chartserver/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,11 +1,5 @@
FROM photon:2.0
FROM cicd.harbor.bitsf.xin/harbor-depend/chartserver:base

RUN tdnf install -y shadow sudo >>/dev/null\
&& tdnf clean all \
&& mkdir /chartserver/ \
&& mkdir /harbor/ \
&& groupadd -r -g 10000 chartuser \
&& useradd --no-log-init -m -r -g 10000 -u 10000 chartuser
COPY ./make/photon/chartserver/binary/chartm /chartserver/
COPY ./make/photon/chartserver/docker-entrypoint.sh /docker-entrypoint.sh
COPY ./make/photon/common/install_cert.sh /harbor
Expand Down
8 changes: 8 additions & 0 deletions make/photon/chartserver/Dockerfile-base
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
FROM photon:2.0

RUN tdnf install -y shadow sudo >>/dev/null\
&& tdnf clean all \
&& mkdir /chartserver/ \
&& mkdir /harbor/ \
&& groupadd -r -g 10000 chartuser \
&& useradd --no-log-init -m -r -g 10000 -u 10000 chartuser
8 changes: 1 addition & 7 deletions make/photon/clair/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,11 +1,5 @@
FROM photon:2.0
FROM cicd.harbor.bitsf.xin/harbor-depend/clair:base

RUN tdnf install -y git shadow sudo rpm xz python-xml >>/dev/null\
&& tdnf clean all \
&& mkdir /clair/ \
&& mkdir /harbor \
&& groupadd -r -g 10000 clair \
&& useradd --no-log-init -m -r -g 10000 -u 10000 clair
COPY ./make/photon/clair/binary/clair /clair/
COPY ./make/photon/clair/docker-entrypoint.sh /docker-entrypoint.sh
COPY ./make/photon/clair/dumb-init /dumb-init
Expand Down
8 changes: 8 additions & 0 deletions make/photon/clair/Dockerfile-base
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
FROM photon:2.0

RUN tdnf install -y git shadow sudo rpm xz python-xml >>/dev/null\
&& tdnf clean all \
&& mkdir /clair/ \
&& mkdir /harbor \
&& groupadd -r -g 10000 clair \
&& useradd --no-log-init -m -r -g 10000 -u 10000 clair
7 changes: 1 addition & 6 deletions make/photon/core/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,9 +1,4 @@
FROM photon:2.0

RUN tdnf install sudo tzdata -y >> /dev/null \
&& tdnf clean all \
&& groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor \
&& mkdir /harbor/
FROM cicd.harbor.bitsf.xin/harbor-depend/core:base

HEALTHCHECK CMD curl --fail -s http://127.0.0.1:8080/api/ping || exit 1
COPY ./make/photon/core/harbor_core ./UIVERSION /harbor/
Expand Down
6 changes: 6 additions & 0 deletions make/photon/core/Dockerfile-base
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
FROM photon:2.0

RUN tdnf install sudo tzdata -y >> /dev/null\
&& tdnf clean all \
&& groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor \
&& mkdir /harbor/
20 changes: 1 addition & 19 deletions make/photon/db/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,22 +1,4 @@
FROM photon:2.0

ENV PGDATA /var/lib/postgresql/data

RUN tdnf install -y shadow gzip postgresql >> /dev/null\
&& groupadd -r postgres --gid=999 \
&& useradd -m -r -g postgres --uid=999 postgres \
&& mkdir -p /docker-entrypoint-initdb.d \
&& mkdir -p /run/postgresql \
&& chown -R postgres:postgres /run/postgresql \
&& chmod 2777 /run/postgresql \
&& mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA" \
&& sed -i "s|#listen_addresses = 'localhost'.*|listen_addresses = '*'|g" /usr/share/postgresql/postgresql.conf.sample \
&& sed -i "s|#unix_socket_directories = '/tmp'.*|unix_socket_directories = '/run/postgresql'|g" /usr/share/postgresql/postgresql.conf.sample \
&& tdnf clean all

RUN tdnf erase -y toybox && tdnf install -y util-linux net-tools

VOLUME /var/lib/postgresql/data
FROM cicd.harbor.bitsf.xin/harbor-depend/db:base

COPY ./make/photon/db/docker-entrypoint.sh /docker-entrypoint.sh
COPY ./make/photon/db/docker-healthcheck.sh /docker-healthcheck.sh
Expand Down
19 changes: 19 additions & 0 deletions make/photon/db/Dockerfile-base
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
FROM photon:2.0

ENV PGDATA /var/lib/postgresql/data

RUN tdnf install -y shadow gzip postgresql >> /dev/null\
&& groupadd -r postgres --gid=999 \
&& useradd -m -r -g postgres --uid=999 postgres \
&& mkdir -p /docker-entrypoint-initdb.d \
&& mkdir -p /run/postgresql \
&& chown -R postgres:postgres /run/postgresql \
&& chmod 2777 /run/postgresql \
&& mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA" \
&& sed -i "s|#listen_addresses = 'localhost'.*|listen_addresses = '*'|g" /usr/share/postgresql/postgresql.conf.sample \
&& sed -i "s|#unix_socket_directories = '/tmp'.*|unix_socket_directories = '/run/postgresql'|g" /usr/share/postgresql/postgresql.conf.sample \
&& tdnf clean all

RUN tdnf erase -y toybox && tdnf install -y util-linux net-tools

VOLUME /var/lib/postgresql/data
6 changes: 1 addition & 5 deletions make/photon/jobservice/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,8 +1,4 @@
FROM photon:2.0

RUN tdnf install sudo tzdata -y >> /dev/null \
&& tdnf clean all \
&& groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor
FROM cicd.harbor.bitsf.xin/harbor-depend/jobservice:base

COPY ./make/photon/jobservice/harbor_jobservice /harbor/

Expand Down
5 changes: 5 additions & 0 deletions make/photon/jobservice/Dockerfile-base
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
FROM photon:2.0

RUN tdnf install sudo tzdata -y >> /dev/null\
&& tdnf clean all \
&& groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor
7 changes: 1 addition & 6 deletions make/photon/log/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,9 +1,4 @@
FROM photon:2.0

RUN tdnf install -y cronie rsyslog logrotate shadow tar gzip sudo >> /dev/null\
&& mkdir /var/spool/rsyslog \
&& groupadd -r -g 10000 syslog && useradd --no-log-init -r -g 10000 -u 10000 syslog \
&& tdnf clean all
FROM cicd.harbor.bitsf.xin/harbor-depend/log:base

COPY ./make/photon/log/rsyslog.conf /etc/rsyslog.conf

Expand Down
6 changes: 6 additions & 0 deletions make/photon/log/Dockerfile-base
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
FROM photon:2.0

RUN tdnf install -y cronie rsyslog logrotate shadow tar gzip sudo >> /dev/null\
&& mkdir /var/spool/rsyslog \
&& groupadd -r -g 10000 syslog && useradd --no-log-init -r -g 10000 -u 10000 syslog \
&& tdnf clean all
10 changes: 1 addition & 9 deletions make/photon/nginx/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,12 +1,4 @@
FROM photon:2.0

RUN tdnf install sudo nginx -y >> /dev/null\
&& tdnf clean all \
&& groupadd -r -g 10000 nginx && useradd --no-log-init -r -g 10000 -u 10000 nginx \
&& ln -sf /dev/stdout /var/log/nginx/access.log \
&& ln -sf /dev/stderr /var/log/nginx/error.log

VOLUME /var/cache/nginx /var/log/nginx /run
FROM cicd.harbor.bitsf.xin/harbor-depend/nginx:base

EXPOSE 8080

Expand Down
9 changes: 9 additions & 0 deletions make/photon/nginx/Dockerfile-base
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
FROM photon:2.0

RUN tdnf install sudo nginx -y >> /dev/null\
&& tdnf clean all \
&& groupadd -r -g 10000 nginx && useradd --no-log-init -r -g 10000 -u 10000 nginx \
&& ln -sf /dev/stdout /var/log/nginx/access.log \
&& ln -sf /dev/stderr /var/log/nginx/error.log

VOLUME /var/cache/nginx /var/log/nginx /run
13 changes: 3 additions & 10 deletions make/photon/portal/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,15 @@ RUN cp -r /portal_src/* /build_dir \
&& npm run release


FROM photon:2.0
FROM cicd.harbor.bitsf.xin/harbor-depend/portal:base

COPY make/photon/portal/nginx.conf /etc/nginx/nginx.conf

COPY --from=nodeportal /build_dir/dist /usr/share/nginx/html
COPY --from=nodeportal /build_dir/swagger.yaml /usr/share/nginx/html
COPY --from=nodeportal /build_dir/swagger.json /usr/share/nginx/html
COPY --from=nodeportal /build_dir/LICENSE /usr/share/nginx/html

COPY make/photon/portal/nginx.conf /etc/nginx/nginx.conf

RUN tdnf install -y nginx sudo >> /dev/null \
&& ln -sf /dev/stdout /var/log/nginx/access.log \
&& ln -sf /dev/stderr /var/log/nginx/error.log \
&& groupadd -r -g 10000 nginx && useradd --no-log-init -r -g 10000 -u 10000 nginx \
&& chown -R nginx:nginx /etc/nginx \
&& tdnf clean all

EXPOSE 8080
VOLUME /var/cache/nginx /var/log/nginx /run

Expand Down
8 changes: 8 additions & 0 deletions make/photon/portal/Dockerfile-base
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
FROM photon:2.0

RUN tdnf install -y nginx sudo >> /dev/null \
&& ln -sf /dev/stdout /var/log/nginx/access.log \
&& ln -sf /dev/stderr /var/log/nginx/error.log \
&& groupadd -r -g 10000 nginx && useradd --no-log-init -r -g 10000 -u 10000 nginx \
&& chown -R nginx:nginx /etc/nginx \
&& tdnf clean all
10 changes: 4 additions & 6 deletions make/photon/prepare/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,18 +1,16 @@
FROM photon:2.0
FROM cicd.harbor.bitsf.xin/harbor-depend/prepare:base

ENV LANG en_US.UTF-8

WORKDIR /usr/src/app

RUN mkdir -p /harbor_make

RUN tdnf install -y python3 \
&& tdnf install -y python3-pip
RUN pip3 install pipenv==2018.11.26

COPY make/photon/prepare /usr/src/app
RUN set -ex && pipenv install --deploy --system

ENTRYPOINT [ "python3", "main.py" ]

VOLUME ["/harbor_make"]
VOLUME ["/harbor_make"]

COPY make/harbor.yml /input/harbor.yml
8 changes: 8 additions & 0 deletions make/photon/prepare/Dockerfile-base
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
FROM photon:2.0

ENV LANG en_US.UTF-8

RUN tdnf install -y python3 \
&& tdnf install -y python3-pip
RUN pip3 install pipenv==2018.11.26

2 changes: 1 addition & 1 deletion make/photon/prepare/templates/nginx/nginx.https.conf.jinja
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,6 @@ http {
server {
listen 8080;
#server_name harbordomain.com;
return 308 https://$host$request_uri;
return 308 https://{{https_redirect}}$request_uri;
}
}
1 change: 1 addition & 0 deletions make/photon/prepare/utils/docker_compose.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

docker_compose_template_path = os.path.join(templates_dir, 'docker_compose', 'docker-compose.yml.jinja')
docker_compose_yml_path = '/compose_location/docker-compose.yml'
os.makedirs('/compose_location',exist_ok=True)

# render docker-compose
def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum):
Expand Down
1 change: 1 addition & 0 deletions make/photon/prepare/utils/nginx.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ def render_nginx_template(config_dict):
nginx_conf,
uid=DEFAULT_UID,
gid=DEFAULT_GID,
https_redirect='$host' + ('https_port' in config_dict and (":" + str(config_dict['https_port'])) or ""),
ssl_cert=SSL_CERT_PATH,
ssl_cert_key=SSL_CERT_KEY_PATH)
location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTPS
Expand Down
4 changes: 1 addition & 3 deletions make/photon/redis/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
FROM photon:2.0

RUN tdnf install -y redis sudo
FROM cicd.harbor.bitsf.xin/harbor-depend/redis:base

VOLUME /var/lib/redis
WORKDIR /var/lib/redis
Expand Down
3 changes: 3 additions & 0 deletions make/photon/redis/Dockerfile-base
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
FROM photon:2.0

RUN tdnf install -y redis sudo
11 changes: 1 addition & 10 deletions make/photon/registry/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,13 +1,4 @@
FROM photon:2.0

MAINTAINER wangyan@vmware.com

# The original script in the docker offical registry image.
RUN tdnf install sudo -y >> /dev/null\
&& tdnf clean all \
&& mkdir /harbor/ \
&& mkdir -p /etc/registry \
&& groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor
FROM cicd.harbor.bitsf.xin/harbor-depend/registry:base

COPY ./make/photon/common/install_cert.sh /harbor
COPY ./make/photon/registry/entrypoint.sh /
Expand Down
8 changes: 8 additions & 0 deletions make/photon/registry/Dockerfile-base
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
FROM photon:2.0

# The original script in the docker offical registry image.
RUN tdnf install sudo -y >> /dev/null\
&& tdnf clean all \
&& mkdir /harbor/ \
&& mkdir -p /etc/registry \
&& groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor
10 changes: 1 addition & 9 deletions make/photon/registryctl/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,12 +1,4 @@
FROM photon:2.0

MAINTAINER wangyan@vmware.com

RUN tdnf install sudo -y >> /dev/null \
&& tdnf clean all \
&& groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor \
&& mkdir -p /etc/registry \
&& mkdir /harbor/
FROM cicd.harbor.bitsf.xin/harbor-depend/registryctl:base

COPY ./make/photon/common/install_cert.sh /harbor
COPY ./make/photon/registry/binary/registry /usr/bin
Expand Down
7 changes: 7 additions & 0 deletions make/photon/registryctl/Dockerfile-base
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
FROM photon:2.0

RUN tdnf install sudo -y >> /dev/null \
&& tdnf clean all \
&& groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor \
&& mkdir -p /etc/registry \
&& mkdir /harbor/