optimized cicd

Signed-off-by: Ziming Zhang <zziming@vmware.com>
Change-Id: If900968d2afb8a55ed15b279354d427bb0c93b24

make/photon/prepare/utils/docker_compose.py

@@ -6,6 +6,7 @@
 
 docker_compose_template_path = os.path.join(templates_dir, 'docker_compose', 'docker-compose.yml.jinja')
 docker_compose_yml_path = '/compose_location/docker-compose.yml'
+os.makedirs('/compose_location',exist_ok=True)
 
 # render docker-compose
 def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum):

tests/cicd/.gitignore

@@ -0,0 +1 @@
+build.*

tests/cicd/fixcicdharbor.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+import yaml,os
+print("fix cicd harbor")
+
+config=yaml.safe_load(open('/input/harbor.yml'))
+config['hostname']=os.environ.get('IP', '127.0.0.1')
+config['data_volume']=os.environ.get('data_volume', '/data')
+config['http']['port']=os.environ.get('HTTP_PORT', 80)
+config['https']={}
+config['https']['port']=os.environ.get('HTTPS_PORT', 443)
+config['https']['certificate']=os.environ.get('certificate', '/cert/server.crt')
+config['https']['private_key']=os.environ.get('private_key', '/cert/server.key')
+config['log']['local']['location']=os.environ.get('data_volume', '/data')+'/logs'
+
+yaml.dump(config, open('/input/harbor.yml', 'w+'))
+
+versions=yaml.safe_load(open('versions'))
+versions['VERSION_TAG']=os.environ.get('TAG', 'dev')
+yaml.dump(versions, open('versions', 'w+'))
+
+import main
+try:
+    main.main()
+except SystemExit as e:
+    if e.code != 0:
+        raise e
+
+compose=yaml.safe_load(open('/compose_location/docker-compose.yml'))
+NAMESPACE=os.environ.get('NAMESPACE', 'goharbor')
+for s in compose['services'].values():
+    s['image']=s['image'].replace('goharbor'+"/", NAMESPACE+'/')
+    s['container_name']=s['container_name']+"-"+versions['VERSION_TAG']
+    if isinstance(s['networks'], dict):
+        nn={}
+        for n in s['networks']:
+            nn[n+"-"+versions['VERSION_TAG']]=s['networks'][n]
+        s['networks']=nn
+    else:
+        nn=[]
+        for n in s['networks']:
+            nn.append(n+"-"+versions['VERSION_TAG'])
+        s['networks']=nn
+nn={}
+for n in compose['networks']:
+    nn[n+"-"+versions['VERSION_TAG']]=compose['networks'][n]
+compose['networks']=nn
+yaml.dump(compose, open('/compose_location/docker-compose.yml', 'w+'))

tests/cicd/startcicdharbor.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+if [ -z "$2" ];then echo "$0 <ip> <buildnum> [http_port] [https_port]";exit 1;fi
+IP=$1
+BUILDNUM=$2
+HTTP_PORT=${3:-80}
+HTTPS_PORT=${4:-443}
+
+TAG=build.$BUILDNUM
+NAMESPACE="cicd.harbor.bitsf.xin/harbor-dev"
+data_path=$(pwd)/$TAG/data
+mkdir -p $data_path
+config_dir=$(pwd)/$TAG/common/config
+mkdir -p $config_dir
+mkdir -p $data_path/logs
+compose_file=$(pwd)/$TAG/docker-compose.yml
+touch $compose_file
+secret_dir=$data_path/secret
+mkdir -p $secret_dir
+cert_path=$data_path/cert
+mkdir -p $cert_path
+
+docker pull $NAMESPACE/registry-photon:v2.7.1-patch-2819
+docker tag $NAMESPACE/registry-photon:v2.7.1-patch-2819 $NAMESPACE/registry-photon:v2.7.1-patch-2819-$TAG
+for name in prepare harbor-registryctl nginx-photon harbor-portal harbor-jobservice harbor-core harbor-db redis-photon harbor-log; do
+  docker pull $NAMESPACE/$name:$TAG
+done
+
+curl https://raw.githubusercontent.com/goharbor/harbor/master/tests/harbor_ca.key -o $cert_path/harbor_ca.key
+curl https://raw.githubusercontent.com/goharbor/harbor/master/tests/harbor_ca.crt -o $cert_path/harbor_ca.crt
+openssl req \
+    -newkey rsa:4096 -nodes -sha256 -keyout $cert_path/$IP.key \
+    -out $cert_path/$IP.csr -subj "/C=CN/ST=PEK/L=Bei Jing/O=VMware/CN=HarborManager"
+echo subjectAltName = IP:$IP > $cert_path/extfile.cnf
+openssl x509 -req -days 365 -sha256 -in $cert_path/$IP.csr -CA $cert_path/harbor_ca.crt \
+	-CAkey $cert_path/harbor_ca.key -CAcreateserial -CAserial $cert_path/$IP.srl -extfile $cert_path/extfile.cnf -out $cert_path/$IP.crt
+
+docker run --rm -v $(pwd)/fixcicdharbor.py:/usr/src/app/fixcicdharbor.py \
+                    -v $data_path:/data:z \
+                    -v $compose_file:/compose_location/docker-compose.yml:z \
+                    -v $config_dir:/config:z \
+                    -v $secret_dir:/secret:z \
+                    -v $cert_path/$IP.key:/hostfs/cert/server.key:z \
+                    -v $cert_path/$IP.crt:/hostfs/cert/server.crt:z \
+                    -e IP=$IP -e HTTP_PORT=$HTTP_PORT -e HTTPS_PORT=$HTTPS_PORT \
+                    -e data_volume=$data_path \
+                    -e TAG=$TAG -e NAMESPACE=$NAMESPACE \
+                    --entrypoint ./fixcicdharbor.py \
+                    $NAMESPACE/prepare:$TAG \
+                    || exit 1
+
+sudo chmod -R +r $TAG
+sudo chmod -R 700 $data_path/database
+
+cd $TAG
+docker-compose down
+docker-compose up -d
+
+echo enjoy you harbor at http://$IP:$HTTP_PORT

tests/cicd/stopcicdharbor.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+if [ -z "$1" ];then echo "$0 <buildnum> [action]";exit 1;fi
+BUILDNUM=$1
+ACTION=${2:-stop}
+
+for name in nginx harbor-jobservice harbor-portal harbor-core registry registryctl harbor-db redis harbor-log; do
+  docker $ACTION $name-build.$BUILDNUM
+done