Merge pull request #10694 from danielpacak/feature/install_with_trivy

chore(install): Add --with-trivy arg to the installation script

Makefile

@@ -77,6 +77,7 @@ REGISTRYPROJECTNAME=goharbor
 DEVFLAG=true
 NOTARYFLAG=false
 CLAIRFLAG=false
+TRIVYFLAG=false
 HTTPPROXY=
 BUILDBIN=false
 MIGRATORFLAG=false
@@ -104,6 +105,8 @@ MIGRATORVERSION=$(VERSIONTAG)
 REDISVERSION=$(VERSIONTAG)
 NOTARYMIGRATEVERSION=v3.5.4
 CLAIRADAPTERVERSION=v1.0.1
+TRIVYVERSION=v0.4.3
+TRIVYADAPTERVERSION=v0.2.3
 
 # version of chartmuseum
 CHARTMUSEUMVERSION=v0.9.0
@@ -117,6 +120,8 @@ REGISTRY_VERSION: $(REGISTRYVERSION)
 NOTARY_VERSION: $(NOTARYVERSION)
 CLAIR_VERSION: $(CLAIRVERSION)
 CLAIR_ADAPTER_VERSION: $(CLAIRADAPTERVERSION)
+TRIVY_VERSION: $(TRIVYVERSION)
+TRIVY_ADAPTER_VERSION: $(TRIVYADAPTERVERSION)
 CHARTMUSEUM_VERSION: $(CHARTMUSEUMVERSION)
 endef
 
@@ -193,6 +198,9 @@ endif
 ifeq ($(CLAIRFLAG), true)
 	PREPARECMD_PARA+= --with-clair
 endif
+ifeq ($(TRIVYFLAG), true)
+	PREPARECMD_PARA+= --with-trivy
+endif
 # append chartmuseum parameters if set
 ifeq ($(CHARTFLAG), true)
     PREPARECMD_PARA+= --with-chartmuseum
@@ -276,6 +284,9 @@ endif
 ifeq ($(CLAIRFLAG), true)
 	DOCKERSAVE_PARA+= goharbor/clair-photon:$(CLAIRVERSION)-$(VERSIONTAG) goharbor/clair-adapter-photon:$(CLAIRADAPTERVERSION)-$(VERSIONTAG)
 endif
+ifeq ($(TRIVYFLAG), true)
+	DOCKERSAVE_PARA+= goharbor/trivy-adapter-photon:$(TRIVYADAPTERVERSION)-$(VERSIONTAG)
+endif
 ifeq ($(MIGRATORFLAG), true)
 	DOCKERSAVE_PARA+= goharbor/harbor-migrator:$(MIGRATORVERSION)
 endif
@@ -348,14 +359,16 @@ prepare: update_prepare_version
 
 build:
 	make -f $(MAKEFILEPATH_PHOTON)/Makefile $(BUILDTARGET) -e DEVFLAG=$(DEVFLAG) -e GOBUILDIMAGE=$(GOBUILDIMAGE) \
-	 -e REGISTRYVERSION=$(REGISTRYVERSION) -e REGISTRY_SRC_TAG=$(REGISTRY_SRC_TAG) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \
+	 -e REGISTRYVERSION=$(REGISTRYVERSION) -e REGISTRY_SRC_TAG=$(REGISTRY_SRC_TAG) -e NGINXVERSION=$(NGINXVERSION) \
+	 -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \
+	 -e TRIVYVERSION=$(TRIVYVERSION) -e TRIVYADAPTERVERSION=$(TRIVYADAPTERVERSION) \
 	 -e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRADAPTERVERSION=$(CLAIRADAPTERVERSION) -e VERSIONTAG=$(VERSIONTAG) \
 	 -e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) \
 	 -e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER) \
 	 -e NPM_REGISTRY=$(NPM_REGISTRY) -e BASEIMAGETAG=$(BASEIMAGETAG)
 
 build_base_docker:
-	@for name in chartserver clair clair-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
+	@for name in chartserver clair clair-adapter trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
 		echo $$name ; \
 		$(DOCKERBUILD) --pull -f $(MAKEFILEPATH_PHOTON)/$$name/Dockerfile.base -t goharbor/harbor-$$name-base:$(BASEIMAGETAG) . ; \
 		$(PUSHSCRIPTPATH)/$(PUSHSCRIPTNAME) goharbor/harbor-$$name-base:$(BASEIMAGETAG) $(REGISTRYUSER) $(REGISTRYPASSWORD) ; \

make/harbor.yml.tmpl

@@ -141,6 +141,7 @@ _version: 1.10.0
 #   jobservice_db_index: 2
 #   chartmuseum_db_index: 3
 #   clair_db_index: 4
+#   trivy_db_index: 5
 
 # Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
 # uaa:

make/install.sh

@@ -10,13 +10,16 @@ set +o noglob
 usage=$'Please set hostname and other necessary attributes in harbor.yml first. DO NOT use localhost or 127.0.0.1 for hostname, because Harbor needs to be accessed by external clients.
 Please set --with-notary if needs enable Notary in Harbor, and set ui_url_protocol/ssl_cert/ssl_cert_key in harbor.yml bacause notary must run under https. 
 Please set --with-clair if needs enable Clair in Harbor
+Please set --with-tivy if needs enable Trivy in Harbor
 Please set --with-chartmuseum if needs enable Chartmuseum in Harbor'
 item=0
 
 # notary is not enabled by default
 with_notary=$false
 # clair is not enabled by default
 with_clair=$false
+# trivy is not enabled by default
+with_trivy=$false
 # chartmuseum is not enabled by default
 with_chartmuseum=$false
 
@@ -29,6 +32,8 @@ while [ $# -gt 0 ]; do
             with_notary=true;;
             --with-clair)
             with_clair=true;;
+            --with-trivy)
+            with_trivy=true;;
             --with-chartmuseum)
             with_chartmuseum=true;;
             *)
@@ -70,6 +75,10 @@ if [ $with_clair ]
 then
     prepare_para="${prepare_para} --with-clair"
 fi
+if [ $with_trivy ]
+then
+    prepare_para="${prepare_para} --with-trivy"
+fi
 if [ $with_chartmuseum ]
 then
     prepare_para="${prepare_para} --with-chartmuseum"

make/photon/Makefile

@@ -60,11 +60,18 @@ DOCKERIMAGENAME_POSTGRESQL=goharbor/postgresql-photon
 DOCKERFILEPATH_CLAIR=$(DOCKERFILEPATH)/clair
 DOCKERFILENAME_CLAIR=Dockerfile
 DOCKERIMAGENAME_CLAIR=goharbor/clair-photon
+CLAIR_ADAPTER_DOWNLOAD_URL=https://github.com/goharbor/harbor-scanner-clair/releases/download/$(CLAIRADAPTERVERSION)/harbor-scanner-clair_$(CLAIRADAPTERVERSION:v%=%)_Linux_x86_64.tar.gz
 
 DOCKERFILEPATH_CLAIR_ADAPTER=$(DOCKERFILEPATH)/clair-adapter
 DOCKERFILENAME_CLAIR_ADAPTER=Dockerfile
 DOCKERIMAGENAME_CLAIR_ADAPTER=goharbor/clair-adapter-photon
 
+DOCKERFILEPATH_TRIVY_ADAPTER=$(DOCKERFILEPATH)/trivy-adapter
+DOCKERFILENAME_TRIVY_ADAPTER=Dockerfile
+DOCKERIMAGENAME_TRIVY_ADAPTER=goharbor/trivy-adapter-photon
+TRIVY_DOWNLOAD_URL=https://github.com/aquasecurity/trivy/releases/download/$(TRIVYVERSION)/trivy_$(TRIVYVERSION:v%=%)_Linux-64bit.tar.gz
+TRIVY_ADAPTER_DOWNLOAD_URL=https://github.com/aquasecurity/harbor-scanner-trivy/releases/download/$(TRIVYADAPTERVERSION)/harbor-scanner-trivy_$(TRIVYADAPTERVERSION:v%=%)_Linux_x86_64.tar.gz
+
 DOCKERFILEPATH_NGINX=$(DOCKERFILEPATH)/nginx
 DOCKERFILENAME_NGINX=Dockerfile
 DOCKERIMAGENAME_NGINX=goharbor/nginx-photon
@@ -148,17 +155,40 @@ _build_clair_adapter:
 	@if [ "$(CLAIRFLAG)" = "true" ] ; then \
 		if [ "$(BUILDBIN)" != "true" ] ; then \
 			rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary && mkdir -p $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary && \
-			$(call _extract_archive, https://github.com/goharbor/harbor-scanner-clair/releases/download/$(CLAIRADAPTERVERSION)/harbor-scanner-clair_$(CLAIRADAPTERVERSION:v%=%)_Linux_x86_64.tar.gz, $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/) && \
+			$(call _extract_archive, $(CLAIR_ADAPTER_DOWNLOAD_URL), $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/) && \
 			mv $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/scanner-clair $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/harbor-scanner-clair; \
 		else \
-			cd $(DOCKERFILEPATH_CLAIR_ADAPTER) && $(DOCKERFILEPATH_CLAIR_ADAPTER)/builder $(CLAIRADAPTERVERSION) && cd - ; \
+			cd $(DOCKERFILEPATH_CLAIR_ADAPTER) && $(DOCKERFILEPATH_CLAIR_ADAPTER)/builder.sh $(CLAIRADAPTERVERSION) && cd - ; \
 		fi ; \
-		echo "building clair adapter container for photon..." ; \
-		$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) -f $(DOCKERFILEPATH_CLAIR_ADAPTER)/$(DOCKERFILENAME_CLAIR_ADAPTER) -t $(DOCKERIMAGENAME_CLAIR_ADAPTER):$(CLAIRADAPTERVERSION)-$(VERSIONTAG) . ; \
+		echo "Building Clair adapter container for photon..." ; \
+		$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) \
+			-f $(DOCKERFILEPATH_CLAIR_ADAPTER)/$(DOCKERFILENAME_CLAIR_ADAPTER) \
+			-t $(DOCKERIMAGENAME_CLAIR_ADAPTER):$(CLAIRADAPTERVERSION)-$(VERSIONTAG) . ; \
 		rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary; \
 		echo "Done." ; \
 	fi
 
+_build_trivy_adapter:
+	@if [ "$(TRIVYFLAG)" = "true" ] ; then \
+		rm -rf $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary && mkdir -p $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary ; \
+		echo "Downloading Trivy scanner $(TRIVYVERSION)..." ; \
+		$(call _extract_archive, $(TRIVY_DOWNLOAD_URL), $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary/) ; \
+		if [ "$(BUILDBIN)" != "true" ] ; then \
+			echo "Downloading Trivy adapter $(TRIVYADAPTERVERSION)..." ; \
+			$(call _extract_archive, $(TRIVY_ADAPTER_DOWNLOAD_URL), $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary/) ; \
+		else \
+			echo "Building Trivy adapter $(TRIVYADAPTERVERSION) from sources..." ; \
+			cd $(DOCKERFILEPATH_TRIVY_ADAPTER) && $(DOCKERFILEPATH_TRIVY_ADAPTER)/builder.sh $(TRIVYADAPTERVERSION) && cd - ; \
+		fi ; \
+		echo "Building Trivy adapter container for photon..." ; \
+		$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) \
+			--build-arg trivy_version=$(TRIVYVERSION) \
+			-f $(DOCKERFILEPATH_TRIVY_ADAPTER)/$(DOCKERFILENAME_TRIVY_ADAPTER) \
+			-t $(DOCKERIMAGENAME_TRIVY_ADAPTER):$(TRIVYADAPTERVERSION)-$(VERSIONTAG) . ; \
+		rm -rf $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary; \
+		echo "Done." ; \
+	fi
+
 _build_chart_server:
 	@if [ "$(CHARTFLAG)" = "true" ] ; then \
 		if [ "$(BUILDBIN)" != "true" ] ; then \
@@ -231,7 +261,7 @@ define _get_binary
 	$(WGET) --timeout 30 --no-check-certificate $1 -O $2 || exit 1
 endef
 
-build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_clair_adapter _build_redis _build_migrator _build_chart_server
+build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_clair_adapter _build_trivy_adapter _build_redis _build_migrator _build_chart_server
 
 cleanimage:
 	@echo "cleaning image for photon..."

make/photon/clair-adapter/builder → make/photon/clair-adapter/builder.sh

@@ -15,25 +15,25 @@ set -e
 mkdir -p binary
 rm -rf binary/harbor-scanner-clair || true
 
-cd `dirname $0`
+cd $(dirname $0)
 cur=$PWD
 
-# the temp folder to store distribution source code...
-TEMP=`mktemp -d ${TMPDIR-/tmp}/clair-adapter.XXXXXX`
+# The temporary directory to clone Clair adapter source code
+TEMP=$(mktemp -d ${TMPDIR-/tmp}/clair-adapter.XXXXXX)
 git clone https://github.com/goharbor/harbor-scanner-clair.git $TEMP
 cd $TEMP; git checkout $VERSION; cd -
 
-echo 'build the clair adapter binary bases on the golang:1.13.4'
+echo "Building Clair adapter binary based on golang:1.13.4..."
 cp Dockerfile.binary $TEMP
 docker build -f $TEMP/Dockerfile.binary -t clair-adapter-golang $TEMP
 
-echo 'copy the clair adapter binary to local...'
+echo "Copying Clair adapter binary from the container to the local directory..."
 ID=$(docker create clair-adapter-golang)
 docker cp $ID:/go/src/github.com/goharbor/harbor-scanner-clair/harbor-scanner-clair binary
 
 docker rm -f $ID
 docker rmi -f clair-adapter-golang
 
-echo "Build clair adapter binary success, then to build photon image..."
+echo "Building Clair adapter binary finished successfully"
 cd $cur
 rm -rf $TEMP

make/photon/prepare/main.py

@@ -15,6 +15,7 @@
 from utils.log import prepare_log_configs
 from utils.clair import prepare_clair
 from utils.clair_adapter import prepare_clair_adapter
+from utils.trivy_adapter import prepare_trivy_adapter
 from utils.chart import prepare_chartmuseum
 from utils.docker_compose import prepare_docker_compose
 from utils.nginx import prepare_nginx, nginx_confd_dir
@@ -27,11 +28,12 @@
 @click.option('--conf', default=input_config_path, help="the path of Harbor configuration file")
 @click.option('--with-notary', is_flag=True, help="the Harbor instance is to be deployed with notary")
 @click.option('--with-clair', is_flag=True, help="the Harbor instance is to be deployed with clair")
+@click.option('--with-trivy', is_flag=True, help="the Harbor instance is to be deployed with Trivy")
 @click.option('--with-chartmuseum', is_flag=True, help="the Harbor instance is to be deployed with chart repository supporting")
-def main(conf, with_notary, with_clair, with_chartmuseum):
+def main(conf, with_notary, with_clair, with_trivy, with_chartmuseum):
 
     delfile(config_dir)
-    config_dict = parse_yaml_config(conf, with_notary=with_notary, with_clair=with_clair, with_chartmuseum=with_chartmuseum)
+    config_dict = parse_yaml_config(conf, with_notary=with_notary, with_clair=with_clair, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
     try:
         validate(config_dict, notary_mode=with_notary)
     except Exception as e:
@@ -41,7 +43,7 @@ def main(conf, with_notary, with_clair, with_chartmuseum):
 
     prepare_log_configs(config_dict)
     prepare_nginx(config_dict)
-    prepare_core(config_dict, with_notary=with_notary, with_clair=with_clair, with_chartmuseum=with_chartmuseum)
+    prepare_core(config_dict, with_notary=with_notary, with_clair=with_clair, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
     prepare_registry(config_dict)
     prepare_registry_ctl(config_dict)
     prepare_db(config_dict)
@@ -63,10 +65,13 @@ def main(conf, with_notary, with_clair, with_chartmuseum):
         prepare_clair(config_dict)
         prepare_clair_adapter(config_dict)
 
+    if with_trivy:
+        prepare_trivy_adapter(config_dict)
+
     if with_chartmuseum:
         prepare_chartmuseum(config_dict)
 
-    prepare_docker_compose(config_dict, with_clair, with_notary, with_chartmuseum)
+    prepare_docker_compose(config_dict, with_clair, with_trivy, with_notary, with_chartmuseum)
 
 if __name__ == '__main__':
-    main()
+    main()

make/photon/prepare/templates/core/env.jinja

@@ -26,6 +26,7 @@ CORE_SECRET={{core_secret}}
 JOBSERVICE_SECRET={{jobservice_secret}}
 WITH_NOTARY={{with_notary}}
 WITH_CLAIR={{with_clair}}
+WITH_TRIVY={{with_trivy}}
 CLAIR_DB_PASSWORD={{clair_db_password}}
 CLAIR_DB_HOST={{clair_db_host}}
 CLAIR_DB_PORT={{clair_db_port}}
@@ -37,6 +38,7 @@ CORE_LOCAL_URL={{core_local_url}}
 JOBSERVICE_URL={{jobservice_url}}
 CLAIR_URL={{clair_url}}
 CLAIR_ADAPTER_URL={{clair_adapter_url}}
+TRIVY_ADAPTER_URL={{trivy_adapter_url}}
 NOTARY_URL={{notary_url}}
 REGISTRY_STORAGE_PROVIDER_NAME={{storage_provider_name}}
 READ_ONLY=false

make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja

@@ -437,6 +437,35 @@ services:
     env_file:
       ./common/config/clair-adapter/env
 {% endif %}
+{% if with_trivy %}
+  trivy-adapter:
+    container_name: trivy-adapter
+    image: goharbor/trivy-adapter-photon:{{trivy_adapter_version}}
+    restart: always
+    cap_drop:
+      - ALL
+    dns_search: .
+    networks:
+      - harbor
+{% if external_redis == False %}
+    depends_on:
+      - redis
+{% endif %}
+    volumes:
+      - type: bind
+        source: {{data_volume}}/trivy-adapter/trivy
+        target: /home/scanner/.cache/trivy
+      - type: bind
+        source: {{data_volume}}/trivy-adapter/reports
+        target: /home/scanner/.cache/reports
+    logging:
+      driver: "syslog"
+      options:
+        syslog-address: "tcp://127.0.0.1:1514"
+        tag: "trivy-adapter"
+    env_file:
+      ./common/config/trivy-adapter/env
+{% endif %}
 {% if with_chartmuseum %}
   chartmuseum:
     container_name: chartmuseum
@@ -491,4 +520,4 @@ networks:
 {% if with_chartmuseum %}
   harbor-chartmuseum:
     external: false
-{% endif %}
+{% endif %}

make/photon/prepare/templates/trivy-adapter/env.jinja

@@ -0,0 +1,10 @@
+SCANNER_LOG_LEVEL={{log_level}}
+SCANNER_STORE_REDIS_URL={{redis_url_trivy}}
+SCANNER_STORE_REDIS_NAMESPACE=harbor.scanner.trivy:store
+SCANNER_JOB_QUEUE_REDIS_URL={{redis_url_trivy}}
+SCANNER_JOB_QUEUE_REDIS_NAMESPACE=harbor.scanner.trivy:job-queue
+SCANNER_TRIVY_CACHE_DIR=/home/scanner/.cache/trivy
+SCANNER_TRIVY_REPORTS_DIR=/home/scanner/.cache/reports
+SCANNER_TRIVY_VULN_TYPE=os,library
+SCANNER_TRIVY_SEVERITY=UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL
+SCANNER_TRIVY_IGNORE_UNFIXED=false