add contour support (#659)

Signed-off-by: Ziming Zhang <zziming@vmware.com>

.github/workflows/tests.yml

@@ -104,7 +104,7 @@ jobs:
       - name: Install Kubernetes
         uses: helm/kind-action@v1.1.0
         with:
-          version: v0.9.0
+          version: v0.10.0
           cluster_name: harbor
           config: .github/kind.yaml
 
@@ -131,7 +131,7 @@ jobs:
 
   harbor-tests:
     runs-on: ubuntu-latest
-    name: K8S v${{ matrix.k8sVersion }} (CM v${{ matrix.certManager }} ${{ matrix.storage }})
+    name: K8S v${{ matrix.k8sVersion }} (CM v${{ matrix.certManager }} ${{ matrix.samples }})
     env:
       USE_EXISTING_CLUSTER: true
       operatorNamespace: harbor-operator-ns
@@ -148,7 +148,7 @@ jobs:
 
         # https://snapcraft.io/microk8s
         k8sVersion:
-        - "1.18.8"
+        - "1.18.15"
 #        - "1.19.1"
         - "1.20.2"
 
@@ -187,7 +187,7 @@ jobs:
     - name: Install Kubernetes v${{ matrix.k8sVersion }}
       uses: helm/kind-action@v1.1.0
       with:
-        version: v0.9.0
+        version: v0.10.0
         node_image: kindest/node:v${{ matrix.k8sVersion }}
         cluster_name: harbor
         config: .github/kind.yaml
@@ -321,7 +321,7 @@ jobs:
     - uses: actions/upload-artifact@v2
       if: ${{ failure() }}
       with:
-        name: harbor_v${{ matrix.k8sVersion }}_v${{ matrix.certManager }}_${{ matrix.storage }}
+        name: harbor_v${{ matrix.k8sVersion }}_v${{ matrix.certManager }}_${{ matrix.samples }}
         path: /tmp/harbor
 
     - name: fetch logs
@@ -333,7 +333,7 @@ jobs:
     - uses: actions/upload-artifact@v2
       if: ${{ failure() }}
       with:
-        name: kind_v${{ matrix.k8sVersion }}_v${{ matrix.certManager }}_${{ matrix.storage }}
+        name: kind_v${{ matrix.k8sVersion }}_v${{ matrix.certManager }}_${{ matrix.samples }}
         path: /tmp/logs
 
     - name: Get logs for debug
@@ -343,6 +343,218 @@ jobs:
         kubectl get all -n "${operatorNamespace}" -o wide
         kubectl logs -n "${operatorNamespace}" -l 'control-plane=harbor-operator' --all-containers --tail=1000
 
+  harbor-contour:
+    runs-on: ubuntu-latest
+    name: contour K8S v${{ matrix.k8sVersion }} (CM v${{ matrix.certManager }} ${{ matrix.samples }})
+    env:
+      USE_EXISTING_CLUSTER: true
+      operatorNamespace: harbor-operator-ns
+      dockerImage: harbor-operator:dev_test
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # https://github.com/jetstack/cert-manager/tags
+        certManager:
+          - "1.2.0"
+
+        # https://snapcraft.io/microk8s
+        k8sVersion:
+          - "1.20.2"
+
+        samples:
+          - "full_stack.yaml"
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.14
+
+      - uses: azure/setup-kubectl@v1
+        with:
+          version: 'latest'
+
+      - name: Cache go mod
+        uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Prepare memory storage for etcd of kind cluster
+        run: |
+          # Use memory storage for etcd of the kind cluster, see https://github.com/kubernetes-sigs/kind/issues/845 for more info
+          mkdir -p /tmp/lib/etcd
+          sudo mount -t tmpfs tmpfs /tmp/lib/etcd
+
+      - name: Install Kubernetes v${{ matrix.k8sVersion }}
+        uses: helm/kind-action@v1.1.0
+        with:
+          version: v0.10.0
+          node_image: kindest/node:v${{ matrix.k8sVersion }}
+          cluster_name: harbor
+          config: .github/kind.yaml
+
+      - name: Install CertManager v${{ matrix.certManager }}
+        run: |
+          kubectl apply -f "https://github.com/jetstack/cert-manager/releases/download/v${{ matrix.certManager }}/cert-manager.yaml"
+          sleep 5
+          time kubectl -n cert-manager wait --for=condition=Available deployment --all --timeout 300s
+
+      - name: Install Contour
+        run: |
+          kubectl apply -f https://projectcontour.io/quickstart/contour.yaml
+          sleep 5
+          kubectl patch daemonsets -n projectcontour envoy -p '{"spec":{"template":{"spec":{"nodeSelector":{"ingress-ready":"true"},"tolerations":[{"key":"node-role.kubernetes.io/master","operator":"Equal","effect":"NoSchedule"}]}}}}'
+          sleep 5
+          kubectl get all -n projectcontour
+          time kubectl wait --namespace projectcontour --for=condition=ready pod --selector=app=envoy --timeout=100s || kubectl get all -n projectcontour
+          time kubectl wait --namespace projectcontour --for=condition=ready pod --selector=app=envoy --timeout=100s
+
+      - name: build harbor-operator
+        run: |
+          make manifests docker-build IMG=${dockerImage} GIT_COMMIT=${{ github.sha }}
+          kind load docker-image ${dockerImage} --name harbor
+
+      - name: install harbor-operator
+        run: |
+          cd manifests/cluster
+          kustomize edit add secret github-token --disableNameSuffixHash --from-literal=GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
+          kustomize edit add patch --path patch/github-token.yaml
+          kustomize edit set image goharbor/harbor-operator=${dockerImage}
+          kustomize build | kubectl apply -f -
+
+          if ! time kubectl -n ${operatorNamespace} wait --for=condition=Available deployment --all --timeout 300s; then
+            kubectl get all -n ${operatorNamespace}
+            exit 1
+          fi
+
+      - name: install harbor
+        run: |
+          set -ex
+          IP=`hostname -I | awk '{print $1}'`
+          echo "IP=$IP" >> $GITHUB_ENV
+          CORE_HOST=core.$IP.nip.io
+          NOTARY_HOST=notary.$IP.nip.io
+          MINIO_HOST=minio.$IP.nip.io
+          echo "CORE_HOST=$CORE_HOST" >> $GITHUB_ENV
+          echo "NOTARY_HOST=$NOTARY_HOST" >> $GITHUB_ENV
+          echo "MINIO_HOST=$MINIO_HOST" >> $GITHUB_ENV
+          samplefile=${{ matrix.samples }}
+          sed -i "s/core.harbor.domain/$CORE_HOST/g" manifests/samples/$samplefile
+          sed -i "s/notary.harbor.domain/$NOTARY_HOST/g" manifests/samples/$samplefile
+          sed -i "s/minio.harbor.domain/$MINIO_HOST/g" manifests/samples/$samplefile
+          sed -i "s/controller: default/controller: contour/g" manifests/samples/$samplefile
+          sed -i "s/logLevel: info/logLevel: debug/g" manifests/samples/$samplefile
+
+          kubectl apply -f manifests/samples/$samplefile
+          for i in $(seq 1 7);do
+            sleep 30
+            echo $i
+            kubectl -n cluster-sample-ns get all
+          done
+
+          function wait-for-condition () {
+            time kubectl -n cluster-sample-ns wait --for=condition=$1 harborcluster harborcluster-sample --timeout $2
+          }
+
+          if ! wait-for-condition InProgress=False 600s && ! wait-for-condition Failed=False 60s; then
+            echo install harbor failed
+            kubectl describe harborcluster -n cluster-sample-ns
+            kubectl describe harbor -n cluster-sample-ns
+            kubectl get all -n cluster-sample-ns
+
+            for n in $(kubectl -n cluster-sample-ns get po |grep -v Running|grep -v NAME|awk '{print $1}');do
+              echo describe $n
+              kubectl -n cluster-sample-ns describe pod $n
+              echo show log $n
+              kubectl -n cluster-sample-ns logs --tail 100 $n || true
+            done
+            kubectl logs -l control-plane=harbor-operator -n ${operatorNamespace} --tail 100
+            free -h
+            exit 1
+          else
+            kubectl -n cluster-sample-ns get all -o wide
+            kubectl get harbor -n cluster-sample-ns -o wide
+            kubectl get harborcluster -n cluster-sample-ns -o wide
+          fi
+          free -h
+          df -h
+
+      - name: test harbor
+        run: |
+          set -ex
+          free -h
+          df -h
+          curl https://$CORE_HOST/api/v2.0/systeminfo -i -k -f
+          sudo mkdir -p /etc/docker/certs.d/$CORE_HOST
+          kubectl -n cluster-sample-ns get secret sample-public-certificate -o jsonpath='{.data.ca\.crt}' \
+            | base64 --decode \
+            | sudo tee /etc/docker/certs.d/$CORE_HOST/harbor_ca.crt
+          # docker login, create image, docker push, docker pull
+          docker login $CORE_HOST -u admin -p Harbor12345 || (kubectl -n cluster-sample-ns get po;kubectl -n cluster-sample-ns logs -l goharbor.io/operator-controller=core;exit 1)
+          docker run busybox dd if=/dev/urandom of=test count=10 bs=1MB
+          DOCKERID=`docker ps -l -q`
+          docker commit $DOCKERID $CORE_HOST/library/busybox:test
+          docker push $CORE_HOST/library/busybox:test
+          docker pull $CORE_HOST/library/busybox:test
+
+      - name: apidb test
+        run: |
+          git clone https://github.com/goharbor/harbor -b v2.2.1
+          cd harbor
+          sed -i '15i\    ${dfout}=  Run  df -h\n    Log To Console  ${dfout}' tests/resources/APITest-Util.robot
+          cd ..
+          kubectl -n cluster-sample-ns patch deploy harborcluster-sample-harbor-harbor-core -p '{"spec":{"template":{"spec":{"containers":[{"name":"core","env":[{"name":"GC_TIME_WINDOW_HOURS","value":"0"}]}]}}}}'
+          sleep 5
+          kubectl -n cluster-sample-ns wait --for=condition=Ready -l job-type!=minio-init pod --all --timeout 600s
+          docker run --rm -t --privileged -v `pwd`/harbor:/drone -v /etc/docker/certs.d/$CORE_HOST/:/ca -v /etc/docker/certs.d/$CORE_HOST/:/root/.docker/tls/$NOTARY_HOST/ -e NOTARY_URL=https://$NOTARY_HOST -w /drone goharbor/harbor-e2e-engine:2.6.3 \
+            python -u -m robot -v DOCKER_USER:${{ secrets.DOCKER_USER }} -v DOCKER_PWD:${{ secrets.DOCKER_TOKEN }} -v ip:$CORE_HOST -v ip1: -v HARBOR_PASSWORD:Harbor12345 -X \
+            --exclude gc --exclude metrics --exclude singularity --exclude proxy_cache \
+            /drone/tests/robot-cases/Group1-Nightly/Setup.robot /drone/tests/robot-cases/Group0-BAT/API_DB.robot
+          free -m
+          df -h
+
+      - name: fetch harbor logs
+        if: ${{ failure() }}
+        run: |
+          df -h
+          free -m
+          mkdir -p /tmp/harbor
+          for name in core jobservice registry registryctl trivy chartmuseum notaryserver notarysigner portal; do \
+            kubectl -n cluster-sample-ns logs -l "goharbor.io/operator-controller=$name" --all-containers > /tmp/harbor/$name.log ; \
+          done
+          kubectl -n cluster-sample-ns logs -l "application=spilo" --all-containers > /tmp/harbor/db.log
+          kubectl -n cluster-sample-ns logs -l "app.kubernetes.io/component=redis" --all-containers > /tmp/harbor/redis.log
+
+      - uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          name: contour_harbor_v${{ matrix.k8sVersion }}_v${{ matrix.certManager }}_${{ matrix.samples }}
+          path: /tmp/harbor
+
+      - name: fetch logs
+        if: ${{ failure() }}
+        run: |
+          mkdir  -p /tmp/logs
+          kind export logs --name harbor /tmp/logs
+
+      - uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          name: contour_kind_v${{ matrix.k8sVersion }}_v${{ matrix.certManager }}_${{ matrix.samples }}
+          path: /tmp/logs
+
+      - name: Get logs for debug
+        if: ${{ failure() }}
+        run: |
+          set -x
+          kubectl get all -n "${operatorNamespace}" -o wide
+          kubectl logs -n "${operatorNamespace}" -l 'control-plane=harbor-operator' --all-containers --tail=1000
+
   chart-tests:
     runs-on: ubuntu-latest
     name: chart K8S v${{ matrix.k8sVersion }} (CM v${{ matrix.certManager }})
@@ -360,7 +572,7 @@ jobs:
 
         # https://snapcraft.io/microk8s
         k8sVersion:
-          - "1.19.1"
+          - "1.19.7"
 
         ingress:
           - "0.35.0"
@@ -393,7 +605,7 @@ jobs:
       - name: Install Kubernetes v${{ matrix.k8sVersion }}
         uses: helm/kind-action@v1.1.0
         with:
-          version: v0.9.0
+          version: v0.10.0
           node_image: kindest/node:v${{ matrix.k8sVersion }}
           cluster_name: harbor
           config: .github/kind.yaml

apis/goharbor.io/v1alpha3/harbor_types.go

@@ -714,8 +714,6 @@ type HarborExposeIngressSpec struct {
 
 	// +kubebuilder:validation:Optional
 	// +kubebuilder:default="default"
-	// +kubebuilder:validation:Type="string"
-	// +kubebuilder:validation:Enum={"gce","ncp","default"}
 	// Set to the type of ingress controller.
 	Controller harbormetav1.IngressController `json:"controller,omitempty"`
 

apis/meta/v1alpha1/ingress_controller.go

@@ -1,7 +1,7 @@
 package v1alpha1
 
 // +kubebuilder:validation:Type=string
-// +kubebuilder:validation:Enum={"default","gce","ncp"}
+// +kubebuilder:validation:Enum={"default","gce","ncp","contour"}
 // Type of ingress controller if it has specific requirements.
 type IngressController string
 
@@ -12,4 +12,8 @@ const (
 	IngressControllerGCE IngressController = "gce"
 	// NSX-T Container Plugin ingress controller.
 	IngressControllerNCP IngressController = "ncp"
+	// Contour ingress controller.
+	IngressControllerContour IngressController = "contour"
+	// ingress-controller name.
+	IngressControllerAnnotationName = "goharbor.io/ingress-controller"
 )

controllers/goharbor/core/services.go

@@ -13,6 +13,7 @@ import (
 func (r *Reconciler) GetService(ctx context.Context, core *goharborv1.Core) (*corev1.Service, error) {
 	name := r.NormalizeName(ctx, core.GetName())
 	namespace := core.GetNamespace()
+	annotations := map[string]string{}
 
 	var ports []corev1.ServicePort
 
@@ -23,6 +24,10 @@ func (r *Reconciler) GetService(ctx context.Context, core *goharborv1.Core) (*co
 			TargetPort: intstr.FromString(harbormetav1.CoreHTTPSPortName),
 			Protocol:   corev1.ProtocolTCP,
 		})
+
+		if v, ok := core.Annotations[harbormetav1.IngressControllerAnnotationName]; ok && v == string(harbormetav1.IngressControllerContour) {
+			annotations["projectcontour.io/upstream-protocol.tls"] = harbormetav1.PortalHTTPSPortName
+		}
 	} else {
 		ports = append(ports, corev1.ServicePort{
 			Name:       harbormetav1.CoreHTTPPortName,
@@ -45,7 +50,7 @@ func (r *Reconciler) GetService(ctx context.Context, core *goharborv1.Core) (*co
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        name,
 			Namespace:   namespace,
-			Annotations: core.Spec.Metrics.AddPrometheusAnnotations(nil),
+			Annotations: core.Spec.Metrics.AddPrometheusAnnotations(annotations),
 		},
 		Spec: corev1.ServiceSpec{
 			Ports: ports,

controllers/goharbor/harbor/core.go

@@ -406,13 +406,19 @@ func (r *Reconciler) GetCore(ctx context.Context, harbor *goharborv1.Harbor) (*g
 		return nil, errors.Wrap(err, "cannot get database configuration")
 	}
 
+	annotation := map[string]string{
+		harbormetav1.NetworkPoliciesAnnotationName: harbormetav1.NetworkPoliciesAnnotationDisabled,
+	}
+
+	if harbor.Spec.Expose.Core.Ingress != nil {
+		annotation[harbormetav1.IngressControllerAnnotationName] = string(harbor.Spec.Expose.Core.Ingress.Controller)
+	}
+
 	return &goharborv1.Core{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      name,
-			Namespace: namespace,
-			Annotations: version.SetVersion(map[string]string{
-				harbormetav1.NetworkPoliciesAnnotationName: harbormetav1.NetworkPoliciesAnnotationDisabled,
-			}, harbor.Spec.Version),
+			Name:        name,
+			Namespace:   namespace,
+			Annotations: version.SetVersion(annotation, harbor.Spec.Version),
 		},
 		Spec: goharborv1.CoreSpec{
 			ComponentSpec: harbor.GetComponentSpec(ctx, harbormetav1.CoreComponent),