diff --git a/pkg/harbor/model.go b/pkg/harbor/model.go
index 8c4a117f..65357dbf 100644
--- a/pkg/harbor/model.go
+++ b/pkg/harbor/model.go
@@ -6,8 +6,6 @@ import (
 	"fmt"
 	"net/url"
 	"time"
-
-	"golang.org/x/xerrors"
 )
 
 // Severity represents the severity of a image/component in terms of vulnerability.
@@ -83,9 +81,19 @@ type ScanRequest struct {
 func (c ScanRequest) GetImageRef() (imageRef string, insecureRegistry bool, err error) {
 	registryURL, err := url.Parse(c.Registry.URL)
 	if err != nil {
-		return imageRef, insecureRegistry, xerrors.Errorf("parsing registry URL: %w", err)
+		err = fmt.Errorf("parsing registry URL: %w", err)
+		return
+	}
+
+	port := registryURL.Port()
+	if port == "" && registryURL.Scheme == "http" {
+		port = "80"
 	}
-	imageRef = fmt.Sprintf("%s/%s@%s", registryURL.Host, c.Artifact.Repository, c.Artifact.Digest)
+	if port == "" && registryURL.Scheme == "https" {
+		port = "443"
+	}
+
+	imageRef = fmt.Sprintf("%s:%s/%s@%s", registryURL.Hostname(), port, c.Artifact.Repository, c.Artifact.Digest)
 	insecureRegistry = "http" == registryURL.Scheme
 	return
 }
diff --git a/pkg/harbor/model_test.go b/pkg/harbor/model_test.go
index 55536b3f..5946018b 100644
--- a/pkg/harbor/model_test.go
+++ b/pkg/harbor/model_test.go
@@ -15,48 +15,63 @@ func TestScanRequest_GetImageRef(t *testing.T) {
 		expectedError    string
 	}{
 		{
-			name: "mongo",
+			name: "Should get imageRef when URL scheme is HTTP and port is not specified",
 			request: ScanRequest{
 				Registry: Registry{
-					URL: "https://core.harbor.domain",
+					URL: "http://core.harbor.domain",
 				},
 				Artifact: Artifact{
 					Repository: "library/mongo",
 					Digest:     "test:ABC",
 				},
 			},
-			expectedImageRef: "core.harbor.domain/library/mongo@test:ABC",
-			expectedInsecure: false,
+			expectedImageRef: "core.harbor.domain:80/library/mongo@test:ABC",
+			expectedInsecure: true,
 		},
 		{
-			name: "nginx",
+			name: "Should get imageRef when URL scheme is HTTP and port is specified",
 			request: ScanRequest{
 				Registry: Registry{
-					URL: "https://core.harbor.domain:443",
+					URL: "http://harbor-harbor-registry:5000",
 				},
-				Artifact: Artifact{Repository: "library/nginx",
-					Digest: "test:DEF",
+				Artifact: Artifact{
+					Repository: "scanners/mongo",
+					Digest:     "test:GHI",
 				},
 			},
-			expectedImageRef: "core.harbor.domain:443/library/nginx@test:DEF",
-			expectedInsecure: false,
+			expectedImageRef: "harbor-harbor-registry:5000/scanners/mongo@test:GHI",
+			expectedInsecure: true,
 		},
 		{
-			name: "harbor",
+			name: "Should get imageRef when URL scheme is HTTPS and port is not specified",
 			request: ScanRequest{
 				Registry: Registry{
-					URL: "http://harbor-harbor-registry:5000",
+					URL: "https://core.harbor.domain",
 				},
 				Artifact: Artifact{
-					Repository: "scanners/mongo",
-					Digest:     "test:GHI",
+					Repository: "library/mongo",
+					Digest:     "test:ABC",
 				},
 			},
-			expectedImageRef: "harbor-harbor-registry:5000/scanners/mongo@test:GHI",
-			expectedInsecure: true,
+			expectedImageRef: "core.harbor.domain:443/library/mongo@test:ABC",
+			expectedInsecure: false,
+		},
+		{
+			name: "Should get imageRef when URL scheme is HTTPS and port is specified",
+			request: ScanRequest{
+				Registry: Registry{
+					URL: "https://core.harbor.domain:8443",
+				},
+				Artifact: Artifact{Repository: "library/nginx",
+					Digest: "test:DEF",
+				},
+			},
+			expectedImageRef: "core.harbor.domain:8443/library/nginx@test:DEF",
+			expectedInsecure: false,
 		},
+
 		{
-			name: "invalid registry url",
+			name: "Should return error when registry URL is invalid",
 			request: ScanRequest{
 				Registry: Registry{
 					URL: `"http://foo%bar@www.example.com/"`,
diff --git a/pkg/scan/controller_test.go b/pkg/scan/controller_test.go
index 2eb243e7..2da872d2 100644
--- a/pkg/scan/controller_test.go
+++ b/pkg/scan/controller_test.go
@@ -62,7 +62,7 @@ func TestController_Scan(t *testing.T) {
 				Method: "Scan",
 				Args: []interface{}{
 					trivy.ImageRef{
-						Name:     "core.harbor.domain/library/mongo@sha256:917f5b7f4bef1b35ee90f03033f33a81002511c1e0767fd44276d4bd9cd2fa8e",
+						Name:     "core.harbor.domain:443/library/mongo@sha256:917f5b7f4bef1b35ee90f03033f33a81002511c1e0767fd44276d4bd9cd2fa8e",
 						Auth:     trivy.RegistryAuth{Username: "user", Password: "password"},
 						Insecure: false,
 					},
@@ -109,7 +109,7 @@ func TestController_Scan(t *testing.T) {
 				Method: "Scan",
 				Args: []interface{}{
 					trivy.ImageRef{
-						Name:     "core.harbor.domain/library/mongo@sha256:917f5b7f4bef1b35ee90f03033f33a81002511c1e0767fd44276d4bd9cd2fa8e",
+						Name:     "core.harbor.domain:443/library/mongo@sha256:917f5b7f4bef1b35ee90f03033f33a81002511c1e0767fd44276d4bd9cd2fa8e",
 						Auth:     trivy.RegistryAuth{Username: "user", Password: "password"},
 						Insecure: false,
 					},