Replace skopeo (#1322)

* Updated dependencies
Signed-off-by: Tomasz Baranski <tbaransk@redhat.com>

* Replace skopeo with containers API.

This commit removes dependence on skopeo (binary) and uses containers
API. By doing that we're able to opimize the use of storage (scratch)
space, storage I/O and download bandwith.

Signed-off-by: Tomasz Baranski <tbaransk@redhat.com>

* Fixing rebase - dependencies kerfuffle.

Signed-off-by: Tomasz Baranski <tbaransk@redhat.com>

* Handling docker-format images as well as OCI.

Signed-off-by: Tomasz Baranski <tbaransk@redhat.com>

* Fix for missing code-generator module.

Signed-off-by: Tomasz Baranski <tbaransk@redhat.com>

* Remove regex, image file in registry images are matched by a path
prefix.

Signed-off-by: Tomasz Baranski <tbaransk@redhat.com>

* Added nginx proxy in front of docker registry for a rate-limited access.

Signed-off-by: Tomasz Baranski <tbaransk@redhat.com>

WORKSPACE

@@ -230,15 +230,6 @@ http_file(
     ],
 )
 
-http_file(
-    name = "skopeo",
-    sha256 = "9ae93d78a41face16d842c4da4ffc07bc8b119fbcd23d436b41a44a5643d4dc0",
-    urls = [
-        "http://download.fedoraproject.org/pub/fedora/linux/updates/31/Everything/x86_64/Packages/s/skopeo-0.1.41-1.fc31.x86_64.rpm",
-        "https://storage.googleapis.com/builddeps/9ae93d78a41face16d842c4da4ffc07bc8b119fbcd23d436b41a44a5643d4dc0",
-    ],
-)
-
 http_file(
     name = "ostree-libs",
     sha256 = "4011ad8b367db9d528d47202d07c287a958d4bd11a56b11618818dcb3be55bc6",
@@ -272,6 +263,14 @@ http_file(
     ],
 )
 
+http_file(
+    name = "device-mapper-libs",
+    sha256 = "0ebd37bcd6d2beb5692b7c7e3d94b90a26d45b059696d954b502d85d738b7732",
+    urls = [
+        "http://download.fedoraproject.org/pub/fedora/linux/releases/31/Everything/x86_64/os/Packages/d/device-mapper-libs-1.02.163-2.fc31.x86_64.rpm",
+    ],
+)
+
 http_file(
     name = "device-mapper-event",
     sha256 = "9dfb6c534d23d3058d83dfaf669544b58318c34351ae46e7341cdeee51be2ab8",

cmd/cdi-importer/BUILD.bazel

@@ -11,7 +11,6 @@ rpm_image(
         "@xen-libs//file",
         "@libaio//file",
         "@capstone//file",
-        "@skopeo//file",
         "@ostree-libs//file",
         "@containers-common//file",
     ],

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/RHsyseng/operator-utils v0.0.0-20190906175225-942a3f9c85a9
 	github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30
 	github.com/blang/semver v3.5.1+incompatible
+	github.com/containers/image/v5 v5.5.1
 	github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
 	github.com/elazarl/goproxy v0.0.0-20190911111923-ecfe977594f1 // indirect
 	github.com/emicklei/go-restful v2.9.6+incompatible
@@ -29,15 +30,15 @@ require (
 	github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190725173916-b56e63a643cc
 	github.com/operator-framework/operator-marketplace v0.0.0-20190617165322-1cbd32624349
 	github.com/ovirt/go-ovirt v4.3.4+incompatible
-	github.com/pkg/errors v0.8.1
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.1.0
 	github.com/prometheus/client_model v0.2.0
 	github.com/rs/cors v1.7.0
 	github.com/smartystreets/assertions v1.0.1 // indirect
 	github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
-	github.com/ulikunitz/xz v0.5.6
+	github.com/ulikunitz/xz v0.5.7
 	go.uber.org/multierr v1.3.0 // indirect
-	golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd
+	golang.org/x/sys v0.0.0-20200519105757-fe76b779f299
 	gopkg.in/fsnotify.v1 v1.4.7
 	gopkg.in/ini.v1 v1.48.0 // indirect
 	gopkg.in/square/go-jose.v2 v2.3.1

manifests/templates/file-host.yaml.in

@@ -110,3 +110,6 @@ spec:
   - name: tls
     port: 443
     targetPort: 443
+  - name: rate-limit-registry-proxy
+    port: 83
+    targetPort: 83

pkg/image/BUILD.bazel

@@ -5,7 +5,7 @@ go_library(
     srcs = [
         "filefmt.go",
         "qemu.go",
-        "skopeo.go",
+        "transport.go",
         "validate.go",
     ],
     importpath = "kubevirt.io/containerized-data-importer/pkg/image",
@@ -14,6 +14,10 @@ go_library(
         "//pkg/common:go_default_library",
         "//pkg/system:go_default_library",
         "//pkg/util:go_default_library",
+        "//vendor/github.com/containers/image/v5/image:go_default_library",
+        "//vendor/github.com/containers/image/v5/pkg/blobinfocache:go_default_library",
+        "//vendor/github.com/containers/image/v5/transports/alltransports:go_default_library",
+        "//vendor/github.com/containers/image/v5/types:go_default_library",
         "//vendor/github.com/pkg/errors:go_default_library",
         "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/github.com/prometheus/client_model/go:go_default_library",
@@ -28,12 +32,11 @@ go_test(
         "filefmt_test.go",
         "qemu_suite_test.go",
         "qemu_test.go",
-        "skopeo_test.go",
+        "transport_test.go",
     ],
     embed = [":go_default_library"],
     deps = [
         "//pkg/system:go_default_library",
-        "//pkg/util:go_default_library",
         "//tests/reporters:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/ginkgo/extensions/table:go_default_library",