Can't skip tls cert verify when fetching blob from external urls

[shabbywu]

Description

I have write a container image distribution client which will fill urls into manifest.

When i push some image via my distribution client to my self-signed image registry, containerd can not pull the image down.
And the stdout is following:

root@kind-control-plane:/etc/containerd# ctr --debug image pull --hosts-dir /etc/containerd/certs.d *********************/namespace:3.31.0
DEBU[0000] fetching                                      image="*********************/namespace:3.31.0"
DEBU[0000] loading host directory                        dir=/etc/containerd/certs.d/*********************
DEBU[0000] resolving                                     host=*********************
DEBU[0000] do request                                    host=********************* request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, */*" request.header.user-agent=containerd/v1.7.1 request.method=HEAD url="https://*********************/v2/namespace/manifests/3.31.0"
DEBU[0000] fetch response received                       host=********************* response.header.accept-ranges=bytes response.header.cache-control=no-cache response.header.connection=keep-alive response.header.content-length=5994 response.header.content-range="bytes 0-5993/5994" response.header.content-type="application/vnd.docker.distribution.manifest.v2+json; charset=UTF-8" response.header.date="Thu, 11 Apr 2024 07:52:59 GMT" response.header.docker-content-digest="sha256:8dd17dbabe21905fdf332f6b459041fb101e8146321a9bb70dab3c47fdc1c75f" response.header.docker-distribution-api-version=registry/2.0 response.header.etag=8dd17dbabe21905fdf332f6b459041fb101e8146321a9bb70dab3c47fdc1c75f response.header.last-modified="Wed, 10 Apr 2024 10:24:13 GMT" response.header.strict-transport-security="max-age=15724800; includeSubDomains" response.header.x-checksum-md5=45be8eac1bf2655bbe07071f872d6c5c response.header.x-checksum-sha256=8dd17dbabe21905fdf332f6b459041fb101e8146321a9bb70dab3c47fdc1c75f response.status="200 OK" url="https://*********************/v2/namespace/manifests/3.31.0"
DEBU[0000] resolved                                      desc.digest="sha256:8dd17dbabe21905fdf332f6b459041fb101e8146321a9bb70dab3c47fdc1c75f" host=*********************
DEBU[0000] loading host directory                        dir=/etc/containerd/certs.d/*********************
DEBU[0000] fetch                                         digest="sha256:8dd17dbabe21905fdf332f6b459041fb101e8146321a9bb70dab3c47fdc1c75f" mediatype=application/vnd.docker.distribution.manifest.v2+json size=5994
DEBU[0000] do request                                    digest="sha256:8dd17dbabe21905fdf332f6b459041fb101e8146321a9bb70dab3c47fdc1c75f" mediatype=application/vnd.docker.distribution.manifest.v2+json request.header.accept="application/vnd.docker.distribution.manifest.v2+json, */*" request.header.user-agent=containerd/v1.7.1 request.method=GET size=5994 url="https://*********************/v2/namespace/manifests/sha256:8dd17dbabe21905fdf332f6b459041fb101e8146321a9bb70dab3c47fdc1c75f"
DEBU[0000] fetch response received                       digest="sha256:8dd17dbabe21905fdf332f6b459041fb101e8146321a9bb70dab3c47fdc1c75f" mediatype=application/vnd.docker.distribution.manifest.v2+json response.header.accept-ranges=bytes response.header.cache-control=no-cache response.header.connection=keep-alive response.header.content-length=5994 response.header.content-range="bytes 0-5993/5994" response.header.content-type="application/vnd.docker.distribution.manifest.v2+json; charset=UTF-8" response.header.date="Thu, 11 Apr 2024 07:52:59 GMT" response.header.docker-content-digest="sha256:8dd17dbabe21905fdf332f6b459041fb101e8146321a9bb70dab3c47fdc1c75f" response.header.docker-distribution-api-version=registry/2.0 response.header.etag=8dd17dbabe21905fdf332f6b459041fb101e8146321a9bb70dab3c47fdc1c75f response.header.last-modified="Wed, 10 Apr 2024 10:24:13 GMT" response.header.strict-transport-security="max-age=15724800; includeSubDomains" response.header.x-checksum-md5=45be8eac1bf2655bbe07071f872d6c5c response.header.x-checksum-sha256=8dd17dbabe21905fdf332f6b459041fb101e8146321a9bb70dab3c47fdc1c75f response.status="200 OK" size=5994 url="https://*********************/v2/namespace/manifests/sha256:8dd17dbabe21905fdf332f6b459041fb101e8146321a9bb70dab3c47fdc1c75f"
DEBU[0000] fetch                                         digest="sha256:5bd8be99bc69145407f1d426b687c47b12de79835ceb8fd600e74f5daebc3e1e" mediatype=application/vnd.docker.container.image.v1+json size=6213
DEBU[0000] fetch                                         digest="sha256:af6abec2f064ed746fde7fefa8a8f8b80f69eba381101234abaa2bb83599f8a2" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=224
DEBU[0000] fetch                                         digest="sha256:d63a0b160b04d073c89627a88d58fc4c9c6a716bec67efc25e76ff4a8987640f" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=461
DEBU[0000] fetch                                         digest="sha256:80834f702dcc21037d782c4527d13e740492ddf5c00a9868c46f879efc847bc8" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=34243
DEBU[0000] fetch                                         digest="sha256:046daf81a49b312c2d6362fede63a720e726468f197f1e0fe281d234beda9e37" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=1271
DEBU[0000] fetch                                         digest="sha256:8c8c59a8fafad7fe6cec01abcaa66e1fd772f076da7decd39b41f7ba131186d1" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=96
DEBU[0000] fetch                                         digest="sha256:7b3919ea94f783745160c32683d774e95a16737472b52aab2b1903a6b315f3b1" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=157885177
DEBU[0000] fetch                                         digest="sha256:661497d950bbd018c4b678233682fdca881c0a2c10dbcd8ac54e2a155290ee27" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=5042
DEBU[0000] fetch                                         digest="sha256:aba2e8037cba9d246762be254afeb9bd119b7700a6cac8e2a8855ee37305eb01" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=17658232
DEBU[0000] fetch                                         digest="sha256:7ddb23a34c5e77c701fda27a743655948b4a479bf010413e69c54995722f2705" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=14839299
DEBU[0000] fetch                                         digest="sha256:f1d0528d1d6966721084e61e3300c3151d33cf65ba34561da567b8466f2fc904" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=121229691
DEBU[0000] fetch                                         digest="sha256:5c1e9661fa5abfb08fa15fa3845ba4adacf9036265f8795bedc7c83297589483" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=126
DEBU[0000] fetch                                         digest="sha256:eca9c5cfae59e508e147f7e619f8294b039d7547ad3f5d49869f2b649a813ff7" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=159
DEBU[0000] fetch                                         digest="sha256:36ef7476b252bb358d6dda5be8f001b8d6ecf46de156195e328b2410f5d045e7" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=93
DEBU[0000] fetch                                         digest="sha256:810def1e168ba01cbdce3d17d835d40ba80f9b9c26679c8c2dc9ae98dfe1ca33" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=25665756
DEBU[0000] fetch                                         digest="sha256:3a6254348ab9ca08e67affc3b8c502dc9d5c5f8e8b1b070a98f89805d69919d5" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=815
INFO[0000] request                                       digest="sha256:5bd8be99bc69145407f1d426b687c47b12de79835ceb8fd600e74f5daebc3e1e" mediatype=application/vnd.docker.container.image.v1+json size=6213 url="https://*********************/:443/v2/namespace/blobs/sha256:5bd8be99bc69145407f1d426b687c47b12de79835ceb8fd600e74f5daebc3e1e"
DEBU[0000] do request                                    digest="sha256:5bd8be99bc69145407f1d426b687c47b12de79835ceb8fd600e74f5daebc3e1e" mediatype=application/vnd.docker.container.image.v1+json request.header.accept="application/vnd.docker.container.image.v1+json, */*" request.header.user-agent=containerd/v1.7.1 request.method=GET size=6213 url="https://*********************/:443/v2/namespace/blobs/sha256:5bd8be99bc69145407f1d426b687c47b12de79835ceb8fd600e74f5daebc3e1e"
INFO[0000] request                                       digest="sha256:eca9c5cfae59e508e147f7e619f8294b039d7547ad3f5d49869f2b649a813ff7" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=159 url="https://*********************/:443/v2/namespace/blobs/sha256:eca9c5cfae59e508e147f7e619f8294b039d7547ad3f5d49869f2b649a813ff7"
DEBU[0000] do request                                    digest="sha256:eca9c5cfae59e508e147f7e619f8294b039d7547ad3f5d49869f2b649a813ff7" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/v1.7.1 request.method=GET size=159 url="https://*********************/:443/v2/namespace/blobs/sha256:eca9c5cfae59e508e147f7e619f8294b039d7547ad3f5d49869f2b649a813ff7"
INFO[0000] request                                       digest="sha256:8c8c59a8fafad7fe6cec01abcaa66e1fd772f076da7decd39b41f7ba131186d1" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=96 url="https://*********************/:443/v2/namespace/blobs/sha256:8c8c59a8fafad7fe6cec01abcaa66e1fd772f076da7decd39b41f7ba131186d1"
DEBU[0000] do request                                    digest="sha256:8c8c59a8fafad7fe6cec01abcaa66e1fd772f076da7decd39b41f7ba131186d1" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/v1.7.1 request.method=GET size=96 url="https://*********************/:443/v2/namespace/blobs/sha256:8c8c59a8fafad7fe6cec01abcaa66e1fd772f076da7decd39b41f7ba131186d1"
INFO[0000] request                                       digest="sha256:80834f702dcc21037d782c4527d13e740492ddf5c00a9868c46f879efc847bc8" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip size=34243 url="https://*********************/:443/v2/namespace/blobs/sha256:80834f702dcc21037d782c4527d13e740492ddf5c00a9868c46f879efc847bc8"
DEBU[0000] do request                                    digest="sha256:80834f702dcc21037d782c4527d13e740492ddf5c00a9868c46f879efc847bc8" mediatype=application/vnd.docker.image.rootfs.diff.tar.gzip request.header.accept="application/vnd.docker.image.rootfs.diff.tar.gzip, */*" request.header.user-agent=containerd/v1.7.1 request.method=GET size=34243 url="https://*********************/:443/v2/namespace/blobs/sha256:80834f702dcc21037d782c4527d13e740492ddf5c00a9868c46f879efc847bc8"
ctr: failed to copy: httpReadSeeker: failed open: failed to do request: Get "https://*********************/:443/v2/namespace/blobs/sha256:5bd8be99bc69145407f1d426b687c47b12de79835ceb8fd600e74f5daebc3e1e": tls: failed to verify certificate: x509: certificate is valid for ingress.local, not *********************

As we can see, ctr report an error x509 when httpReadSeeker is failed to open a remote blob file from external url.
And finally, I found that the code should be blame by https://github.com/containerd/containerd/blob/main/core/remotes/docker/fetcher.go#L73, where the host is not provided by https://github.com/containerd/containerd/blob/main/core/remotes/docker/config/hosts.go#L74.

Steps to reproduce the issue

1. Put an image manifest with external urls which pointed to a self-signed cert host
2. pull image via containerd

Describe the results you received and expected

External urls fetcher should follow registry tls verify skip configuration.

What version of containerd are you using?

containerd github.com/containerd/containerd v1.7.1 1677a17

Any other relevant information

The Image Manifest look like:

Show configuration if it is related to CRI plugin.

version = 2

[plugins."io.containerd.grpc.v1.cri".registry]
 config_path = "/etc/containerd/certs.d"

[plugins]
  [plugins."io.containerd.grpc.v1.cri"]
    restrict_oom_score_adj = false
    sandbox_image = "registry.k8s.io/pause:3.7"
    tolerate_missing_hugepages_controller = true
    [plugins."io.containerd.grpc.v1.cri".containerd]
      default_runtime_name = "runc"
      discard_unpacked_layers = true
      snapshotter = "overlayfs"
      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
          base_runtime_spec = "/etc/containerd/cri-base.json"
          runtime_type = "io.containerd.runc.v2"
          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
            SystemdCgroup = false
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler]
          base_runtime_spec = "/etc/containerd/cri-base.json"
          runtime_type = "io.containerd.runc.v2"
          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler.options]
            SystemdCgroup = false

[proxy_plugins]
  [proxy_plugins.fuse-overlayfs]
    address = "/run/containerd-fuse-overlayfs.sock"
    type = "snapshot"
[debug]
  level = "trace"