containerd does not honor cert.d configured mirrors

[atlasloewenherz]

Description

versions

containerd: containerd github.com/containerd/containerd 1.5.9-0ubuntu3.1

ubuntu: 22.04

configuration:

root@gitlab:/etc/containerd# cat config.toml | egrep -v "(^#.*|^$)"
version = 2
plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options.SystemdCgroup = "true"
plugins."io.containerd.grpc.v1.cri".registry.config_path = "/etc/containerd/certs.d"
root@gitlab:/etc/containerd#

contents of /etc/containerd/certs.d

root@gitlab:~# tree /etc/containerd/certs.d/
/etc/containerd/certs.d/
├── _default
│   └── hosts.toml
└── docker.io
    └── hosts.toml

2 directories, 2 files
root@gitlab:~#

_defaul/hosts.toml

root@gitlab:~# cat /etc/containerd/certs.d/_default/hosts.toml
server = "*"
[host."https://mirror.local.tld:443/v2"]
  capabilities = ["pull", "resolve"]
root@gitlab:~#

docker.io/hosts

root@gitlab:~# cat /etc/containerd/certs.d/docker.io/hosts.toml
server = "https://registry-1.docker.io"
[host."https://mirror.local.tld:443/v2"]
  capabilities = ["pull", "resolve"]
root@gitlab:~#

configuration dump:

disabled_plugins = []
imports = ["/etc/containerd/config.toml"]
oom_score = 0
plugin_dir = ""
required_plugins = []
root = "/var/lib/containerd"
state = "/run/containerd"
version = 2

[cgroup]
  path = ""

[debug]
  address = ""
  format = ""
  gid = 0
  level = ""
  uid = 0

[grpc]
  address = "/run/containerd/containerd.sock"
  gid = 0
  max_recv_message_size = 16777216
  max_send_message_size = 16777216
  tcp_address = ""
  tcp_tls_cert = ""
  tcp_tls_key = ""
  uid = 0

[metrics]
  address = ""
  grpc_histogram = false

[plugins]

  [plugins."io.containerd.gc.v1.scheduler"]
    deletion_threshold = 0
    mutation_threshold = 100
    pause_threshold = 0.02
    schedule_delay = "0s"
    startup_delay = "100ms"

  [plugins."io.containerd.grpc.v1.cri"]
    disable_apparmor = false
    disable_cgroup = false
    disable_hugetlb_controller = true
    disable_proc_mount = false
    disable_tcp_service = true
    enable_selinux = false
    enable_tls_streaming = false
    ignore_image_defined_volumes = false
    max_concurrent_downloads = 3
    max_container_log_line_size = 16384
    netns_mounts_under_state_dir = false
    restrict_oom_score_adj = false
    sandbox_image = "k8s.gcr.io/pause:3.5"
    selinux_category_range = 1024
    stats_collect_period = 10
    stream_idle_timeout = "4h0m0s"
    stream_server_address = "127.0.0.1"
    stream_server_port = "0"
    systemd_cgroup = false
    tolerate_missing_hugetlb_controller = true
    unset_seccomp_profile = ""

    [plugins."io.containerd.grpc.v1.cri".cni]
      bin_dir = "/opt/cni/bin"
      conf_dir = "/etc/cni/net.d"
      conf_template = ""
      max_conf_num = 1
    [plugins."io.containerd.grpc.v1.cri".containerd]
      default_runtime_name = "runc"
      disable_snapshot_annotations = true
      discard_unpacked_layers = false
      no_pivot = false
      snapshotter = "overlayfs"

      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
        base_runtime_spec = ""
        container_annotations = []
        pod_annotations = []
        privileged_without_host_devices = false
        runtime_engine = ""
        runtime_root = ""
        runtime_type = ""

        [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]

      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]

        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
          base_runtime_spec = ""
          container_annotations = []
          pod_annotations = []
          privileged_without_host_devices = false
          runtime_engine = ""
          runtime_root = ""
          runtime_type = ""

          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
            SystemdCgroup = "true"

      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
        base_runtime_spec = ""
        container_annotations = []
        pod_annotations = []
        privileged_without_host_devices = false
        runtime_engine = ""
        runtime_root = ""
        runtime_type = ""

        [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options]

    [plugins."io.containerd.grpc.v1.cri".image_decryption]
      key_model = "node"

    [plugins."io.containerd.grpc.v1.cri".registry]
      config_path = "/etc/containerd/certs.d"

      [plugins."io.containerd.grpc.v1.cri".registry.auths]

      [plugins."io.containerd.grpc.v1.cri".registry.configs]

      [plugins."io.containerd.grpc.v1.cri".registry.headers]

      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]

    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
      tls_cert_file = ""
      tls_key_file = ""

  [plugins."io.containerd.internal.v1.opt"]
    path = "/opt/containerd"

  [plugins."io.containerd.internal.v1.restart"]
    interval = "10s"

  [plugins."io.containerd.metadata.v1.bolt"]
    content_sharing_policy = "shared"

  [plugins."io.containerd.monitor.v1.cgroups"]
    no_prometheus = false

  [plugins."io.containerd.runtime.v1.linux"]
    no_shim = false
    runtime = "runc"
    runtime_root = ""
    shim = "containerd-shim"
    shim_debug = false

  [plugins."io.containerd.runtime.v2.task"]
    platforms = ["linux/amd64"]

  [plugins."io.containerd.service.v1.diff-service"]
    default = ["walking"]

  [plugins."io.containerd.snapshotter.v1.aufs"]
    root_path = ""

  [plugins."io.containerd.snapshotter.v1.btrfs"]
    root_path = ""

  [plugins."io.containerd.snapshotter.v1.devmapper"]
    async_remove = false
    base_image_size = ""
    pool_name = ""
    root_path = ""

  [plugins."io.containerd.snapshotter.v1.native"]
    root_path = ""

  [plugins."io.containerd.snapshotter.v1.overlayfs"]
    root_path = ""

  [plugins."io.containerd.snapshotter.v1.zfs"]
    root_path = ""

[proxy_plugins]

[stream_processors]

  [stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
    accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
    args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
    env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
    path = "ctd-decoder"
    returns = "application/vnd.oci.image.layer.v1.tar"

  [stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]
    accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
    args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
    env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
    path = "ctd-decoder"
    returns = "application/vnd.oci.image.layer.v1.tar+gzip"

[timeouts]
  "io.containerd.timeout.shim.cleanup" = "5s"
  "io.containerd.timeout.shim.load" = "5s"
  "io.containerd.timeout.shim.shutdown" = "3s"
  "io.containerd.timeout.task.state" = "2s"

[ttrpc]
  address = ""
  gid = 0
  uid = 0´´´

systemctl restart containerd
============================

```bash
Feb  5 22:10:14 gitlab containerd[20988]: time="2023-02-05T22:10:14.340853181Z" level=info msg="Start cri plugin with config {PluginConfig:{ContainerdConfig:{Snapshotter:overlayfs DefaultRuntimeName:runc DefaultRuntime:{Type: Engine: PodAnnotations:[] ContainerAnnotations:[] Root: Options:map[] PrivilegedWithoutHostDevices:false BaseRuntimeSpec:} UntrustedWorkloadRuntime:{Type: Engine: PodAnnotations:[] ContainerAnnotations:[] Root: Options:map[] PrivilegedWithoutHostDevices:false BaseRuntimeSpec:} Runtimes:map[runc:{Type:io.containerd.runc.v2 Engine: PodAnnotations:[] ContainerAnnotations:[] Root: Options:map[BinaryName: CriuImagePath: CriuPath: CriuWorkPath: IoGid:0 IoUid:0 NoNewKeyring:false NoPivotRoot:false Root: ShimCgroup: SystemdCgroup:false] PrivilegedWithoutHostDevices:false BaseRuntimeSpec:}] NoPivot:false DisableSnapshotAnnotations:true DiscardUnpackedLayers:false} CniConfig:{NetworkPluginBinDir:/opt/cni/bin NetworkPluginConfDir:/etc/cni/net.d NetworkPluginMaxConfNum:1 NetworkPluginConfTemplate:} Registry:{ConfigPath: Mirrors:map[] Configs:map[] Auths:map[] Headers:map[]} ImageDecryption:{KeyModel:node} DisableTCPService:true StreamServerAddress:127.0.0.1 StreamServerPort:0 StreamIdleTimeout:4h0m0s EnableSelinux:false SelinuxCategoryRange:1024 SandboxImage:k8s.gcr.io/pause:3.5 StatsCollectPeriod:10 SystemdCgroup:false EnableTLSStreaming:false X509KeyPairStreaming:{TLSCertFile: TLSKeyFile:} MaxContainerLogLineSize:16384 DisableCgroup:false DisableApparmor:false RestrictOOMScoreAdj:false MaxConcurrentDownloads:3 DisableProcMount:false UnsetSeccompProfile: TolerateMissingHugetlbController:true DisableHugetlbController:true IgnoreImageDefinedVolumes:false NetNSMountsUnderStateDir:false} ContainerdRootDir:/var/lib/containerd ContainerdEndpoint:/run/containerd/containerd.sock RootDir:/var/lib/containerd/io.containerd.grpc.v1.cri StateDir:/run/containerd/io.containerd.grpc.v1.cri}"

snippets from the /var/log/syslog

####
Registry:{ConfigPath: Mirrors:map[] Configs:map[] Auths:map[] Headers:map[]}
#####

Steps to reproduce the issue

1. install configure according to the docs
2. restart the containerd
3. pull an image

Describe the results you received and expected

normally pulling an image would use the configured mirrors but this is not happening

What version of containerd are you using?

containerd: containerd github.com/containerd/containerd 1.5.9-0ubuntu3.1

Any other relevant information

ubuntu: 22.04

root@gitlab:~# runc -v
runc version 1.1.0-0ubuntu1.1
spec: 1.0.2-dev
go: go1.18.1
libseccomp: 2.5.3

uname -a

Linux gitlab 5.15.0-1026-kvm #31-Ubuntu SMP Fri Jan 6 16:06:08 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux

Show configuration if it is related to CRI plugin.

root@gitlab:/etc/containerd# cat config.toml | egrep -v "(^#.*|^$)"
version = 2
plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options.SystemdCgroup = "true"
plugins."io.containerd.grpc.v1.cri".registry.config_path = "/etc/containerd/certs.d"
root@gitlab:/etc/containerd#

[dmcgowan]

Is the mirror configuration working for docker.io? The _default directory has not been backported to the 1.5 release.

[atlasloewenherz]

@dmcgowan thanks for your feedback!

Unfortunately none of the mirror's configuration is working 😕. as you pointed out that the "_default" approache is not yet backported i will remove it from my configuration and leave it here to not modify the initial question.

Thanks

[kettlereve]

I can confirm this problem. None of the configured mirror are working, I am running an AKS 1.24.9 cluster. Any news on this topic? @dmcgowan

[dmcgowan]

You can reach out to AKS support. Check the logs for ConfigPath being set correctly and an INFO level log like

INFO[2023-04-23T22:22:10.765739499-07:00] trying next host                              error="failed to do request: Head \"https://mirror.local.tld:443/v2/library/alpine/manifests/latest?ns=docker.io\": dial tcp: lookup mirror.local.tld: no such host" host="mirror.local.tld:443"

[Sathyam-Unicommerce]

Facing the same issue, the configured server in the hosts.toml are not getting picked up. I passed a host as http server and getting the following error: http: server gave HTTP response to HTTPS client

[dmcgowan]

If the ConfigPath is set correctly and being picked up, check the logs for the reason why a host is not being used.

[dmcgowan]

@atlasloewenherz Registry:{ConfigPath: Mirrors:map[] Configs:map[] Auths:map[] Headers:map[]} is not expected, you should see Registry:{ConfigPath:/etc/containerd/certs.d Mirrors:map[] Configs:map[] Auths:map[] Headers:map[]} if containerd is loading the right config. It does require a restart of containerd to update the ConfigPath. Once ConfigPath is set, then the hosts.toml can be updated without restarting.