diff --git a/go.mod b/go.mod index 23662927753..a564c51d331 100644 --- a/go.mod +++ b/go.mod @@ -2,11 +2,6 @@ module github.com/containerd/nerdctl/v2 go 1.22.7 -// FIXME: -// github.com/docker/docker/pkg/sysinfo has been replaced by a fork kept under ./pkg2/sysinfo -// as Moby is not going to move to containerd v2 anytime soon or fix these transient dependencies. -// We should still move back to upstream in the future, and remove our copy. - require ( github.com/Masterminds/semver/v3 v3.3.1 github.com/Microsoft/go-winio v0.6.2 @@ -35,8 +30,8 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 github.com/cyphar/filepath-securejoin v0.4.1 github.com/distribution/reference v0.6.0 - github.com/docker/cli v27.5.1+incompatible - github.com/docker/docker v27.5.1+incompatible + github.com/docker/cli v28.0.0+incompatible + github.com/docker/docker v28.0.0+incompatible github.com/docker/go-connections v0.5.0 github.com/docker/go-units v0.5.0 github.com/fahedouch/go-logrotate v0.2.1 @@ -48,7 +43,6 @@ require ( github.com/klauspost/compress v1.18.0 github.com/mattn/go-isatty v0.0.20 github.com/moby/sys/mount v0.3.4 - github.com/moby/sys/mountinfo v0.7.2 github.com/moby/sys/signal v0.7.1 github.com/moby/sys/userns v0.1.0 github.com/moby/term v0.5.2 @@ -106,6 +100,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/locker v1.0.1 // indirect + github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/sys/sequential v0.6.0 // indirect github.com/moby/sys/symlink v0.3.0 // indirect github.com/moby/sys/user v0.3.0 // indirect diff --git a/go.sum b/go.sum index 298c8f0bfac..bac446a0db1 100644 --- a/go.sum +++ b/go.sum @@ -90,10 +90,10 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= -github.com/docker/cli v27.5.1+incompatible h1:JB9cieUT9YNiMITtIsguaN55PLOHhBSz3LKVc6cqWaY= -github.com/docker/cli v27.5.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker v27.5.1+incompatible h1:4PYU5dnBYqRQi0294d1FBECqT9ECWeQAIfE8q4YnPY8= -github.com/docker/docker v27.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/cli v28.0.0+incompatible h1:ido37VmLUqEp+5NFb9icd6BuBB+SNDgCn+5kPCr2buA= +github.com/docker/cli v28.0.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker v28.0.0+incompatible h1:Olh0KS820sJ7nPsBKChVhk5pzqcwDR15fumfAd/p9hM= +github.com/docker/docker v28.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo= github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= diff --git a/pkg/infoutil/infoutil_freebsd.go b/pkg/infoutil/infoutil_freebsd.go index b47c63df524..40cd76f8bf3 100644 --- a/pkg/infoutil/infoutil_freebsd.go +++ b/pkg/infoutil/infoutil_freebsd.go @@ -17,8 +17,9 @@ package infoutil import ( + "github.com/docker/docker/pkg/sysinfo" + "github.com/containerd/nerdctl/v2/pkg/inspecttypes/dockercompat" - "github.com/containerd/nerdctl/v2/pkg/sysinfo" ) const UnameO = "FreeBSD" diff --git a/pkg/infoutil/infoutil_linux.go b/pkg/infoutil/infoutil_linux.go index 93d22ee941c..61ea9ce4bca 100644 --- a/pkg/infoutil/infoutil_linux.go +++ b/pkg/infoutil/infoutil_linux.go @@ -18,9 +18,11 @@ package infoutil import ( "fmt" + "runtime" "strings" "github.com/docker/docker/pkg/meminfo" + "github.com/docker/docker/pkg/sysinfo" "github.com/containerd/cgroups/v3" @@ -28,7 +30,6 @@ import ( "github.com/containerd/nerdctl/v2/pkg/defaults" "github.com/containerd/nerdctl/v2/pkg/inspecttypes/dockercompat" "github.com/containerd/nerdctl/v2/pkg/rootlessutil" - "github.com/containerd/nerdctl/v2/pkg/sysinfo" ) const UnameO = "GNU/Linux" @@ -113,15 +114,19 @@ func fulfillPlatformInfo(info *dockercompat.Info) { if !info.IPv4Forwarding { info.Warnings = append(info.Warnings, "WARNING: IPv4 forwarding is disabled") } - info.BridgeNfIptables = !mobySysInfo.BridgeNFCallIPTablesDisabled + // FIXME: BridgeNFCallIP6TablesDisabled is deprecated in moby and always false now + // Figure out what we want to do with this + info.BridgeNfIptables = true // !mobySysInfo.BridgeNFCallIPTablesDisabled if !info.BridgeNfIptables { info.Warnings = append(info.Warnings, "WARNING: bridge-nf-call-iptables is disabled") } - info.BridgeNfIP6tables = !mobySysInfo.BridgeNFCallIP6TablesDisabled + // FIXME: BridgeNFCallIP6TablesDisabled is deprecated in moby and always false now + // Figure out what we want to do with this + info.BridgeNfIP6tables = true // !mobySysInfo.BridgeNFCallIP6TablesDisabled if !info.BridgeNfIP6tables { info.Warnings = append(info.Warnings, "WARNING: bridge-nf-call-ip6tables is disabled") } - info.NCPU = sysinfo.NumCPU() + info.NCPU = runtime.NumCPU() memLimit, err := meminfo.Read() if err != nil { info.Warnings = append(info.Warnings, fmt.Sprintf("failed to read mem info: %v", err)) diff --git a/pkg/infoutil/infoutil_windows.go b/pkg/infoutil/infoutil_windows.go index a8075078605..7758b905997 100644 --- a/pkg/infoutil/infoutil_windows.go +++ b/pkg/infoutil/infoutil_windows.go @@ -22,13 +22,13 @@ import ( "strings" "github.com/docker/docker/pkg/meminfo" + "github.com/docker/docker/pkg/sysinfo" "golang.org/x/sys/windows" "golang.org/x/sys/windows/registry" "github.com/containerd/log" "github.com/containerd/nerdctl/v2/pkg/inspecttypes/dockercompat" - "github.com/containerd/nerdctl/v2/pkg/sysinfo" ) const UnameO = "Microsoft Windows" @@ -204,15 +204,19 @@ func fulfillPlatformInfo(info *dockercompat.Info) { if !info.IPv4Forwarding { info.Warnings = append(info.Warnings, "WARNING: IPv4 forwarding is disabled") } - info.BridgeNfIptables = !mobySysInfo.BridgeNFCallIPTablesDisabled + // FIXME: BridgeNFCallIP6TablesDisabled is deprecated in moby and always false now + // Figure out what we want to do with this + info.BridgeNfIptables = true // !mobySysInfo.BridgeNFCallIPTablesDisabled if !info.BridgeNfIptables { info.Warnings = append(info.Warnings, "WARNING: bridge-nf-call-iptables is disabled") } - info.BridgeNfIP6tables = !mobySysInfo.BridgeNFCallIP6TablesDisabled + // FIXME: BridgeNFCallIP6TablesDisabled is deprecated in moby and always false now + // Figure out what we want to do with this + info.BridgeNfIP6tables = true // !mobySysInfo.BridgeNFCallIP6TablesDisabled if !info.BridgeNfIP6tables { info.Warnings = append(info.Warnings, "WARNING: bridge-nf-call-ip6tables is disabled") } - info.NCPU = sysinfo.NumCPU() + info.NCPU = runtime.NumCPU() memLimit, err := meminfo.Read() if err != nil { info.Warnings = append(info.Warnings, fmt.Sprintf("failed to read mem info: %v", err)) diff --git a/pkg/sysinfo/cgroup2_linux.go b/pkg/sysinfo/cgroup2_linux.go deleted file mode 100644 index d8dbe27eec0..00000000000 --- a/pkg/sysinfo/cgroup2_linux.go +++ /dev/null @@ -1,169 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Portions from https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/pkg/sysinfo/cgroup2_linux.go - Copyright (C) Docker/Moby authors. - Licensed under the Apache License, Version 2.0 - NOTICE: https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/NOTICE -*/ - -package sysinfo // import "github.com/docker/docker/pkg/sysinfo" - -import ( - "context" - "os" - "path" - "strings" - - "github.com/moby/sys/userns" - - "github.com/containerd/cgroups/v3" - cgroupsV2 "github.com/containerd/cgroups/v3/cgroup2" - "github.com/containerd/log" -) - -func newV2(options ...Opt) *SysInfo { - sysInfo := &SysInfo{ - CgroupUnified: true, - cg2GroupPath: "/", - } - for _, o := range options { - o(sysInfo) - } - - ops := []infoCollector{ - applyNetworkingInfo, - applyAppArmorInfo, - applySeccompInfo, - applyCgroupNsInfo, - } - - m, err := cgroupsV2.Load(sysInfo.cg2GroupPath) - if err != nil { - log.G(context.TODO()).Warn(err) - } else { - sysInfo.cg2Controllers = make(map[string]struct{}) - controllers, err := m.Controllers() - if err != nil { - log.G(context.TODO()).Warn(err) - } - for _, c := range controllers { - sysInfo.cg2Controllers[c] = struct{}{} - } - ops = append(ops, - applyMemoryCgroupInfoV2, - applyCPUCgroupInfoV2, - applyIOCgroupInfoV2, - applyCPUSetCgroupInfoV2, - applyPIDSCgroupInfoV2, - applyDevicesCgroupInfoV2, - ) - } - - for _, o := range ops { - o(sysInfo) - } - return sysInfo -} - -func getSwapLimitV2() bool { - _, g, err := cgroups.ParseCgroupFileUnified("/proc/self/cgroup") - if err != nil { - return false - } - - if g == "" { - return false - } - - cGroupPath := path.Join("/sys/fs/cgroup", g, "memory.swap.max") - if _, err = os.Stat(cGroupPath); os.IsNotExist(err) { - return false - } - return true -} - -func applyMemoryCgroupInfoV2(info *SysInfo) { - if _, ok := info.cg2Controllers["memory"]; !ok { - info.Warnings = append(info.Warnings, "Unable to find memory controller") - return - } - - info.MemoryLimit = true - info.SwapLimit = getSwapLimitV2() - info.MemoryReservation = true - info.OomKillDisable = false - info.MemorySwappiness = false - info.KernelMemory = false - info.KernelMemoryTCP = false -} - -func applyCPUCgroupInfoV2(info *SysInfo) { - if _, ok := info.cg2Controllers["cpu"]; !ok { - info.Warnings = append(info.Warnings, "Unable to find cpu controller") - return - } - info.CPUShares = true - info.CPUCfs = true - info.CPURealtime = false -} - -func applyIOCgroupInfoV2(info *SysInfo) { - if _, ok := info.cg2Controllers["io"]; !ok { - info.Warnings = append(info.Warnings, "Unable to find io controller") - return - } - - info.BlkioWeight = true - info.BlkioWeightDevice = true - info.BlkioReadBpsDevice = true - info.BlkioWriteBpsDevice = true - info.BlkioReadIOpsDevice = true - info.BlkioWriteIOpsDevice = true -} - -func applyCPUSetCgroupInfoV2(info *SysInfo) { - if _, ok := info.cg2Controllers["cpuset"]; !ok { - info.Warnings = append(info.Warnings, "Unable to find cpuset controller") - return - } - info.Cpuset = true - - cpus, err := os.ReadFile(path.Join("/sys/fs/cgroup", info.cg2GroupPath, "cpuset.cpus.effective")) - if err != nil { - return - } - info.Cpus = strings.TrimSpace(string(cpus)) - - mems, err := os.ReadFile(path.Join("/sys/fs/cgroup", info.cg2GroupPath, "cpuset.mems.effective")) - if err != nil { - return - } - info.Mems = strings.TrimSpace(string(mems)) -} - -func applyPIDSCgroupInfoV2(info *SysInfo) { - if _, ok := info.cg2Controllers["pids"]; !ok { - info.Warnings = append(info.Warnings, "Unable to find pids controller") - return - } - info.PidsLimit = true -} - -func applyDevicesCgroupInfoV2(info *SysInfo) { - info.CgroupDevicesEnabled = !userns.RunningInUserNS() -} diff --git a/pkg/sysinfo/doc.go b/pkg/sysinfo/doc.go deleted file mode 100644 index 17515398fbd..00000000000 --- a/pkg/sysinfo/doc.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package sysinfo is a copy of https://github.com/moby/moby/tree/master/pkg/sysinfo -// as of cff4f20c44a3a7c882ed73934dec6a77246c6323 -// This may be removed (and replaced by a dependency to moby again) once they -// have migrated to containerd v2. -package sysinfo diff --git a/pkg/sysinfo/numcpu.go b/pkg/sysinfo/numcpu.go deleted file mode 100644 index 9b585a11d88..00000000000 --- a/pkg/sysinfo/numcpu.go +++ /dev/null @@ -1,38 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Portions from https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/pkg/sysinfo/numcpu.go - Copyright (C) Docker/Moby authors. - Licensed under the Apache License, Version 2.0 - NOTICE: https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/NOTICE -*/ - -package sysinfo // import "github.com/docker/docker/pkg/sysinfo" - -import ( - "runtime" -) - -// NumCPU returns the number of CPUs. On Linux and Windows, it returns -// the number of CPUs which are currently online. On other platforms, -// it's the equivalent of [runtime.NumCPU]. -func NumCPU() int { - if ncpu := numCPU(); ncpu > 0 { - return ncpu - } - return runtime.NumCPU() -} diff --git a/pkg/sysinfo/numcpu_linux.go b/pkg/sysinfo/numcpu_linux.go deleted file mode 100644 index de590919ae4..00000000000 --- a/pkg/sysinfo/numcpu_linux.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Portions from https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/pkg/sysinfo/numcpu_linux.go - Copyright (C) Docker/Moby authors. - Licensed under the Apache License, Version 2.0 - NOTICE: https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/NOTICE -*/ - -package sysinfo // import "github.com/docker/docker/pkg/sysinfo" - -import "golang.org/x/sys/unix" - -// numCPU queries the system for the count of threads available -// for use to this process. -// -// Returns 0 on errors. Use |runtime.NumCPU| in that case. -func numCPU() int { - // Gets the affinity mask for a process: The very one invoking this function. - var mask unix.CPUSet - err := unix.SchedGetaffinity(0, &mask) - if err != nil { - return 0 - } - return mask.Count() -} diff --git a/pkg/sysinfo/numcpu_other.go b/pkg/sysinfo/numcpu_other.go deleted file mode 100644 index cd0c676b440..00000000000 --- a/pkg/sysinfo/numcpu_other.go +++ /dev/null @@ -1,31 +0,0 @@ -//go:build !linux && !windows - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Portions from https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/pkg/sysinfo/numcpu_other.go - Copyright (C) Docker/Moby authors. - Licensed under the Apache License, Version 2.0 - NOTICE: https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/NOTICE -*/ - -package sysinfo - -func numCPU() int { - // not implemented - return 0 -} diff --git a/pkg/sysinfo/numcpu_windows.go b/pkg/sysinfo/numcpu_windows.go deleted file mode 100644 index dfa6496896d..00000000000 --- a/pkg/sysinfo/numcpu_windows.go +++ /dev/null @@ -1,59 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Portions from https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/pkg/sysinfo/numcpu_windows.go - Copyright (C) Docker/Moby authors. - Licensed under the Apache License, Version 2.0 - NOTICE: https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/NOTICE -*/ - -package sysinfo // import "github.com/docker/docker/pkg/sysinfo" - -import ( - "unsafe" - - "golang.org/x/sys/windows" -) - -var ( - kernel32 = windows.NewLazySystemDLL("kernel32.dll") - getCurrentProcess = kernel32.NewProc("GetCurrentProcess") - getProcessAffinityMask = kernel32.NewProc("GetProcessAffinityMask") -) - -// Returns bit count of 1, used by NumCPU -func popcnt(x uint64) (n byte) { - x -= (x >> 1) & 0x5555555555555555 - x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 - x += x >> 4 - x &= 0x0f0f0f0f0f0f0f0f - x *= 0x0101010101010101 - return byte(x >> 56) -} - -func numCPU() int { - // Gets the affinity mask for a process - var mask, sysmask uintptr - currentProcess, _, _ := getCurrentProcess.Call() - ret, _, _ := getProcessAffinityMask.Call(currentProcess, uintptr(unsafe.Pointer(&mask)), uintptr(unsafe.Pointer(&sysmask))) - if ret == 0 { - return 0 - } - // For every available thread a bit is set in the mask. - ncpu := int(popcnt(uint64(mask))) - return ncpu -} diff --git a/pkg/sysinfo/sysinfo.go b/pkg/sysinfo/sysinfo.go deleted file mode 100644 index 0e150222c9b..00000000000 --- a/pkg/sysinfo/sysinfo.go +++ /dev/null @@ -1,193 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Portions from https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/pkg/sysinfo/sysinfo.go - Copyright (C) Docker/Moby authors. - Licensed under the Apache License, Version 2.0 - NOTICE: https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/NOTICE -*/ - -// Package sysinfo stores information about which features a kernel supports. -package sysinfo // import "github.com/docker/docker/" - -import "github.com/docker/docker/pkg/parsers" - -// Opt for New(). -type Opt func(info *SysInfo) - -// SysInfo stores information about which features a kernel supports. -// TODO Windows: Factor out platform specific capabilities. -type SysInfo struct { - // Whether the kernel supports AppArmor or not - AppArmor bool - // Whether the kernel supports Seccomp or not - Seccomp bool - - cgroupMemInfo - cgroupCPUInfo - cgroupBlkioInfo - cgroupCpusetInfo - cgroupPids - - // Whether the kernel supports cgroup namespaces or not - CgroupNamespaces bool - - // Whether IPv4 forwarding is supported or not, if this was disabled, networking will not work - IPv4ForwardingDisabled bool - - // Whether bridge-nf-call-iptables is supported or not - BridgeNFCallIPTablesDisabled bool - - // Whether bridge-nf-call-ip6tables is supported or not - BridgeNFCallIP6TablesDisabled bool - - // Whether the cgroup has the mountpoint of "devices" or not - CgroupDevicesEnabled bool - - // Whether the cgroup is in unified mode (v2). - CgroupUnified bool - - // Warnings contains a slice of warnings that occurred while collecting - // system information. These warnings are intended to be informational - // messages for the user, and can either be logged or returned to the - // client; they are not intended to be parsed / used for other purposes, - // and do not have a fixed format. - Warnings []string - - // cgMounts is the list of cgroup v1 mount paths, indexed by subsystem, to - // inspect availability of subsystems. - cgMounts map[string]string //nolint:unused - - // cg2GroupPath is the cgroup v2 group path to inspect availability of the controllers. - cg2GroupPath string //nolint:unused - - // cg2Controllers is an index of available cgroup v2 controllers. - cg2Controllers map[string]struct{} //nolint:unused -} - -type cgroupMemInfo struct { - // Whether memory limit is supported or not - MemoryLimit bool - - // Whether swap limit is supported or not - SwapLimit bool - - // Whether soft limit is supported or not - MemoryReservation bool - - // Whether OOM killer disable is supported or not - OomKillDisable bool - - // Whether memory swappiness is supported or not - MemorySwappiness bool - - // Whether kernel memory limit is supported or not. This option is used to - // detect support for kernel-memory limits on API < v1.42. Kernel memory - // limit (`kmem.limit_in_bytes`) is not supported on cgroups v2, and has been - // removed in kernel 5.4. - KernelMemory bool - - // Whether kernel memory TCP limit is supported or not. Kernel memory TCP - // limit (`memory.kmem.tcp.limit_in_bytes`) is not supported on cgroups v2. - KernelMemoryTCP bool -} - -type cgroupCPUInfo struct { - // Whether CPU shares is supported or not - CPUShares bool - - // Whether CPU CFS (Completely Fair Scheduler) is supported - CPUCfs bool - - // Whether CPU real-time scheduler is supported - CPURealtime bool -} - -type cgroupBlkioInfo struct { - // Whether Block IO weight is supported or not - BlkioWeight bool - - // Whether Block IO weight_device is supported or not - BlkioWeightDevice bool - - // Whether Block IO read limit in bytes per second is supported or not - BlkioReadBpsDevice bool - - // Whether Block IO write limit in bytes per second is supported or not - BlkioWriteBpsDevice bool - - // Whether Block IO read limit in IO per second is supported or not - BlkioReadIOpsDevice bool - - // Whether Block IO write limit in IO per second is supported or not - BlkioWriteIOpsDevice bool -} - -type cgroupCpusetInfo struct { - // Whether Cpuset is supported or not - Cpuset bool - - // Available Cpuset's cpus - Cpus string - - // Available Cpuset's memory nodes - Mems string -} - -type cgroupPids struct { - // Whether Pids Limit is supported or not - PidsLimit bool -} - -// IsCpusetCpusAvailable returns `true` if the provided string set is contained -// in cgroup's cpuset.cpus set, `false` otherwise. -// If error is not nil a parsing error occurred. -func (c cgroupCpusetInfo) IsCpusetCpusAvailable(provided string) (bool, error) { - return isCpusetListAvailable(provided, c.Cpus) -} - -// IsCpusetMemsAvailable returns `true` if the provided string set is contained -// in cgroup's cpuset.mems set, `false` otherwise. -// If error is not nil a parsing error occurred. -func (c cgroupCpusetInfo) IsCpusetMemsAvailable(provided string) (bool, error) { - return isCpusetListAvailable(provided, c.Mems) -} - -func isCpusetListAvailable(provided, available string) (bool, error) { - parsedAvailable, err := parsers.ParseUintList(available) - if err != nil { - return false, err - } - // 8192 is the normal maximum number of CPUs in Linux, so accept numbers up to this - // or more if we actually have more CPUs. - maxCPUs := 8192 - for m := range parsedAvailable { - if m > maxCPUs { - maxCPUs = m - } - } - parsedProvided, err := parsers.ParseUintListMaximum(provided, maxCPUs) - if err != nil { - return false, err - } - for k := range parsedProvided { - if !parsedAvailable[k] { - return false, nil - } - } - return true, nil -} diff --git a/pkg/sysinfo/sysinfo_linux.go b/pkg/sysinfo/sysinfo_linux.go deleted file mode 100644 index f8b50440878..00000000000 --- a/pkg/sysinfo/sysinfo_linux.go +++ /dev/null @@ -1,330 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Portions from https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/pkg/sysinfo/sysinfo_linux.go - Copyright (C) Docker/Moby authors. - Licensed under the Apache License, Version 2.0 - NOTICE: https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/NOTICE -*/ - -package sysinfo // import "github.com/docker/docker/pkg/sysinfo" - -import ( - "context" - "fmt" - "os" - "path" - "strings" - "sync" - - "github.com/moby/sys/mountinfo" - - "github.com/containerd/cgroups/v3" - "github.com/containerd/cgroups/v3/cgroup1" - "github.com/containerd/containerd/v2/pkg/seccomp" - "github.com/containerd/log" -) - -var ( - readMountinfoOnce sync.Once - readMountinfoErr error - cgroupMountinfo []*mountinfo.Info -) - -// readCgroupMountinfo returns a list of cgroup v1 mounts (i.e. the ones -// with fstype of "cgroup") for the current running process. -// -// The results are cached (to avoid re-reading mountinfo which is relatively -// expensive), so it is assumed that cgroup mounts are not being changed. -func readCgroupMountinfo() ([]*mountinfo.Info, error) { - readMountinfoOnce.Do(func() { - cgroupMountinfo, readMountinfoErr = mountinfo.GetMounts( - mountinfo.FSTypeFilter("cgroup"), - ) - }) - - return cgroupMountinfo, readMountinfoErr -} - -func findCgroupV1Mountpoints() (map[string]string, error) { - mounts, err := readCgroupMountinfo() - if err != nil { - return nil, err - } - - allSubsystems, err := cgroup1.ParseCgroupFile("/proc/self/cgroup") - if err != nil { - return nil, fmt.Errorf("failed to parse cgroup information: %v", err) - } - - allMap := make(map[string]bool) - for s := range allSubsystems { - allMap[s] = false - } - - mps := make(map[string]string) - for _, mi := range mounts { - for _, opt := range strings.Split(mi.VFSOptions, ",") { - seen, known := allMap[opt] - if known && !seen { - allMap[opt] = true - mps[strings.TrimPrefix(opt, "name=")] = mi.Mountpoint - } - } - if len(mps) >= len(allMap) { - break - } - } - return mps, nil -} - -type infoCollector func(info *SysInfo) - -// WithCgroup2GroupPath specifies the cgroup v2 group path to inspect availability -// of the controllers. -// -// WithCgroup2GroupPath is expected to be used for rootless mode with systemd driver. -// -// e.g. g = "/user.slice/user-1000.slice/user@1000.service" -func WithCgroup2GroupPath(g string) Opt { - return func(o *SysInfo) { - if p := path.Clean(g); p != "" { - o.cg2GroupPath = p - } - } -} - -// New returns a new SysInfo, using the filesystem to detect which features -// the kernel supports. -func New(options ...Opt) *SysInfo { - if cgroups.Mode() == cgroups.Unified { - return newV2(options...) - } - return newV1() -} - -func newV1() *SysInfo { - var ( - err error - sysInfo = &SysInfo{} - ) - - ops := []infoCollector{ - applyNetworkingInfo, - applyAppArmorInfo, - applySeccompInfo, - applyCgroupNsInfo, - } - - sysInfo.cgMounts, err = findCgroupV1Mountpoints() - if err != nil { - log.G(context.TODO()).Warn(err) - } else { - ops = append(ops, - applyMemoryCgroupInfo, - applyCPUCgroupInfo, - applyBlkioCgroupInfo, - applyCPUSetCgroupInfo, - applyPIDSCgroupInfo, - applyDevicesCgroupInfo, - ) - } - - for _, o := range ops { - o(sysInfo) - } - return sysInfo -} - -// applyMemoryCgroupInfo adds the memory cgroup controller information to the info. -func applyMemoryCgroupInfo(info *SysInfo) { - mountPoint, ok := info.cgMounts["memory"] - if !ok { - info.Warnings = append(info.Warnings, "Your kernel does not support cgroup memory limit") - return - } - info.MemoryLimit = ok - - info.SwapLimit = cgroupEnabled(mountPoint, "memory.memsw.limit_in_bytes") - if !info.SwapLimit { - info.Warnings = append(info.Warnings, "Your kernel does not support swap memory limit") - } - info.MemoryReservation = cgroupEnabled(mountPoint, "memory.soft_limit_in_bytes") - if !info.MemoryReservation { - info.Warnings = append(info.Warnings, "Your kernel does not support memory reservation") - } - info.OomKillDisable = cgroupEnabled(mountPoint, "memory.oom_control") - if !info.OomKillDisable { - info.Warnings = append(info.Warnings, "Your kernel does not support oom control") - } - info.MemorySwappiness = cgroupEnabled(mountPoint, "memory.swappiness") - if !info.MemorySwappiness { - info.Warnings = append(info.Warnings, "Your kernel does not support memory swappiness") - } - - // Option is deprecated, but still accepted on API < v1.42 with cgroups v1, - // so setting the field to allow feature detection. - info.KernelMemory = cgroupEnabled(mountPoint, "memory.kmem.limit_in_bytes") - - // Option is deprecated in runc, but still accepted in our API, so setting - // the field to allow feature detection, but don't warn if it's missing, to - // make the daemon logs a bit less noisy. - info.KernelMemoryTCP = cgroupEnabled(mountPoint, "memory.kmem.tcp.limit_in_bytes") -} - -// applyCPUCgroupInfo adds the cpu cgroup controller information to the info. -func applyCPUCgroupInfo(info *SysInfo) { - mountPoint, ok := info.cgMounts["cpu"] - if !ok { - info.Warnings = append(info.Warnings, "Unable to find cpu cgroup in mounts") - return - } - - info.CPUShares = cgroupEnabled(mountPoint, "cpu.shares") - if !info.CPUShares { - info.Warnings = append(info.Warnings, "Your kernel does not support CPU shares") - } - - info.CPUCfs = cgroupEnabled(mountPoint, "cpu.cfs_quota_us") - if !info.CPUCfs { - info.Warnings = append(info.Warnings, "Your kernel does not support CPU CFS scheduler") - } - - info.CPURealtime = cgroupEnabled(mountPoint, "cpu.rt_period_us") - if !info.CPURealtime { - info.Warnings = append(info.Warnings, "Your kernel does not support CPU realtime scheduler") - } -} - -// applyBlkioCgroupInfo adds the blkio cgroup controller information to the info. -func applyBlkioCgroupInfo(info *SysInfo) { - mountPoint, ok := info.cgMounts["blkio"] - if !ok { - info.Warnings = append(info.Warnings, "Unable to find blkio cgroup in mounts") - return - } - - info.BlkioWeight = cgroupEnabled(mountPoint, "blkio.weight") - if !info.BlkioWeight { - info.Warnings = append(info.Warnings, "Your kernel does not support cgroup blkio weight") - } - - info.BlkioWeightDevice = cgroupEnabled(mountPoint, "blkio.weight_device") - if !info.BlkioWeightDevice { - info.Warnings = append(info.Warnings, "Your kernel does not support cgroup blkio weight_device") - } - - info.BlkioReadBpsDevice = cgroupEnabled(mountPoint, "blkio.throttle.read_bps_device") - if !info.BlkioReadBpsDevice { - info.Warnings = append(info.Warnings, "Your kernel does not support cgroup blkio throttle.read_bps_device") - } - - info.BlkioWriteBpsDevice = cgroupEnabled(mountPoint, "blkio.throttle.write_bps_device") - if !info.BlkioWriteBpsDevice { - info.Warnings = append(info.Warnings, "Your kernel does not support cgroup blkio throttle.write_bps_device") - } - info.BlkioReadIOpsDevice = cgroupEnabled(mountPoint, "blkio.throttle.read_iops_device") - if !info.BlkioReadIOpsDevice { - info.Warnings = append(info.Warnings, "Your kernel does not support cgroup blkio throttle.read_iops_device") - } - - info.BlkioWriteIOpsDevice = cgroupEnabled(mountPoint, "blkio.throttle.write_iops_device") - if !info.BlkioWriteIOpsDevice { - info.Warnings = append(info.Warnings, "Your kernel does not support cgroup blkio throttle.write_iops_device") - } -} - -// applyCPUSetCgroupInfo adds the cpuset cgroup controller information to the info. -func applyCPUSetCgroupInfo(info *SysInfo) { - mountPoint, ok := info.cgMounts["cpuset"] - if !ok { - info.Warnings = append(info.Warnings, "Unable to find cpuset cgroup in mounts") - return - } - info.Cpuset = ok - - var err error - - cpus, err := os.ReadFile(path.Join(mountPoint, "cpuset.cpus")) - if err != nil { - return - } - info.Cpus = strings.TrimSpace(string(cpus)) - - mems, err := os.ReadFile(path.Join(mountPoint, "cpuset.mems")) - if err != nil { - return - } - info.Mems = strings.TrimSpace(string(mems)) -} - -// applyPIDSCgroupInfo adds whether the pids cgroup controller is available to the info. -func applyPIDSCgroupInfo(info *SysInfo) { - _, ok := info.cgMounts["pids"] - if !ok { - info.Warnings = append(info.Warnings, "Unable to find pids cgroup in mounts") - return - } - info.PidsLimit = true -} - -// applyDevicesCgroupInfo adds whether the devices cgroup controller is available to the info. -func applyDevicesCgroupInfo(info *SysInfo) { - _, ok := info.cgMounts["devices"] - info.CgroupDevicesEnabled = ok -} - -// applyNetworkingInfo adds networking information to the info. -func applyNetworkingInfo(info *SysInfo) { - info.IPv4ForwardingDisabled = !readProcBool("/proc/sys/net/ipv4/ip_forward") - info.BridgeNFCallIPTablesDisabled = !readProcBool("/proc/sys/net/bridge/bridge-nf-call-iptables") - info.BridgeNFCallIP6TablesDisabled = !readProcBool("/proc/sys/net/bridge/bridge-nf-call-ip6tables") -} - -// applyAppArmorInfo adds whether AppArmor is enabled to the info. -func applyAppArmorInfo(info *SysInfo) { - if _, err := os.Stat("/sys/kernel/security/apparmor"); !os.IsNotExist(err) { - if _, err := os.ReadFile("/sys/kernel/security/apparmor/profiles"); err == nil { - info.AppArmor = true - } - } -} - -// applyCgroupNsInfo adds whether cgroupns is enabled to the info. -func applyCgroupNsInfo(info *SysInfo) { - if _, err := os.Stat("/proc/self/ns/cgroup"); !os.IsNotExist(err) { - info.CgroupNamespaces = true - } -} - -// applySeccompInfo checks if Seccomp is supported, via CONFIG_SECCOMP. -func applySeccompInfo(info *SysInfo) { - info.Seccomp = seccomp.IsEnabled() -} - -func cgroupEnabled(mountPoint, name string) bool { - _, err := os.Stat(path.Join(mountPoint, name)) - return err == nil -} - -func readProcBool(path string) bool { - val, err := os.ReadFile(path) - if err != nil { - return false - } - return strings.TrimSpace(string(val)) == "1" -} diff --git a/pkg/sysinfo/sysinfo_linux_test.go b/pkg/sysinfo/sysinfo_linux_test.go deleted file mode 100644 index 7bfc3b6995d..00000000000 --- a/pkg/sysinfo/sysinfo_linux_test.go +++ /dev/null @@ -1,147 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Portions from https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/pkg/sysinfo/sysinfo_linux_test.go - Copyright (C) Docker/Moby authors. - Licensed under the Apache License, Version 2.0 - NOTICE: https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/NOTICE -*/ - -package sysinfo // import "github.com/docker/docker/pkg/sysinfo" - -import ( - "os" - "path" - "path/filepath" - "testing" - - "golang.org/x/sys/unix" - "gotest.tools/v3/assert" - - "github.com/containerd/nerdctl/v2/pkg/rootlessutil" -) - -func TestReadProcBool(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "test-sysinfo-proc") - assert.NilError(t, err) - defer os.RemoveAll(tmpDir) - - procFile := filepath.Join(tmpDir, "read-proc-bool") - err = os.WriteFile(procFile, []byte("1"), 0o644) - assert.NilError(t, err) - - if !readProcBool(procFile) { - t.Fatal("expected proc bool to be true, got false") - } - - if err := os.WriteFile(procFile, []byte("0"), 0o644); err != nil { - t.Fatal(err) - } - if readProcBool(procFile) { - t.Fatal("expected proc bool to be false, got true") - } - - if readProcBool(path.Join(tmpDir, "no-exist")) { - t.Fatal("should be false for non-existent entry") - } -} - -func TestCgroupEnabled(t *testing.T) { - cgroupDir, err := os.MkdirTemp("", "cgroup-test") - assert.NilError(t, err) - defer os.RemoveAll(cgroupDir) - - if cgroupEnabled(cgroupDir, "test") { - t.Fatal("cgroupEnabled should be false") - } - - err = os.WriteFile(path.Join(cgroupDir, "test"), []byte{}, 0o644) - assert.NilError(t, err) - - if !cgroupEnabled(cgroupDir, "test") { - t.Fatal("cgroupEnabled should be true") - } -} - -func TestNew(t *testing.T) { - sysInfo := New() - assert.Assert(t, sysInfo != nil) - checkSysInfo(t, sysInfo) -} - -func checkSysInfo(t *testing.T, sysInfo *SysInfo) { - // Check if Seccomp is supported, via CONFIG_SECCOMP.then sysInfo.Seccomp must be TRUE , else FALSE - if err := unix.Prctl(unix.PR_GET_SECCOMP, 0, 0, 0, 0); err != unix.EINVAL { - // Make sure the kernel has CONFIG_SECCOMP_FILTER. - if err := unix.Prctl(unix.PR_SET_SECCOMP, unix.SECCOMP_MODE_FILTER, 0, 0, 0); err != unix.EINVAL { - assert.Assert(t, sysInfo.Seccomp) - } - } else { - assert.Assert(t, !sysInfo.Seccomp) - } -} - -func TestNewAppArmorEnabled(t *testing.T) { - // Check if AppArmor is supported. then it must be TRUE , else FALSE - if _, err := os.Stat("/sys/kernel/security/apparmor"); err != nil { - t.Skip("AppArmor Must be Enabled") - } - - // FIXME: rootless is not allowed to read the profile - if rootlessutil.IsRootless() { - t.Skip("containerd v2 aftermath: test skipped for rootless") - } - sysInfo := New() - assert.Assert(t, sysInfo.AppArmor) -} - -func TestNewAppArmorDisabled(t *testing.T) { - // Check if AppArmor is supported. then it must be TRUE , else FALSE - if _, err := os.Stat("/sys/kernel/security/apparmor"); !os.IsNotExist(err) { - t.Skip("AppArmor Must be Disabled") - } - - sysInfo := New() - assert.Assert(t, !sysInfo.AppArmor) -} - -func TestNewCgroupNamespacesEnabled(t *testing.T) { - // If cgroup namespaces are supported in the kernel, then sysInfo.CgroupNamespaces should be TRUE - if _, err := os.Stat("/proc/self/ns/cgroup"); err != nil { - t.Skip("cgroup namespaces must be enabled") - } - - sysInfo := New() - assert.Assert(t, sysInfo.CgroupNamespaces) -} - -func TestNewCgroupNamespacesDisabled(t *testing.T) { - // If cgroup namespaces are *not* supported in the kernel, then sysInfo.CgroupNamespaces should be FALSE - if _, err := os.Stat("/proc/self/ns/cgroup"); !os.IsNotExist(err) { - t.Skip("cgroup namespaces must be disabled") - } - - sysInfo := New() - assert.Assert(t, !sysInfo.CgroupNamespaces) -} - -func TestNumCPU(t *testing.T) { - cpuNumbers := NumCPU() - if cpuNumbers <= 0 { - t.Fatal("CPU returned must be greater than zero") - } -} diff --git a/pkg/sysinfo/sysinfo_other.go b/pkg/sysinfo/sysinfo_other.go deleted file mode 100644 index c48608a1909..00000000000 --- a/pkg/sysinfo/sysinfo_other.go +++ /dev/null @@ -1,31 +0,0 @@ -//go:build !linux - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Portions from https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/pkg/sysinfo/sysinfo_other.go - Copyright (C) Docker/Moby authors. - Licensed under the Apache License, Version 2.0 - NOTICE: https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/NOTICE -*/ - -package sysinfo // import "github.com/docker/docker/pkg/sysinfo" - -// New returns an empty SysInfo for non linux for now. -func New(options ...Opt) *SysInfo { - return &SysInfo{} -} diff --git a/pkg/sysinfo/sysinfo_test.go b/pkg/sysinfo/sysinfo_test.go deleted file mode 100644 index 045036a602d..00000000000 --- a/pkg/sysinfo/sysinfo_test.go +++ /dev/null @@ -1,49 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Portions from https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/pkg/sysinfo/sysinfo_test.go - Copyright (C) Docker/Moby authors. - Licensed under the Apache License, Version 2.0 - NOTICE: https://github.com/moby/moby/blob/cff4f20c44a3a7c882ed73934dec6a77246c6323/NOTICE -*/ - -package sysinfo // import "github.com/docker/docker/pkg/sysinfo" - -import "testing" - -func TestIsCpusetListAvailable(t *testing.T) { - cases := []struct { - provided string - available string - res bool - err bool - }{ - {"1", "0-4", true, false}, - {"01,3", "0-4", true, false}, - {"", "0-7", true, false}, - {"1--42", "0-7", false, true}, - {"1-42", "00-1,8,,9", false, true}, - {"1,41-42", "43,45", false, false}, - {"0-3", "", false, false}, - } - for _, c := range cases { - r, err := isCpusetListAvailable(c.provided, c.available) - if (c.err && err == nil) && r != c.res { - t.Fatalf("Expected pair: %v, %v for %s, %s. Got %v, %v instead", c.res, c.err, c.provided, c.available, (c.err && err == nil), r) - } - } -}