diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d26bd36d111..72318a824fc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -262,8 +262,7 @@ jobs: - name: Set up vagrant run: | sudo apt-get update - sudo apt-get install -y libvirt-daemon libvirt-daemon-system vagrant vagrant-libvirt - sudo systemctl enable --now libvirtd + sudo apt-get install -y virtualbox vagrant - name: Boot VM run: | ln -sf Vagrantfile.freebsd Vagrantfile diff --git a/Vagrantfile.freebsd b/Vagrantfile.freebsd index 08db3dddc4f..b79e079d9bc 100644 --- a/Vagrantfile.freebsd +++ b/Vagrantfile.freebsd @@ -17,7 +17,7 @@ # Vagrantfile for FreeBSD Vagrant.configure("2") do |config| - config.vm.box = "generic/freebsd13" + config.vm.box = "freebsd/FreeBSD-13.2-STABLE" memory = 2048 cpus = 1 @@ -25,10 +25,6 @@ Vagrant.configure("2") do |config| v.memory = memory v.cpus = cpus end - config.vm.provider :libvirt do |v| - v.memory = memory - v.cpus = cpus - end config.vm.synced_folder ".", "/vagrant", type: "rsync" @@ -36,9 +32,20 @@ Vagrant.configure("2") do |config| sh.inline = <<~SHELL #!/usr/bin/env bash set -eux -o pipefail - pkg install -y go containerd runj + pkg install -y go containerd runj containernetworking-plugins cd /vagrant go install ./cmd/nerdctl + + mkdir -p /etc/nerdctl + cat << EOF > /etc/nerdctl/runj.ext.json + { + "network": { + "vnet": { + "mode": "new" + } + } + } + EOF SHELL end @@ -55,9 +62,20 @@ Vagrant.configure("2") do |config| sh.inline = <<~SHELL #!/usr/bin/env bash set -eux -o pipefail + + # Firewall config, needed for CNI plugins. + cat << EOF > /etc/pf.conf + nat on em0 inet from to any -> (em0) + rdr-anchor "cni-rdr/*" + table + EOF + + service pf onestart + daemon -o containerd.out containerd sleep 3 - /root/go/bin/nerdctl run --rm --net=none dougrabson/freebsd-minimal:13 echo "Nerdctl is up and running." + /root/go/bin/nerdctl run --rm --net=none dougrabson/freebsd-small:13 echo "Nerdctl is up and running." + /root/go/bin/nerdctl run --rm dougrabson/freebsd-small:13 nc -zw1 1.1.1.1 443 SHELL end diff --git a/cmd/nerdctl/container_run.go b/cmd/nerdctl/container_run.go index df6e4521806..30efcac876d 100644 --- a/cmd/nerdctl/container_run.go +++ b/cmd/nerdctl/container_run.go @@ -51,7 +51,7 @@ func newRunCommand() *cobra.Command { longHelp += "WARNING: `nerdctl run` is experimental on Windows and currently broken (https://github.com/containerd/nerdctl/issues/28)" case "freebsd": longHelp += "\n" - longHelp += "WARNING: `nerdctl run` is experimental on FreeBSD and currently requires `--net=none` (https://github.com/containerd/nerdctl/blob/main/docs/freebsd.md)" + longHelp += "WARNING: `nerdctl run` is experimental on FreeBSD (https://github.com/containerd/nerdctl/blob/main/docs/freebsd.md)" } var runCommand = &cobra.Command{ Use: "run [flags] IMAGE [COMMAND] [ARG...]", diff --git a/docs/freebsd.md b/docs/freebsd.md index 4ad17aedba3..cfdfb0e009b 100644 --- a/docs/freebsd.md +++ b/docs/freebsd.md @@ -12,22 +12,54 @@ You will need the most up-to-date containerd build along with a containerd shim, such as [runj](https://github.com/samuelkarp/runj). Follow the build instructions in the respective repositories. +The runj runtime must be configured to create vnet jails by default. To do this, you need to create + +`/etc/nerdctl/runj.ext.json` with the following content. + +``` +{ + "network": { + "vnet": { + "mode": "new" + } + } +} +``` + ## Usage You can use the `dougrabson/freebsd13.2-small` image to run a FreeBSD 13 jail: ```sh -nerdctl run --net none -it dougrabson/freebsd13.2-small +nerdctl run --net=none -it dougrabson/freebsd13.2-small ``` Alternatively use `--platform` parameter to run linux containers ```sh -nerdctl run --platform linux --net none -it amazonlinux:2 +nerdctl run --platform linux --net=none -it amazonlinux:2 +``` + +:warning: running linux containers requires `linux64` module loaded: + +``` +kldload linux64 ``` -## Limitations & Bugs +## CNI networking -- :warning: CNI & CNI plugins are not yet ported to FreeBSD. The only supported - network type is `none` +| :construction: CNI networking requires host OS to be version 13.3 and higher. Lower versions are not guaranteed to work. | +|---------------------------------------------------------------------------------------------------------------------------------| + +CNI plugins can be installed from the repository + +```sh +pkg install net/containernetworking-plugins +``` + +You can then drop the `--net=none` flag and run commands as usual. + +```sh +nerdctl run -it dougrabson/freebsd13.2-small ping 1.1.1.1 +``` diff --git a/pkg/cmd/container/create.go b/pkg/cmd/container/create.go index 88246e3d7b3..3199ec2c5d0 100644 --- a/pkg/cmd/container/create.go +++ b/pkg/cmd/container/create.go @@ -225,11 +225,8 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa } opts = append(opts, umaskOpts...) - rtCOpts, err := generateRuntimeCOpts(options.GOptions.CgroupManager, options.Runtime) - if err != nil { - return nil, nil, err - } - cOpts = append(cOpts, rtCOpts...) + rtOpts := generateRuntimeOpts(options) + cOpts = append(cOpts, rtOpts) lCOpts, err := withContainerLabels(options.Label, options.LabelFile) if err != nil { diff --git a/pkg/cmd/container/run_runtime.go b/pkg/cmd/container/run_runtime.go index 20a121d5dad..77db86fb8b6 100644 --- a/pkg/cmd/container/run_runtime.go +++ b/pkg/cmd/container/run_runtime.go @@ -23,37 +23,53 @@ import ( "github.com/containerd/containerd" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/oci" + runtimeoptions "github.com/containerd/containerd/pkg/runtimeoptions/v1" "github.com/containerd/containerd/plugin" runcoptions "github.com/containerd/containerd/runtime/v2/runc/options" + "github.com/containerd/nerdctl/pkg/api/types" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" ) -func generateRuntimeCOpts(cgroupManager, runtimeStr string) ([]containerd.NewContainerOpts, error) { +func generateRuntimeOpts(options types.ContainerCreateOptions) containerd.NewContainerOpts { + runtimeOpts := generateRuntimeCOpts(options.GOptions.CgroupManager, options.Runtime) + + if options.Runtime == "wtf.sbk.runj.v1" { + runtimeOpts = generateRunjOpts() + } + + return runtimeOpts +} + +func generateRunjOpts() containerd.NewContainerOpts { + return containerd.WithRuntime("wtf.sbk.runj.v1", &runtimeoptions.Options{ + ConfigPath: "/etc/nerdctl/runj.ext.json", + }) +} + +func generateRuntimeCOpts(cgroupManager, runtimeStr string) containerd.NewContainerOpts { runtime := plugin.RuntimeRuncV2 - var ( - runcOpts runcoptions.Options - runtimeOpts interface{} = &runcOpts - ) + var runcOpts runcoptions.Options + if cgroupManager == "systemd" { runcOpts.SystemdCgroup = true } if runtimeStr != "" { - if strings.HasPrefix(runtimeStr, "io.containerd.") || runtimeStr == "wtf.sbk.runj.v1" { + if strings.HasPrefix(runtimeStr, "io.containerd.") { runtime = runtimeStr if !strings.HasPrefix(runtimeStr, "io.containerd.runc.") { if cgroupManager == "systemd" { logrus.Warnf("cannot set cgroup manager to %q for runtime %q", cgroupManager, runtimeStr) } - runtimeOpts = nil + return nil } } else { // runtimeStr is a runc binary runcOpts.BinaryName = runtimeStr } } - o := containerd.WithRuntime(runtime, runtimeOpts) - return []containerd.NewContainerOpts{o}, nil + + return containerd.WithRuntime(runtime, &runcOpts) } // WithSysctls sets the provided sysctls onto the spec diff --git a/pkg/containerutil/container_network_manager.go b/pkg/containerutil/container_network_manager.go index 77e8dfaeabc..8806d6a0e13 100644 --- a/pkg/containerutil/container_network_manager.go +++ b/pkg/containerutil/container_network_manager.go @@ -49,36 +49,21 @@ const ( func withCustomResolvConf(src string) func(context.Context, oci.Client, *containers.Container, *oci.Spec) error { return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error { - s.Mounts = append(s.Mounts, specs.Mount{ - Destination: "/etc/resolv.conf", - Type: "bind", - Source: src, - Options: []string{"bind", mountutil.DefaultPropagationMode}, // writable - }) + s.Mounts = append(s.Mounts, bindMount(src, "/etc/resolv.conf")) return nil } } func withCustomEtcHostname(src string) func(context.Context, oci.Client, *containers.Container, *oci.Spec) error { return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error { - s.Mounts = append(s.Mounts, specs.Mount{ - Destination: "/etc/hostname", - Type: "bind", - Source: src, - Options: []string{"bind", mountutil.DefaultPropagationMode}, // writable - }) + s.Mounts = append(s.Mounts, bindMount(src, "/etc/hostname")) return nil } } func withCustomHosts(src string) func(context.Context, oci.Client, *containers.Container, *oci.Spec) error { return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error { - s.Mounts = append(s.Mounts, specs.Mount{ - Destination: "/etc/hosts", - Type: "bind", - Source: src, - Options: []string{"bind", mountutil.DefaultPropagationMode}, // writable - }) + s.Mounts = append(s.Mounts, bindMount(src, "/etc/hosts")) return nil } } @@ -556,3 +541,20 @@ func nonZeroMapValues(values map[string]interface{}) []string { return nonZero } + +func bindMount(source, destination string) specs.Mount { + fstype := "bind" + options := []string{"bind", mountutil.DefaultPropagationMode} + + if runtime.GOOS == "freebsd" { + fstype = "nullfs" + options = []string{"rw"} + } + + return specs.Mount{ + Destination: destination, + Type: fstype, + Source: source, + Options: options, // writable + } +} diff --git a/pkg/containerutil/container_network_manager_linux.go b/pkg/containerutil/container_network_manager_unix.go similarity index 99% rename from pkg/containerutil/container_network_manager_linux.go rename to pkg/containerutil/container_network_manager_unix.go index 45bfd4523d0..6d7b16240a1 100644 --- a/pkg/containerutil/container_network_manager_linux.go +++ b/pkg/containerutil/container_network_manager_unix.go @@ -1,3 +1,5 @@ +//go:build linux || freebsd + /* Copyright The containerd Authors. diff --git a/pkg/containerutil/container_network_manager_other.go b/pkg/containerutil/container_network_manager_unsupported.go similarity index 97% rename from pkg/containerutil/container_network_manager_other.go rename to pkg/containerutil/container_network_manager_unsupported.go index 500872f57c3..fd50ea9d7bf 100644 --- a/pkg/containerutil/container_network_manager_other.go +++ b/pkg/containerutil/container_network_manager_unsupported.go @@ -1,4 +1,4 @@ -//go:build darwin || freebsd || netbsd || openbsd +//go:build darwin || netbsd || openbsd /* Copyright The containerd Authors. diff --git a/pkg/defaults/defaults_freebsd.go b/pkg/defaults/defaults_freebsd.go index 620006a5e9a..b83800c907d 100644 --- a/pkg/defaults/defaults_freebsd.go +++ b/pkg/defaults/defaults_freebsd.go @@ -28,8 +28,7 @@ func DataRoot() string { } func CNIPath() string { - // default: /opt/cni/bin - return gocni.DefaultCNIDir + return "/usr/local/libexec/cni" } func CNINetConfPath() string { diff --git a/pkg/ocihook/ocihook.go b/pkg/ocihook/ocihook.go index d49b12bbfb0..733e2620e60 100644 --- a/pkg/ocihook/ocihook.go +++ b/pkg/ocihook/ocihook.go @@ -266,29 +266,6 @@ func getExtraHosts(state *specs.State) (map[string]string, error) { return hosts, nil } -func getNetNSPath(state *specs.State) (string, error) { - // If we have a network-namespace annotation we use it over the passed Pid. - netNsPath, netNsFound := state.Annotations[NetworkNamespace] - if netNsFound { - if _, err := os.Stat(netNsPath); err != nil { - return "", err - } - - return netNsPath, nil - } - - if state.Pid == 0 && !netNsFound { - return "", errors.New("both state.Pid and the netNs annotation are unset") - } - - // We dont't have a networking namespace annotation, but we have a PID. - s := fmt.Sprintf("/proc/%d/ns/net", state.Pid) - if _, err := os.Stat(s); err != nil { - return "", err - } - return s, nil -} - func getPortMapOpts(opts *handlerOpts) ([]gocni.NamespaceOpts, error) { if len(opts.ports) > 0 { if !rootlessutil.IsRootlessChild() { diff --git a/pkg/ocihook/ocihook_freebsd.go b/pkg/ocihook/ocihook_freebsd.go index ef9df017c98..950598499f9 100644 --- a/pkg/ocihook/ocihook_freebsd.go +++ b/pkg/ocihook/ocihook_freebsd.go @@ -16,7 +16,15 @@ package ocihook +import ( + "github.com/opencontainers/runtime-spec/specs-go" +) + func loadAppArmor() { //noop return } + +func getNetNSPath(state *specs.State) (string, error) { + return state.ID, nil +} diff --git a/pkg/ocihook/ocihook_nonfreebsd.go b/pkg/ocihook/ocihook_nonfreebsd.go new file mode 100644 index 00000000000..1429ad3fcce --- /dev/null +++ b/pkg/ocihook/ocihook_nonfreebsd.go @@ -0,0 +1,50 @@ +//go:build !freebsd + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ocihook + +import ( + "errors" + "fmt" + "os" + + "github.com/opencontainers/runtime-spec/specs-go" +) + +func getNetNSPath(state *specs.State) (string, error) { + // If we have a network-namespace annotation we use it over the passed Pid. + netNsPath, netNsFound := state.Annotations[NetworkNamespace] + if netNsFound { + if _, err := os.Stat(netNsPath); err != nil { + return "", err + } + + return netNsPath, nil + } + + if state.Pid == 0 && !netNsFound { + return "", errors.New("both state.Pid and the netNs annotation are unset") + } + + // We dont't have a networking namespace annotation, but we have a PID. + s := fmt.Sprintf("/proc/%d/ns/net", state.Pid) + if _, err := os.Stat(s); err != nil { + return "", err + } + return s, nil +}