FreeBSD: CNI plugins

FreeBSD has the CNI plugins ported:
https://www.freshports.org/net/containernetworking-plugins/. This
allows us to enable CNI networking for FreeBSD containers.

This change adapts the existing linux codebase to work on freebsd:

- containerutil: use nullfs instead of bind mounts for resolv.conf,
  etc.
- ocihook: freebsd's bridge plugin uses jail names in contrast to
  linux's network namespace usages
- container creation: configure runj runtime to create vnet jails by
  default

Signed-off-by: Artem Khramov <akhramov@pm.me>

.github/workflows/test.yml

@@ -262,8 +262,7 @@ jobs:
       - name: Set up vagrant
         run: |
           sudo apt-get update
-          sudo apt-get install -y libvirt-daemon libvirt-daemon-system vagrant vagrant-libvirt
-          sudo systemctl enable --now libvirtd
+          sudo apt-get install -y virtualbox vagrant
       - name: Boot VM
         run: |
           ln -sf Vagrantfile.freebsd Vagrantfile

Vagrantfile.freebsd

@@ -17,28 +17,35 @@
 
 # Vagrantfile for FreeBSD
 Vagrant.configure("2") do |config|
-  config.vm.box = "generic/freebsd13"
+  config.vm.box = "freebsd/FreeBSD-13.2-STABLE"
 
   memory = 2048
   cpus = 1
   config.vm.provider :virtualbox do |v, o|
     v.memory = memory
     v.cpus = cpus
   end
-  config.vm.provider :libvirt do |v|
-    v.memory = memory
-    v.cpus = cpus
-  end
 
   config.vm.synced_folder ".", "/vagrant", type: "rsync"
 
   config.vm.provision "install", type: "shell", run: "once" do |sh|
     sh.inline = <<~SHELL
         #!/usr/bin/env bash
         set -eux -o pipefail
-        pkg install -y go containerd runj
+        pkg install -y go containerd runj containernetworking-plugins
         cd /vagrant
         go install ./cmd/nerdctl
+
+        mkdir -p /etc/nerdctl
+        cat << EOF > /etc/nerdctl/runj.ext.json
+        {
+          "network": {
+            "vnet": {
+              "mode": "new"
+            }
+          }
+        }
+        EOF
     SHELL
   end
 
@@ -55,9 +62,20 @@ Vagrant.configure("2") do |config|
     sh.inline = <<~SHELL
         #!/usr/bin/env bash
         set -eux -o pipefail
+
+        # Firewall config, needed for CNI plugins.
+        cat << EOF > /etc/pf.conf
+        nat on em0 inet from <cni-nat> to any -> (em0)
+        rdr-anchor "cni-rdr/*"
+        table <cni-nat>
+        EOF
+
+        service pf onestart
+
         daemon -o containerd.out containerd
         sleep 3
-        /root/go/bin/nerdctl run --rm --net=none dougrabson/freebsd-minimal:13 echo "Nerdctl is up and running."
+        /root/go/bin/nerdctl run --rm --net=none dougrabson/freebsd-small:13 echo "Nerdctl is up and running."
+        /root/go/bin/nerdctl run --rm dougrabson/freebsd-small:13 nc -zw1 1.1.1.1 443
     SHELL
   end
 

cmd/nerdctl/container_run.go

@@ -51,7 +51,7 @@ func newRunCommand() *cobra.Command {
 		longHelp += "WARNING: `nerdctl run` is experimental on Windows and currently broken (https://github.com/containerd/nerdctl/issues/28)"
 	case "freebsd":
 		longHelp += "\n"
-		longHelp += "WARNING: `nerdctl run` is experimental on FreeBSD and currently requires `--net=none` (https://github.com/containerd/nerdctl/blob/main/docs/freebsd.md)"
+		longHelp += "WARNING: `nerdctl run` is experimental on FreeBSD (https://github.com/containerd/nerdctl/blob/main/docs/freebsd.md)"
 	}
 	var runCommand = &cobra.Command{
 		Use:               "run [flags] IMAGE [COMMAND] [ARG...]",

docs/freebsd.md

@@ -12,22 +12,54 @@ You will need the most up-to-date containerd build along with a containerd shim,
 such as [runj](https://github.com/samuelkarp/runj). Follow the build
 instructions in the respective repositories.
 
+The runj runtime must be configured to create vnet jails by default. To do this, you need to create
+
+`/etc/nerdctl/runj.ext.json` with the following content.
+
+```
+{
+  "network": {
+    "vnet": {
+      "mode": "new"
+    }
+  }
+}
+```
+
 ## Usage
 
 You can use the `dougrabson/freebsd13.2-small` image to run a FreeBSD 13 jail:
 
 ```sh
-nerdctl run --net none -it dougrabson/freebsd13.2-small
+nerdctl run --net=none -it dougrabson/freebsd13.2-small
 ```
 
 Alternatively use `--platform` parameter to run linux containers
 
 ```sh
-nerdctl run --platform linux --net none -it amazonlinux:2
+nerdctl run --platform linux --net=none -it amazonlinux:2
+```
+
+:warning: running linux containers requires `linux64` module loaded:
+
+```
+kldload linux64
 ```
 
 
-## Limitations & Bugs
+## CNI networking
 
-- :warning: CNI & CNI plugins are not yet ported to FreeBSD. The only supported
-  network type is `none`
+| :construction:        CNI networking requires host OS to be version 13.3 and higher. Lower versions are not guaranteed to work. |
+|---------------------------------------------------------------------------------------------------------------------------------|
+
+CNI plugins can be installed from the repository
+
+```sh
+pkg install net/containernetworking-plugins
+```
+
+You can then drop the `--net=none` flag and run commands as usual.
+
+```sh
+nerdctl run -it dougrabson/freebsd13.2-small ping 1.1.1.1
+```

pkg/cmd/container/create.go

@@ -225,11 +225,8 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa
 	}
 	opts = append(opts, umaskOpts...)
 
-	rtCOpts, err := generateRuntimeCOpts(options.GOptions.CgroupManager, options.Runtime)
-	if err != nil {
-		return nil, nil, err
-	}
-	cOpts = append(cOpts, rtCOpts...)
+	rtOpts := generateRuntimeOpts(options)
+	cOpts = append(cOpts, rtOpts)
 
 	lCOpts, err := withContainerLabels(options.Label, options.LabelFile)
 	if err != nil {

pkg/cmd/container/run_runtime.go

@@ -23,37 +23,53 @@ import (
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/containers"
 	"github.com/containerd/containerd/oci"
+	runtimeoptions "github.com/containerd/containerd/pkg/runtimeoptions/v1"
 	"github.com/containerd/containerd/plugin"
 	runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
+	"github.com/containerd/nerdctl/pkg/api/types"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
 )
 
-func generateRuntimeCOpts(cgroupManager, runtimeStr string) ([]containerd.NewContainerOpts, error) {
+func generateRuntimeOpts(options types.ContainerCreateOptions) containerd.NewContainerOpts {
+	runtimeOpts := generateRuntimeCOpts(options.GOptions.CgroupManager, options.Runtime)
+
+	if options.Runtime == "wtf.sbk.runj.v1" {
+		runtimeOpts = generateRunjOpts()
+	}
+
+	return runtimeOpts
+}
+
+func generateRunjOpts() containerd.NewContainerOpts {
+	return containerd.WithRuntime("wtf.sbk.runj.v1", &runtimeoptions.Options{
+		ConfigPath: "/etc/nerdctl/runj.ext.json",
+	})
+}
+
+func generateRuntimeCOpts(cgroupManager, runtimeStr string) containerd.NewContainerOpts {
 	runtime := plugin.RuntimeRuncV2
-	var (
-		runcOpts    runcoptions.Options
-		runtimeOpts interface{} = &runcOpts
-	)
+	var runcOpts runcoptions.Options
+
 	if cgroupManager == "systemd" {
 		runcOpts.SystemdCgroup = true
 	}
 	if runtimeStr != "" {
-		if strings.HasPrefix(runtimeStr, "io.containerd.") || runtimeStr == "wtf.sbk.runj.v1" {
+		if strings.HasPrefix(runtimeStr, "io.containerd.") {
 			runtime = runtimeStr
 			if !strings.HasPrefix(runtimeStr, "io.containerd.runc.") {
 				if cgroupManager == "systemd" {
 					logrus.Warnf("cannot set cgroup manager to %q for runtime %q", cgroupManager, runtimeStr)
 				}
-				runtimeOpts = nil
+				return nil
 			}
 		} else {
 			// runtimeStr is a runc binary
 			runcOpts.BinaryName = runtimeStr
 		}
 	}
-	o := containerd.WithRuntime(runtime, runtimeOpts)
-	return []containerd.NewContainerOpts{o}, nil
+
+	return containerd.WithRuntime(runtime, &runcOpts)
 }
 
 // WithSysctls sets the provided sysctls onto the spec

pkg/containerutil/container_network_manager.go

@@ -49,36 +49,21 @@ const (
 
 func withCustomResolvConf(src string) func(context.Context, oci.Client, *containers.Container, *oci.Spec) error {
 	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error {
-		s.Mounts = append(s.Mounts, specs.Mount{
-			Destination: "/etc/resolv.conf",
-			Type:        "bind",
-			Source:      src,
-			Options:     []string{"bind", mountutil.DefaultPropagationMode}, // writable
-		})
+		s.Mounts = append(s.Mounts, bindMount(src, "/etc/resolv.conf"))
 		return nil
 	}
 }
 
 func withCustomEtcHostname(src string) func(context.Context, oci.Client, *containers.Container, *oci.Spec) error {
 	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error {
-		s.Mounts = append(s.Mounts, specs.Mount{
-			Destination: "/etc/hostname",
-			Type:        "bind",
-			Source:      src,
-			Options:     []string{"bind", mountutil.DefaultPropagationMode}, // writable
-		})
+		s.Mounts = append(s.Mounts, bindMount(src, "/etc/hostname"))
 		return nil
 	}
 }
 
 func withCustomHosts(src string) func(context.Context, oci.Client, *containers.Container, *oci.Spec) error {
 	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error {
-		s.Mounts = append(s.Mounts, specs.Mount{
-			Destination: "/etc/hosts",
-			Type:        "bind",
-			Source:      src,
-			Options:     []string{"bind", mountutil.DefaultPropagationMode}, // writable
-		})
+		s.Mounts = append(s.Mounts, bindMount(src, "/etc/hosts"))
 		return nil
 	}
 }
@@ -556,3 +541,20 @@ func nonZeroMapValues(values map[string]interface{}) []string {
 
 	return nonZero
 }
+
+func bindMount(source, destination string) specs.Mount {
+	fstype := "bind"
+	options := []string{"bind", mountutil.DefaultPropagationMode}
+
+	if runtime.GOOS == "freebsd" {
+		fstype = "nullfs"
+		options = []string{"rw"}
+	}
+
+	return specs.Mount{
+		Destination: destination,
+		Type:        fstype,
+		Source:      source,
+		Options:     options, // writable
+	}
+}

pkg/containerutil/container_network_manager_linux.go → pkg/containerutil/container_network_manager_unix.go

@@ -1,3 +1,5 @@
+//go:build linux || freebsd
+
 /*
    Copyright The containerd Authors.
 

pkg/containerutil/container_network_manager_other.go → pkg/containerutil/container_network_manager_unsupported.go

@@ -1,4 +1,4 @@
-//go:build darwin || freebsd || netbsd || openbsd
+//go:build darwin || netbsd || openbsd
 
 /*
    Copyright The containerd Authors.

pkg/defaults/defaults_freebsd.go

@@ -28,8 +28,7 @@ func DataRoot() string {
 }
 
 func CNIPath() string {
-	// default: /opt/cni/bin
-	return gocni.DefaultCNIDir
+	return "/usr/local/libexec/cni"
 }
 
 func CNINetConfPath() string {

pkg/ocihook/ocihook.go

@@ -266,29 +266,6 @@ func getExtraHosts(state *specs.State) (map[string]string, error) {
 	return hosts, nil
 }
 
-func getNetNSPath(state *specs.State) (string, error) {
-	// If we have a network-namespace annotation we use it over the passed Pid.
-	netNsPath, netNsFound := state.Annotations[NetworkNamespace]
-	if netNsFound {
-		if _, err := os.Stat(netNsPath); err != nil {
-			return "", err
-		}
-
-		return netNsPath, nil
-	}
-
-	if state.Pid == 0 && !netNsFound {
-		return "", errors.New("both state.Pid and the netNs annotation are unset")
-	}
-
-	// We dont't have a networking namespace annotation, but we have a PID.
-	s := fmt.Sprintf("/proc/%d/ns/net", state.Pid)
-	if _, err := os.Stat(s); err != nil {
-		return "", err
-	}
-	return s, nil
-}
-
 func getPortMapOpts(opts *handlerOpts) ([]gocni.NamespaceOpts, error) {
 	if len(opts.ports) > 0 {
 		if !rootlessutil.IsRootlessChild() {

pkg/ocihook/ocihook_freebsd.go

@@ -16,7 +16,15 @@
 
 package ocihook
 
+import (
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
 func loadAppArmor() {
 	//noop
 	return
 }
+
+func getNetNSPath(state *specs.State) (string, error) {
+	return state.ID, nil
+}

pkg/ocihook/ocihook_nonfreebsd.go

@@ -0,0 +1,50 @@
+//go:build !freebsd
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ocihook
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+func getNetNSPath(state *specs.State) (string, error) {
+	// If we have a network-namespace annotation we use it over the passed Pid.
+	netNsPath, netNsFound := state.Annotations[NetworkNamespace]
+	if netNsFound {
+		if _, err := os.Stat(netNsPath); err != nil {
+			return "", err
+		}
+
+		return netNsPath, nil
+	}
+
+	if state.Pid == 0 && !netNsFound {
+		return "", errors.New("both state.Pid and the netNs annotation are unset")
+	}
+
+	// We dont't have a networking namespace annotation, but we have a PID.
+	s := fmt.Sprintf("/proc/%d/ns/net", state.Pid)
+	if _, err := os.Stat(s); err != nil {
+		return "", err
+	}
+	return s, nil
+}