Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement network plugin #7947

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions cmd/containerd/builtins/builtins.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
_ "github.com/containerd/containerd/gc/scheduler"
_ "github.com/containerd/containerd/leases/plugin"
_ "github.com/containerd/containerd/metadata/plugin"
_ "github.com/containerd/containerd/pkg/net/plugin"
_ "github.com/containerd/containerd/pkg/nri/plugin"
_ "github.com/containerd/containerd/plugins/sandbox"
_ "github.com/containerd/containerd/plugins/streaming"
Expand Down
2 changes: 1 addition & 1 deletion contrib/fuzz/cri_server_fuzzer.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func FuzzCRIServer(data []byte) int {
}
defer client.Close()

c, err := server.NewCRIService(criconfig.Config{}, client, nil)
c, err := server.NewCRIService(criconfig.Config{}, client, nil, nil)
if err != nil {
panic(err)
}
Expand Down
2 changes: 1 addition & 1 deletion integration/image_pull_timeout_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -471,5 +471,5 @@ func initLocalCRIPlugin(client *containerd.Client, tmpDir string, registryCfg cr
RootDir: filepath.Join(criWorkDir, "root"),
StateDir: filepath.Join(criWorkDir, "state"),
}
return criserver.NewCRIService(cfg, client, nil)
return criserver.NewCRIService(cfg, client, nil, nil)
}
2 changes: 2 additions & 0 deletions metadata/gc.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ const (
resourceEnd
// ResourceStream specifies a stream
ResourceStream
// ResourceNetwork specifies a network resource
ResourceNetwork
)

const (
Expand Down
35 changes: 34 additions & 1 deletion pkg/cri/cri.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/containerd/containerd"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/pkg/cri/sbserver"
"github.com/containerd/containerd/pkg/net/compat"
"github.com/containerd/containerd/pkg/nri"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/plugin"
Expand All @@ -48,6 +49,7 @@ func init() {
plugin.EventPlugin,
plugin.ServicePlugin,
plugin.NRIApiPlugin,
plugin.NetworkPlugin,
},
InitFn: initCRIService,
})
Expand Down Expand Up @@ -88,6 +90,7 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {

var s server.CRIService
var nrip nri.API
var netp compat.API
if os.Getenv("ENABLE_CRI_SANDBOXES") != "" {
log.G(ctx).Info("using experimental CRI Sandbox server - unset ENABLE_CRI_SANDBOXES to disable")
s, err = sbserver.NewCRIService(c, client)
Expand All @@ -99,7 +102,12 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
log.G(ctx).Info("NRI service not found, disabling NRI support")
}

s, err = server.NewCRIService(c, client, nrip)
netp, err = getNetworkPlugin(ic)
if err == nil {
log.G(ctx).Info("using experimental network plugin")
}

s, err = server.NewCRIService(c, client, nrip, netp)
}
if err != nil {
return nil, fmt.Errorf("failed to create CRI service: %w", err)
Expand Down Expand Up @@ -166,3 +174,28 @@ func getNRIPlugin(ic *plugin.InitContext) (nri.API, error) {

return api, nil
}

// Get the Network plugin and verify its type.
func getNetworkPlugin(ic *plugin.InitContext) (compat.API, error) {
const (
pluginType = plugin.NetworkPlugin
pluginName = "cni"
)

if os.Getenv("ENABLE_NETWORK_SRV") == "" {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe that this is inspired by ENABLE_CRI_SANDBOXES. But do you think we cannot make this stable enough before releasing 1.7.0? My worry is that nobody use this if that is opt-in.

return nil, nil
}

p, err := ic.GetByID(pluginType, pluginName)
if err != nil {
return nil, err
}

api, ok := p.(compat.API)
if !ok {
return nil, fmt.Errorf("network plugin (%s, %q) has incompatible type %T",
pluginType, pluginName, api)
}

return api, nil
}
13 changes: 7 additions & 6 deletions pkg/cri/server/cni_conf_syncer.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ import (
"path/filepath"
"sync"

cni "github.com/containerd/go-cni"
"github.com/containerd/containerd/pkg/cri/util"
"github.com/containerd/containerd/pkg/net/compat"
"github.com/fsnotify/fsnotify"
"github.com/sirupsen/logrus"
)
Expand All @@ -36,12 +37,12 @@ type cniNetConfSyncer struct {

watcher *fsnotify.Watcher
confDir string
netPlugin cni.CNI
loadOpts []cni.Opt
netPlugin compat.CNI
loadOpts []compat.LoadOpt
}

// newCNINetConfSyncer creates cni network conf syncer.
func newCNINetConfSyncer(confDir string, netPlugin cni.CNI, loadOpts []cni.Opt) (*cniNetConfSyncer, error) {
func newCNINetConfSyncer(confDir string, netPlugin compat.CNI, loadOpts []compat.LoadOpt) (*cniNetConfSyncer, error) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would we eventually use github.com/containerd/containerd/pkg/net instead of github.com/containerd/containerd/pkg/net/compat?

compat.CNI is fine, but types like compat.LoadOpt, compat.API don't tell the intention much. So it may be better to rename the package or merge that into net.

watcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, fmt.Errorf("failed to create fsnotify watcher: %w", err)
Expand Down Expand Up @@ -69,7 +70,7 @@ func newCNINetConfSyncer(confDir string, netPlugin cni.CNI, loadOpts []cni.Opt)
loadOpts: loadOpts,
}

if err := syncer.netPlugin.Load(syncer.loadOpts...); err != nil {
if err := syncer.netPlugin.Load(util.NamespacedContext(), syncer.loadOpts...); err != nil {
logrus.WithError(err).Error("failed to load cni during init, please check CRI plugin status before setting up network for pods")
syncer.updateLastStatus(err)
}
Expand Down Expand Up @@ -97,7 +98,7 @@ func (syncer *cniNetConfSyncer) syncLoop() error {
}
logrus.Debugf("receiving change event from cni conf dir: %s", event)

lerr := syncer.netPlugin.Load(syncer.loadOpts...)
lerr := syncer.netPlugin.Load(util.NamespacedContext(), syncer.loadOpts...)
if lerr != nil {
logrus.WithError(lerr).
Errorf("failed to reload cni configuration after receiving fs change event(%s)", event)
Expand Down
224 changes: 224 additions & 0 deletions pkg/cri/server/netapi_adaptor.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,224 @@
/*
Copyright The containerd Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
"context"
"reflect"
"runtime"
"strings"

"github.com/containerd/containerd/pkg/net"
"github.com/containerd/containerd/pkg/net/compat"
"github.com/containerd/go-cni"
)

// cniAdaptor is created to adapt the APIs of network plugin to their
// counterparts in go-cni.
type cniAdaptor struct {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The adapter is for both g and c. Isn't cni.CNI and compat.CNI mostly compatible?

adapt bool
g cni.CNI
c compat.CNI
}

var _ compat.CNI = (*cniAdaptor)(nil)

//nolint:nolintlint,unused
func newCNIAdaptor(netAPI compat.API, name string, opts ...compat.Opt) (*cniAdaptor, error) {
var err error

c := &cniAdaptor{
adapt: false,
}

if netAPI == nil {
c.adapt = true
}

if c.adapt {
dopts, err := convertOpts(opts)
if err != nil {
return nil, err
}
if c.g, err = cni.New(dopts...); err != nil {
return nil, err
}
} else {
if c.c, err = netAPI.NewCNI(name, opts...); err != nil {
return nil, err
}
}

return c, nil
}

func (c *cniAdaptor) Setup(ctx context.Context, id string, path string, opts ...net.AttachmentOpt) (*compat.Result, error) {
if !c.adapt {
return c.c.Setup(ctx, id, path, opts...)
}

dopts, err := convertNamespaceOpts(opts)
if err != nil {
return nil, err
}

r, err := c.g.Setup(ctx, id, path, dopts...)
if err != nil {
return nil, err
}

return compat.WrapResult(r), nil
}

func (c *cniAdaptor) SetupSerially(ctx context.Context, id string, path string, opts ...net.AttachmentOpt) (*compat.Result, error) {
if !c.adapt {
return c.c.SetupSerially(ctx, id, path, opts...)
}

dopts, err := convertNamespaceOpts(opts)
if err != nil {
return nil, err
}

r, err := c.g.Setup(ctx, id, path, dopts...)
if err != nil {
return nil, err
}

return compat.WrapResult(r), nil
}

func (c *cniAdaptor) Remove(ctx context.Context, id string, path string, opts ...net.AttachmentOpt) error {
if !c.adapt {
return c.c.Remove(ctx, id, path, opts...)
}

dopts, err := convertNamespaceOpts(opts)
if err != nil {
return err
}

return c.g.Remove(ctx, id, path, dopts...)
}

func (c *cniAdaptor) Check(ctx context.Context, id string, path string, opts ...net.AttachmentOpt) error {
if !c.adapt {
return c.c.Check(ctx, id, path, opts...)
}

dopts, err := convertNamespaceOpts(opts)
if err != nil {
return err
}

return c.g.Check(ctx, id, path, dopts...)
}

func (c *cniAdaptor) Load(ctx context.Context, opts ...compat.LoadOpt) error {
if !c.adapt {
return c.c.Load(ctx, opts...)
}

var dopts []cni.Opt
for _, o := range opts {
name := getFunctionName(o)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd like to avoid reflection if possible. Why do we need to know the name?

sl := strings.Split(name, "/")
switch sl[len(sl)-1] {
case "compat.WithLoNetwork":
dopts = append(dopts, cni.WithLoNetwork)
case "compat.WithDefaultConf":
dopts = append(dopts, cni.WithDefaultConf)
}
}

return c.g.Load(dopts...)
}

func (c *cniAdaptor) Status(ctx context.Context) error {
if !c.adapt {
return c.c.Status(ctx)
}
return c.g.Status()
}

func (c *cniAdaptor) GetConfig(ctx context.Context) *cni.ConfigResult {
if !c.adapt {
return c.c.GetConfig(ctx)
}
return c.g.GetConfig()
}

//nolint:nolintlint,unused
func convertOpts(opts []compat.Opt) ([]cni.Opt, error) {
var (
cfg compat.Config
dopts []cni.Opt
)

for _, o := range opts {
if err := o(&cfg); err != nil {
return dopts, err
}
}

if len(cfg.PluginDirs) > 0 {
dopts = append(dopts, cni.WithPluginDir(cfg.PluginDirs))
}
if len(cfg.PluginConfDir) > 0 {
dopts = append(dopts, cni.WithPluginConfDir(cfg.PluginConfDir))
}
if cfg.PluginMaxConfNum > 0 {
dopts = append(dopts, cni.WithPluginMaxConfNum(cfg.PluginMaxConfNum))
}
if len(cfg.Prefix) > 0 {
dopts = append(dopts, cni.WithInterfacePrefix(cfg.Prefix))
}
if cfg.NetworkCount > 0 {
dopts = append(dopts, cni.WithMinNetworkCount(cfg.NetworkCount))
}

return dopts, nil
}

func convertNamespaceOpts(opts []net.AttachmentOpt) ([]cni.NamespaceOpts, error) {
var dopts []cni.NamespaceOpts

args := net.AttachmentArgs{
CapabilityArgs: make(map[string]interface{}),
PluginArgs: make(map[string]string),
}

for _, o := range opts {
if err := o(&args); err != nil {
return dopts, err
}
}

for k, v := range args.PluginArgs {
dopts = append(dopts, cni.WithArgs(k, v))
}

for k, v := range args.CapabilityArgs {
dopts = append(dopts, cni.WithCapability(k, v))
}

return dopts, nil
}

func getFunctionName(i interface{}) string {
return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
}