Skip to content

Commit

Permalink
Implement network plugin
Browse files Browse the repository at this point in the history
Signed-off-by: Henry Wang <henwang@amazon.com>
  • Loading branch information
henry118 committed Jan 10, 2023
1 parent 54ec191 commit 04ce65c
Show file tree
Hide file tree
Showing 26 changed files with 2,941 additions and 47 deletions.
1 change: 1 addition & 0 deletions cmd/containerd/builtins/builtins.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
_ "github.com/containerd/containerd/gc/scheduler"
_ "github.com/containerd/containerd/leases/plugin"
_ "github.com/containerd/containerd/metadata/plugin"
_ "github.com/containerd/containerd/pkg/net/plugin"
_ "github.com/containerd/containerd/pkg/nri/plugin"
_ "github.com/containerd/containerd/plugins/sandbox"
_ "github.com/containerd/containerd/plugins/streaming"
Expand Down
2 changes: 1 addition & 1 deletion integration/image_pull_timeout_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -471,5 +471,5 @@ func initLocalCRIPlugin(client *containerd.Client, tmpDir string, registryCfg cr
RootDir: filepath.Join(criWorkDir, "root"),
StateDir: filepath.Join(criWorkDir, "state"),
}
return criserver.NewCRIService(cfg, client, nil)
return criserver.NewCRIService(cfg, client, nil, nil)
}
34 changes: 33 additions & 1 deletion pkg/cri/cri.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/containerd/containerd"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/pkg/cri/sbserver"
"github.com/containerd/containerd/pkg/net/compat"
"github.com/containerd/containerd/pkg/nri"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/plugin"
Expand Down Expand Up @@ -88,6 +89,7 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {

var s server.CRIService
var nrip nri.API
var netp compat.API
if os.Getenv("ENABLE_CRI_SANDBOXES") != "" {
log.G(ctx).Info("using experimental CRI Sandbox server - unset ENABLE_CRI_SANDBOXES to disable")
s, err = sbserver.NewCRIService(c, client)
Expand All @@ -99,7 +101,12 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
log.G(ctx).Info("NRI service not found, disabling NRI support")
}

s, err = server.NewCRIService(c, client, nrip)
netp, err = getNetworkPlugin(ic)
if err == nil {
log.G(ctx).Info("using experimental network plugin")
}

s, err = server.NewCRIService(c, client, nrip, netp)
}
if err != nil {
return nil, fmt.Errorf("failed to create CRI service: %w", err)
Expand Down Expand Up @@ -166,3 +173,28 @@ func getNRIPlugin(ic *plugin.InitContext) (nri.API, error) {

return api, nil
}

// Get the Network plugin and verify its type.
func getNetworkPlugin(ic *plugin.InitContext) (compat.API, error) {
const (
pluginType = plugin.NetworkPlugin
pluginName = "cni"
)

if os.Getenv("ENABLE_NETWORK_SRV") == "" {
return nil, nil
}

p, err := ic.GetByID(pluginType, pluginName)
if err != nil {
return nil, err
}

api, ok := p.(compat.API)
if !ok {
return nil, fmt.Errorf("network plugin (%s, %q) has incompatible type %T",
pluginType, pluginName, api)
}

return api, nil
}
13 changes: 7 additions & 6 deletions pkg/cri/server/cni_conf_syncer.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ import (
"os"
"sync"

cni "github.com/containerd/go-cni"
"github.com/containerd/containerd/pkg/cri/util"
"github.com/containerd/containerd/pkg/net/compat"
"github.com/fsnotify/fsnotify"
"github.com/sirupsen/logrus"
)
Expand All @@ -35,12 +36,12 @@ type cniNetConfSyncer struct {

watcher *fsnotify.Watcher
confDir string
netPlugin cni.CNI
loadOpts []cni.Opt
netPlugin compat.CNI
loadOpts []compat.LoadOpt
}

// newCNINetConfSyncer creates cni network conf syncer.
func newCNINetConfSyncer(confDir string, netPlugin cni.CNI, loadOpts []cni.Opt) (*cniNetConfSyncer, error) {
func newCNINetConfSyncer(confDir string, netPlugin compat.CNI, loadOpts []compat.LoadOpt) (*cniNetConfSyncer, error) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, fmt.Errorf("failed to create fsnotify watcher: %w", err)
Expand All @@ -61,7 +62,7 @@ func newCNINetConfSyncer(confDir string, netPlugin cni.CNI, loadOpts []cni.Opt)
loadOpts: loadOpts,
}

if err := syncer.netPlugin.Load(syncer.loadOpts...); err != nil {
if err := syncer.netPlugin.Load(util.NamespacedContext(), syncer.loadOpts...); err != nil {
logrus.WithError(err).Error("failed to load cni during init, please check CRI plugin status before setting up network for pods")
syncer.updateLastStatus(err)
}
Expand Down Expand Up @@ -89,7 +90,7 @@ func (syncer *cniNetConfSyncer) syncLoop() error {
}
logrus.Debugf("receiving change event from cni conf dir: %s", event)

lerr := syncer.netPlugin.Load(syncer.loadOpts...)
lerr := syncer.netPlugin.Load(util.NamespacedContext(), syncer.loadOpts...)
if lerr != nil {
logrus.WithError(lerr).
Errorf("failed to reload cni configuration after receiving fs change event(%s)", event)
Expand Down
206 changes: 206 additions & 0 deletions pkg/cri/server/netapi_adaptor.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,206 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
"context"

"github.com/containerd/containerd/pkg/net"
"github.com/containerd/containerd/pkg/net/compat"
"github.com/containerd/go-cni"
)

// cniAdaptor is created to adapt the APIs of network plugin to their
// counterparts in go-cni.
type cniAdaptor struct {
adapt bool
g cni.CNI
c compat.CNI
}

var _ compat.CNI = (*cniAdaptor)(nil)

func newCNIAdaptor(netAPI compat.API, name string, opts ...compat.Opt) (*cniAdaptor, error) {
var err error

c := &cniAdaptor{
adapt: false,
}

if netAPI == nil {
c.adapt = true
}

if c.adapt {
dopts, err := convertOpts(opts)
if err != nil {
return nil, err
}
if c.g, err = cni.New(dopts...); err != nil {
return nil, err
}
} else {
if c.c, err = netAPI.NewCNI(name, opts...); err != nil {
return nil, err
}
}

return c, nil
}

func (c *cniAdaptor) Setup(ctx context.Context, id string, path string, opts ...net.AttachmentOpt) (*compat.Result, error) {
if !c.adapt {
return c.c.Setup(ctx, id, path, opts...)
}

dopts, err := convertNamespaceOpts(opts)
if err != nil {
return nil, err
}

r, err := c.g.Setup(ctx, id, path, dopts...)
if err != nil {
return nil, err
}

return compat.WrapResult(r), nil
}

func (c *cniAdaptor) SetupSerially(ctx context.Context, id string, path string, opts ...net.AttachmentOpt) (*compat.Result, error) {
if !c.adapt {
return c.c.SetupSerially(ctx, id, path, opts...)
}

dopts, err := convertNamespaceOpts(opts)
if err != nil {
return nil, err
}

r, err := c.g.Setup(ctx, id, path, dopts...)
if err != nil {
return nil, err
}

return compat.WrapResult(r), nil
}

func (c *cniAdaptor) Remove(ctx context.Context, id string, path string, opts ...net.AttachmentOpt) error {
if !c.adapt {
return c.c.Remove(ctx, id, path, opts...)
}

dopts, err := convertNamespaceOpts(opts)
if err != nil {
return err
}

return c.g.Remove(ctx, id, path, dopts...)
}

func (c *cniAdaptor) Check(ctx context.Context, id string, path string, opts ...net.AttachmentOpt) error {
if !c.adapt {
return c.c.Check(ctx, id, path, opts...)
}

dopts, err := convertNamespaceOpts(opts)
if err != nil {
return err
}

return c.g.Check(ctx, id, path, dopts...)
}

func (c *cniAdaptor) Load(ctx context.Context, opts ...compat.LoadOpt) error {
if !c.adapt {
return c.c.Load(ctx, opts...)
}

// it's hard to convert load opts. diretly use the code in CRI
dopts := []cni.Opt{cni.WithLoNetwork, cni.WithDefaultConf}

return c.g.Load(dopts...)
}

func (c *cniAdaptor) Status(ctx context.Context) error {
if !c.adapt {
return c.c.Status(ctx)
}
return c.g.Status()
}

func (c *cniAdaptor) GetConfig(ctx context.Context) *cni.ConfigResult {
if !c.adapt {
return c.c.GetConfig(ctx)
}
return c.g.GetConfig()
}

func convertOpts(opts []compat.Opt) ([]cni.Opt, error) {
var (
cfg compat.Config
dopts []cni.Opt
)

for _, o := range opts {
if err := o(&cfg); err != nil {
return dopts, err
}
}

if len(cfg.PluginDirs) > 0 {
dopts = append(dopts, cni.WithPluginDir(cfg.PluginDirs))
}
if len(cfg.PluginConfDir) > 0 {
dopts = append(dopts, cni.WithPluginConfDir(cfg.PluginConfDir))
}
if cfg.PluginMaxConfNum > 0 {
dopts = append(dopts, cni.WithPluginMaxConfNum(cfg.PluginMaxConfNum))
}
if len(cfg.Prefix) > 0 {
dopts = append(dopts, cni.WithInterfacePrefix(cfg.Prefix))
}
if cfg.NetworkCount > 0 {
dopts = append(dopts, cni.WithMinNetworkCount(cfg.NetworkCount))
}

return dopts, nil
}

func convertNamespaceOpts(opts []net.AttachmentOpt) ([]cni.NamespaceOpts, error) {
var dopts []cni.NamespaceOpts

args := net.AttachmentArgs{
CapabilityArgs: make(map[string]interface{}),
PluginArgs: make(map[string]string),
}

for _, o := range opts {
if err := o(&args); err != nil {
return dopts, err
}
}

for k, v := range args.PluginArgs {
dopts = append(dopts, cni.WithArgs(k, v))
}

for k, v := range args.CapabilityArgs {
dopts = append(dopts, cni.WithCapability(k, v))
}

return dopts, nil
}

0 comments on commit 04ce65c

Please sign in to comment.