Skip to content

CNI changes to support baremetal scenario #835

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Mar 26, 2021
Merged

CNI changes to support baremetal scenario #835

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
285352c
Modify CNI for windows baremetal scenario in Azure
nagasurya Feb 17, 2021
93004ff
Fix the CNI result for baremetal case
nagasurya Mar 2, 2021
6459a95
Merge branch 'master' of https://github.com/Azure/azure-container-net…
nagasurya Mar 15, 2021
cb59245
Addressed PR comments from Tamilmani and Ramiro
nagasurya Mar 16, 2021
de11edd
Remove azure-telemetry.exe from cni-baremetal package
nagasurya Mar 16, 2021
0ecc686
Fix formatting in logging
nagasurya Mar 23, 2021
08a78d7
Merge branch 'master' into cni-baremetal-support
nagasurya Mar 24, 2021
b2e6ad9
Addressed Ramiro's PR review comments
nagasurya Mar 25, 2021
8584bf4
Merge pull request #1 from msvik/cni-baremetal
msvik Mar 25, 2021
99995ab
Formatting fix in printf
nagasurya Mar 26, 2021
39a7f0b
Merge pull request #2 from msvik/cni-baremetal
msvik Mar 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
14 changes: 13 additions & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ CNI_BUILD_DIR = $(BUILD_DIR)/cni
ACNCLI_BUILD_DIR = $(BUILD_DIR)/acncli
CNI_MULTITENANCY_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy
CNI_SWIFT_BUILD_DIR = $(BUILD_DIR)/cni-swift
CNI_BAREMETAL_BUILD_DIR = $(BUILD_DIR)/cni-baremetal
CNS_BUILD_DIR = $(BUILD_DIR)/cns
CNMS_BUILD_DIR = $(BUILD_DIR)/cnms
NPM_BUILD_DIR = $(BUILD_DIR)/npm
Expand Down Expand Up @@ -124,6 +125,7 @@ CNI_ARCHIVE_NAME = azure-vnet-cni-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
ACNCLI_ARCHIVE_NAME = acncli-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
CNI_MULTITENANCY_ARCHIVE_NAME = azure-vnet-cni-multitenancy-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
CNI_SWIFT_ARCHIVE_NAME = azure-vnet-cni-swift-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
CNI_BAREMETAL_ARCHIVE_NAME = azure-vnet-cni-baremetal-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
CNS_ARCHIVE_NAME = azure-cns-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
CNMS_ARCHIVE_NAME = azure-cnms-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
NPM_ARCHIVE_NAME = azure-npm-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT)
Expand Down Expand Up @@ -396,6 +398,16 @@ cni-archive:
cd $(CNI_MULTITENANCY_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_MULTITENANCY_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config
chown $(BUILD_USER):$(BUILD_USER) $(CNI_MULTITENANCY_BUILD_DIR)/$(CNI_MULTITENANCY_ARCHIVE_NAME)

#baremetal mode is windows only (at least for now)
ifeq ($(GOOS),windows)
mkdir -p $(CNI_BAREMETAL_BUILD_DIR)
cp cni/azure-$(GOOS)-baremetal.conflist $(CNI_BAREMETAL_BUILD_DIR)/10-azure.conflist
cp $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BAREMETAL_BUILD_DIR)
chmod 0755 $(CNI_BAREMETAL_BUILD_DIR)/azure-vnet$(EXE_EXT)
cd $(CNI_BAREMETAL_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_BAREMETAL_ARCHIVE_NAME) azure-vnet$(EXE_EXT) 10-azure.conflist
chown $(BUILD_USER):$(BUILD_USER) $(CNI_BAREMETAL_BUILD_DIR)/$(CNI_BAREMETAL_ARCHIVE_NAME)
endif

#swift mode is linux only
ifeq ($(GOOS),linux)
mkdir -p $(CNI_SWIFT_BUILD_DIR)
Expand Down Expand Up @@ -462,4 +474,4 @@ test-all:
# run all tests
.PHONY: test-integration
test-integration:
go test -coverpkg=./... -v -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration...
go test -coverpkg=./... -v -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration...
50 changes: 50 additions & 0 deletions cni/azure-windows-baremetal.conflist
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
{
"cniVersion": "0.3.0",
"name": "azure",
"plugins": [
{
"type": "azure-vnet",
"mode": "bridge",
"bridge": "azure0",
"multiTenancy":false,
"enableSnatOnHost":true,
"enableExactMatchForPodName": true,
"executionMode": "baremetal",
"capabilities": {
"portMappings": true
},
"ipam": {
"type": "azure-vnet-ipam"
},
"dns": {
"Nameservers": [
"10.0.0.10",
"168.63.129.16"
],
"Search": [
"svc.cluster.local"
]
},
"AdditionalArgs": [
{
"Name": "EndpointPolicy",
"Value": {
"Type": "OutBoundNAT",
"ExceptionList": [
"10.240.0.0/16",
"10.0.0.0/8"
]
}
},
{
"Name": "EndpointPolicy",
"Value": {
"Type": "ROUTE",
"DestinationPrefix": "10.0.0.0/8",
"NeedEncap": true
}
}
]
}
]
}
3 changes: 2 additions & 1 deletion cni/netconfig.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ type NetworkConfig struct {
DisableHairpinOnHostInterface bool `json:"disableHairpinOnHostInterface,omitempty"`
DisableIPTableLock bool `json:"disableIPTableLock,omitempty"`
CNSUrl string `json:"cnsurl,omitempty"`
ExecutionMode string `json:"executionMode,omitempty"`
Ipam struct {
Type string `json:"type"`
Environment string `json:"environment,omitempty"`
Expand Down Expand Up @@ -130,4 +131,4 @@ func GetPoliciesFromNwCfg(kvp []KVPair) []policy.Policy {
func (nwcfg *NetworkConfig) Serialize() []byte {
bytes, _ := json.Marshal(nwcfg)
return bytes
}
}
138 changes: 116 additions & 22 deletions cni/network/network.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package network

import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
Expand All @@ -24,6 +25,7 @@ import (
"github.com/Azure/azure-container-networking/network"
"github.com/Azure/azure-container-networking/network/policy"
"github.com/Azure/azure-container-networking/platform"
nnscontracts "github.com/Azure/azure-container-networking/proto/nodenetworkservice/3.302.0.744"
"github.com/Azure/azure-container-networking/telemetry"
cniSkel "github.com/containernetworking/cni/pkg/skel"
cniTypes "github.com/containernetworking/cni/pkg/types"
Expand All @@ -34,8 +36,8 @@ const (
dockerNetworkOption = "com.docker.network.generic"
opModeTransparent = "transparent"
// Supported IP version. Currently support only IPv4
ipVersion = "4"
ipamV6 = "azure-vnet-ipamv6"
ipVersion = "4"
ipamV6 = "azure-vnet-ipamv6"
)

// CNI Operation Types
Expand All @@ -60,13 +62,34 @@ const (
jsonFileExtension = ".json"
)

type ExecutionMode string

const (
Default ExecutionMode = "default"
Baremetal ExecutionMode = "baremetal"
)

// NetPlugin represents the CNI network plugin.
type netPlugin struct {
*cni.Plugin
nm network.NetworkManager
ipamInvoker IPAMInvoker
report *telemetry.CNIReport
tb *telemetry.TelemetryBuffer
nnsClient NnsClient
}

// client for node network service
type NnsClient interface {
// Do network port programming for the pod via node network service.
// podName - name of the pod as received from containerD
// nwNamesapce - network namespace name as received from containerD
AddContainerNetworking(ctx context.Context, podName, nwNamespace string) (error, *nnscontracts.ConfigureContainerNetworkingResponse)

// Undo or delete network port programming for the pod via node network service.
// podName - name of the pod as received from containerD
// nwNamesapce - network namespace name as received from containerD
DeleteContainerNetworking(ctx context.Context, podName, nwNamespace string) (error, *nnscontracts.ConfigureContainerNetworkingResponse)
}

// snatConfiguration contains a bool that determines whether CNI enables snat on host and snat for dns
Expand All @@ -76,7 +99,7 @@ type snatConfiguration struct {
}

// NewPlugin creates a new netPlugin object.
func NewPlugin(name string, config *common.PluginConfig) (*netPlugin, error) {
func NewPlugin(name string, config *common.PluginConfig, client NnsClient) (*netPlugin, error) {
// Setup base plugin.
plugin, err := cni.NewPlugin(name, config.Version)
if err != nil {
Expand All @@ -92,8 +115,9 @@ func NewPlugin(name string, config *common.PluginConfig) (*netPlugin, error) {
config.NetApi = nm

return &netPlugin{
Plugin: plugin,
nm: nm,
Plugin: plugin,
nm: nm,
nnsClient: client,
}, nil
}

Expand Down Expand Up @@ -342,11 +366,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {

plugin.report.ContainerName = k8sPodName + ":" + k8sNamespace

if nwCfg.MultiTenancy {
// Initialize CNSClient
cnsclient.InitCnsClient(nwCfg.CNSUrl)
}

k8sContainerID := args.ContainerID
if len(k8sContainerID) == 0 {
errMsg := "Container ID not specified in CNI Args"
Expand All @@ -361,6 +380,24 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
return plugin.Errorf(errMsg)
}

log.Printf("Execution mode :%s", nwCfg.ExecutionMode)
if nwCfg.ExecutionMode == string(Baremetal) {
var res *nnscontracts.ConfigureContainerNetworkingResponse
log.Printf("Baremetal mode. Calling vnet agent for ADD")
err, res = plugin.nnsClient.AddContainerNetworking(context.Background(), k8sPodName, args.Netns)

if err == nil {
result = convertNnsToCniResult(res, args.IfName, k8sPodName, "AddContainerNetworking")
}

return err
}

if nwCfg.MultiTenancy {
// Initialize CNSClient
cnsclient.InitCnsClient(nwCfg.CNSUrl)
}

for _, ns := range nwCfg.PodNamespaceForDualNetwork {
if k8sNamespace == ns {
log.Printf("Enable infravnet for this pod %v in namespace %v", k8sPodName, k8sNamespace)
Expand Down Expand Up @@ -489,7 +526,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
Id: networkId,
Mode: nwCfg.Mode,
MasterIfName: masterIfName,
AdapterName: nwCfg.AdapterName,
AdapterName: nwCfg.AdapterName,
Subnets: []network.SubnetInfo{
{
Family: platform.AfINET,
Expand Down Expand Up @@ -792,6 +829,28 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
plugin.setCNIReportDetails(nwCfg, CNI_DEL, "")
iptables.DisableIPTableLock = nwCfg.DisableIPTableLock

sendMetricFunc := func() {
operationTimeMs := time.Since(startTime).Milliseconds()
cniMetric.Metric = aitelemetry.Metric{
Name: telemetry.CNIDelTimeMetricStr,
Value: float64(operationTimeMs),
CustomDimensions: make(map[string]string),
}
SetCustomDimensions(&cniMetric, nwCfg, err)
telemetry.SendCNIMetric(&cniMetric, plugin.tb)
}

log.Printf("Execution mode :%s", nwCfg.ExecutionMode)
if nwCfg.ExecutionMode == string(Baremetal) {

log.Printf("Baremetal mode. Calling vnet agent for delete container")

// schedule send metric before attempting delete
defer sendMetricFunc()
err, _ = plugin.nnsClient.DeleteContainerNetworking(context.Background(), k8sPodName, args.Netns)
return err
}

if nwCfg.MultiTenancy {
// Initialize CNSClient
cnsclient.InitCnsClient(nwCfg.CNSUrl)
Expand Down Expand Up @@ -859,17 +918,8 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
return err
}

defer func() {
operationTimeMs := time.Since(startTime).Milliseconds()
cniMetric.Metric = aitelemetry.Metric{
Name: telemetry.CNIDelTimeMetricStr,
Value: float64(operationTimeMs),
CustomDimensions: make(map[string]string),
}
SetCustomDimensions(&cniMetric, nwCfg, err)
telemetry.SendCNIMetric(&cniMetric, plugin.tb)
}()

// schedule send metric before attempting delete
defer sendMetricFunc()
// Delete the endpoint.
if err = plugin.nm.DeleteEndpoint(networkId, endpointId); err != nil {
err = plugin.Errorf("Failed to delete endpoint: %v", err)
Expand Down Expand Up @@ -1143,3 +1193,47 @@ func determineSnat() (bool, bool, error) {

return snatConfig.EnableSnatForDns, snatConfig.EnableSnatOnHost, nil
}

func convertNnsToCniResult(
netRes *nnscontracts.ConfigureContainerNetworkingResponse,
ifName string,
podName string,
operationName string) *cniTypesCurr.Result {

// This function does not add interfaces to CNI result. Reason being CRI (containerD in baremetal case)
// only looks for default interface named "eth0" and this default interface is added in the defer
// method of ADD method
result := &cniTypesCurr.Result{}
var resultIpconfigs []*cniTypesCurr.IPConfig

if netRes.Interfaces != nil {
Copy link
Contributor

@ramiro-gamarra ramiro-gamarra Mar 25, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: this check is not needed. looping on nil maps or slices is fine

for i, ni := range netRes.Interfaces {

intIndex := i
for _, ip := range ni.Ipaddresses {

ipWithPrefix := fmt.Sprintf("%s/%s", ip.Ip, ip.PrefixLength)
_, ipNet, err := net.ParseCIDR(ipWithPrefix)
if err != nil {
log.Printf("Error while converting to cni result for %s operation on pod %s. %s",
operationName, podName, err)
continue
}

gateway := net.ParseIP(ip.DefaultGateway)
ipConfig := &cniTypesCurr.IPConfig{
Address: *ipNet,
Gateway: gateway,
Version: ip.Version,
Interface: &intIndex,
}

resultIpconfigs = append(resultIpconfigs, ipConfig)
}
}
}

result.IPs = resultIpconfigs

return result
}
6 changes: 4 additions & 2 deletions cni/network/network_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/Azure/azure-container-networking/common"
"github.com/Azure/azure-container-networking/network"
acnnetwork "github.com/Azure/azure-container-networking/network"
"github.com/Azure/azure-container-networking/nns"
"github.com/Azure/azure-container-networking/telemetry"
cniSkel "github.com/containernetworking/cni/pkg/skel"
)
Expand All @@ -21,7 +22,8 @@ func TestPlugin(t *testing.T) {

mockNetworkManager := acnnetwork.NewMockNetworkmanager()

plugin, _ := NewPlugin(pluginName, config)
grpcClient := &nns.MockGrpcClient{}
plugin, _ := NewPlugin(pluginName, config, grpcClient)
plugin.report = &telemetry.CNIReport{}
plugin.nm = mockNetworkManager

Expand Down Expand Up @@ -68,4 +70,4 @@ func TestPlugin(t *testing.T) {
}
plugin.nm.CreateNetwork(nwInfo)
plugin.Delete(args)
}
}
5 changes: 3 additions & 2 deletions cni/network/plugin/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package main
import (
"encoding/json"
"fmt"
"github.com/Azure/azure-container-networking/nns"
"io/ioutil"
"os"
"reflect"
Expand Down Expand Up @@ -173,7 +174,7 @@ func main() {

cniReport.GetReport(pluginName, version, ipamQueryURL)

netPlugin, err := network.NewPlugin(name, &config)
netPlugin, err := network.NewPlugin(name, &config, &nns.GrpcClient{})
if err != nil {
log.Printf("Failed to create network plugin, err:%v.\n", err)
return
Expand Down Expand Up @@ -261,4 +262,4 @@ func main() {
log.Printf("Sending report succeeded")
}
}
}
}
4 changes: 3 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,13 @@ require (
github.com/spf13/viper v1.3.2
github.com/stretchr/testify v1.7.0
go.opencensus.io v0.22.2 // indirect
golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 // indirect
golang.org/x/net v0.0.0-20191112182307-2180aed22343 // indirect
golang.org/x/sys v0.0.0-20200828161417-c663848e9a16
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
google.golang.org/appengine v1.6.5 // indirect
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013
google.golang.org/grpc v1.27.0
google.golang.org/protobuf v1.25.0
k8s.io/api v0.18.2
k8s.io/apimachinery v0.18.2
k8s.io/client-go v0.18.2
Expand Down
Loading