Skip to content

Commit

Permalink
Pod MTU
Browse files Browse the repository at this point in the history
  • Loading branch information
Joseph Chen committed Feb 8, 2024
1 parent 9a23696 commit 64fdc26
Show file tree
Hide file tree
Showing 4 changed files with 125 additions and 6 deletions.
8 changes: 8 additions & 0 deletions README.md
View file
Expand Up @@ -267,6 +267,14 @@ Default: empty
Specify a comma-separated list of IPv4 CIDRs to exclude from SNAT. For every item in the list an `iptables` rule and off\-VPC
IP rule will be applied. If an item is not a valid ipv4 range it will be skipped. This should be used when `AWS_VPC_K8S_CNI_EXTERNALSNAT=false`.

#### `POD_MTU` (v1.x.x+)

Type: Integer as a String

Default: 9001

Used to configure the MTU size for pod virtual interfaces. The valid range is from `576` to `9001`.

#### `WARM_ENI_TARGET`

Type: Integer as a String
Expand Down
5 changes: 5 additions & 0 deletions cmd/aws-vpc-cni/main.go
View file
Expand Up @@ -88,6 +88,7 @@ const (
envHostCniConfDirPath = "HOST_CNI_CONFDIR_PATH"
envVethPrefix = "AWS_VPC_K8S_CNI_VETHPREFIX"
envEniMTU = "AWS_VPC_ENI_MTU"
envPodMTU = "POD_MTU"
envEnablePodEni = "ENABLE_POD_ENI"
envPodSGEnforcingMode = "POD_SECURITY_GROUP_ENFORCING_MODE"
envPluginLogFile = "AWS_VPC_K8S_PLUGIN_LOG_FILE"
Expand Down Expand Up @@ -279,6 +280,10 @@ func generateJSON(jsonFile string, outFile string, getPrimaryIP func(ipv4 bool)
}
vethPrefix := utils.GetEnv(envVethPrefix, defaultVethPrefix)
mtu := utils.GetEnv(envEniMTU, defaultMTU)
_, found := os.LookupEnv("POD_MTU")
if found {
mtu = utils.GetEnv(envPodMTU, defaultMTU)
}
podSGEnforcingMode := utils.GetEnv(envPodSGEnforcingMode, defaultPodSGEnforcingMode)
pluginLogFile := utils.GetEnv(envPluginLogFile, defaultPluginLogFile)
pluginLogLevel := utils.GetEnv(envPluginLogLevel, defaultPluginLogLevel)
Expand Down
60 changes: 56 additions & 4 deletions test/integration/cni/host_networking_test.go
View file
Expand Up @@ -17,12 +17,11 @@ import (
"strconv"
"time"

v1 "k8s.io/api/core/v1"

"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
"github.com/aws/amazon-vpc-cni-k8s/test/integration/common"
v1 "k8s.io/api/core/v1"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
Expand All @@ -31,6 +30,7 @@ import (
// TODO: Instead of passing the list of pods to the test helper, have the test helper get the pod on node
const (
NEW_MTU_VAL = 1300
NEW_POD_MTU = 1280
NEW_VETH_PREFIX = "veth"
)

Expand All @@ -57,6 +57,10 @@ var _ = Describe("test host networking", func() {
"AWS_VPC_ENI_MTU": DEFAULT_MTU_VAL,
"AWS_VPC_K8S_CNI_VETHPREFIX": DEFAULT_VETH_PREFIX,
})
k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName,
utils.AwsNodeNamespace, utils.AwsNodeName, map[string]struct{}{
"POD_MTU": {},
})
// After updating daemonset pod, we must wait until conflist is updated so that container-runtime calls CNI ADD with the latest VETH prefix and MTU.
// Otherwise, the stale value can cause failures in future test cases.
time.Sleep(utils.PollIntervalMedium)
Expand Down Expand Up @@ -104,14 +108,14 @@ var _ = Describe("test host networking", func() {
common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f)
})

It("Validate Host Networking setup after changing MTU and Veth Prefix", func() {
It("Validate Host Networking setup after changing ENI MTU and Veth Prefix", func() {
deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
Replicas(maxIPPerInterface*2).
PodLabel(podLabelKey, podLabelVal).
NodeName(primaryNode.Name).
Build()

By("Configuring Veth Prefix and MTU value on aws-node daemonset")
By("Configuring Veth Prefix and ENI MTU value on aws-node daemonset")
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
"AWS_VPC_ENI_MTU": strconv.Itoa(NEW_MTU_VAL),
"AWS_VPC_K8S_CNI_VETHPREFIX": NEW_VETH_PREFIX,
Expand Down Expand Up @@ -150,6 +154,54 @@ var _ = Describe("test host networking", func() {
By("validating host networking is teared down correctly")
common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f)
})

It("Validate Host Networking setup after changing Pod MTU and Veth Prefix", func() {
deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
Replicas(maxIPPerInterface*2).
PodLabel(podLabelKey, podLabelVal).
NodeName(primaryNode.Name).
Build()

By("Configuring Veth Prefix and Pod MTU value on aws-node daemonset")
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
"AWS_VPC_ENI_MTU": strconv.Itoa(NEW_MTU_VAL),
"POD_MTU": strconv.Itoa(NEW_POD_MTU),
"AWS_VPC_K8S_CNI_VETHPREFIX": NEW_VETH_PREFIX,
})
// After updating daemonset pod, we must wait until conflist is updated so that container-runtime calls CNI ADD with the new VETH prefix and MTU.
time.Sleep(utils.PollIntervalMedium)

By("creating a deployment to launch pods")
deployment, err = f.K8sResourceManagers.DeploymentManager().
CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout)
Expect(err).ToNot(HaveOccurred())

By("getting the list of pods using IP from primary and secondary ENI")
interfaceTypeToPodList :=
common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal, f)

By("generating the pod networking validation input to be passed to tester")
podNetworkingValidationInput := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs)
podNetworkingValidationInput.VethPrefix = NEW_VETH_PREFIX
podNetworkingValidationInput.ValidateMTU = true
podNetworkingValidationInput.MTU = NEW_POD_MTU
input, err := podNetworkingValidationInput.Serialize()
Expect(err).NotTo(HaveOccurred())

By("validating host networking setup is setup correctly with MTU check as well")
common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f)

By("deleting the deployment to test teardown")
err = f.K8sResourceManagers.DeploymentManager().
DeleteAndWaitTillDeploymentIsDeleted(deployment)
Expect(err).ToNot(HaveOccurred())

By("waiting to allow CNI to tear down networking for terminated pods")
time.Sleep(time.Second * 60)

By("validating host networking is teared down correctly")
common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f)
})
})

Context("when host networking is tested on invalid input", func() {
Expand Down
58 changes: 56 additions & 2 deletions test/integration/ipv6/ipv6_host_networking_test.go
View file
Expand Up @@ -41,7 +41,9 @@ const (
const (
AWS_VPC_ENI_MTU = "AWS_VPC_ENI_MTU"
AWS_VPC_K8S_CNI_VETHPREFIX = "AWS_VPC_K8S_CNI_VETHPREFIX"
POD_MTU = "POD_MTU"
NEW_MTU_VAL = 1300
NEW_POD_MTU = 1280
NEW_VETH_PREFIX = "veth"
DEFAULT_MTU_VAL = "9001"
DEFAULT_VETH_PREFIX = "eni"
Expand All @@ -58,6 +60,10 @@ var _ = Describe("[CANARY] test ipv6 host netns setup", func() {
AWS_VPC_ENI_MTU: DEFAULT_MTU_VAL,
AWS_VPC_K8S_CNI_VETHPREFIX: DEFAULT_VETH_PREFIX,
})
k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName,
utils.AwsNodeNamespace, utils.AwsNodeName, map[string]struct{}{
"POD_MTU": {},
})
// After updating daemonset pod, we must wait until conflist is updated so that container-runtime calls CNI ADD with the latest VETH prefix and MTU.
// Otherwise, the stale value can cause failures in future test cases.
time.Sleep(utils.PollIntervalMedium)
Expand Down Expand Up @@ -98,14 +104,14 @@ var _ = Describe("[CANARY] test ipv6 host netns setup", func() {
ValidateHostNetworking(NetworkingTearDownSucceeds, input)
})

It("Validate host netns setup after changing MTU and Veth Prefix", func() {
It("Validate host netns setup after changing ENI MTU and Veth Prefix", func() {
deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
Replicas(2).
PodLabel(podLabelKey, podLabelVal).
NodeName(primaryNode.Name).
Build()

By("Configuring Veth Prefix and MTU value on aws-node daemonset")
By("Configuring Veth Prefix and ENI MTU value on aws-node daemonset")
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
AWS_VPC_ENI_MTU: strconv.Itoa(NEW_MTU_VAL),
AWS_VPC_K8S_CNI_VETHPREFIX: NEW_VETH_PREFIX,
Expand Down Expand Up @@ -144,6 +150,54 @@ var _ = Describe("[CANARY] test ipv6 host netns setup", func() {
By("validating host networking is teared down correctly")
ValidateHostNetworking(NetworkingTearDownSucceeds, input)
})

It("Validate host netns setup after changing Pod MTU and Veth Prefix", func() {
deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
Replicas(2).
PodLabel(podLabelKey, podLabelVal).
NodeName(primaryNode.Name).
Build()

By("Configuring Veth Prefix and Pod MTU value on aws-node daemonset")
k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
AWS_VPC_ENI_MTU: strconv.Itoa(NEW_MTU_VAL),
POD_MTU: strconv.Itoa(NEW_POD_MTU),
AWS_VPC_K8S_CNI_VETHPREFIX: NEW_VETH_PREFIX,
})
// After updating daemonset pod, we must wait until conflist is updated so that container-runtime calls CNI ADD with the new VETH prefix and MTU.
time.Sleep(utils.PollIntervalMedium)

By("creating a deployment to launch pods")
deployment, err = f.K8sResourceManagers.DeploymentManager().
CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout)
Expect(err).ToNot(HaveOccurred())

By("getting the list of pods using IP from primary and secondary ENI")
interfaceTypeToPodList :=
GetIPv6Pods(podLabelKey, podLabelVal)

By("generating the pod networking validation input to be passed to tester")
podNetworkingValidationInput := GetIPv6PodNetworkingValidationInput(interfaceTypeToPodList)
podNetworkingValidationInput.VethPrefix = NEW_VETH_PREFIX
podNetworkingValidationInput.ValidateMTU = true
podNetworkingValidationInput.MTU = NEW_POD_MTU
input, err := podNetworkingValidationInput.Serialize()
Expect(err).NotTo(HaveOccurred())

By("validating host networking setup is setup correctly with MTU check as well")
ValidateHostNetworking(NetworkingSetupSucceeds, input)

By("deleting the deployment to test teardown")
err = f.K8sResourceManagers.DeploymentManager().
DeleteAndWaitTillDeploymentIsDeleted(deployment)
Expect(err).ToNot(HaveOccurred())

By("waiting to allow CNI to tear down networking for terminated pods")
time.Sleep(time.Second * 60)

By("validating host networking is teared down correctly")
ValidateHostNetworking(NetworkingTearDownSucceeds, input)
})
})

Context("when host netns setup is tested on invalid input", func() {
Expand Down

0 comments on commit 64fdc26

Please sign in to comment.