Set FELIX_CHAININSERTMODE default to "append"

cmd/kops/create_cluster.go

@@ -174,7 +174,7 @@ func (o *CreateClusterOptions) InitDefaults() {
 	o.Yes = false
 	o.Target = cloudup.TargetDirect
 	o.Models = strings.Join(cloudup.CloudupModels, ",")
-	o.Networking = "kubenet"
+	o.Networking = "calico"
 	o.Channel = api.DefaultChannel
 	o.Topology = api.TopologyPublic
 	o.DNSType = string(api.DNSTypePublic)

docs/cli/kops_create_cluster.md

@@ -93,7 +93,7 @@ kops create cluster [flags]
       --master-zones strings             Zones in which to run masters (must be an odd number)
       --model string                     Models to apply (separate multiple models with commas) (default "proto,cloudup")
       --network-cidr string              Set to override the default network CIDR
-      --networking string                Networking mode to use.  kubenet (default), classic, external, kopeio-vxlan (or kopeio), weave, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, romana, amazon-vpc-routed-eni, cilium, cni. (default "kubenet")
+      --networking string                Networking mode to use.  kubenet (default), classic, external, kopeio-vxlan (or kopeio), weave, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, romana, amazon-vpc-routed-eni, cilium, cni. (default "calico")
       --node-count int32                 Set the number of nodes
       --node-security-groups strings     Add precreated additional security groups to nodes.
       --node-size string                 Set instance size for nodes

k8s/crds/kops.k8s.io_clusters.yaml

@@ -2490,7 +2490,16 @@ spec:
                 calico:
                   description: CalicoNetworkingSpec declares that we want Calico networking
                   properties:
+                    chainInsertMode:
+                      description: 'ChainInsertMode controls whether Felix inserts
+                        rules to the top of iptables chains, or appends to the bottom.
+                        Leaving the default option is safest to prevent accidentally
+                        breaking connectivity. Default: ''insert'' (other options:
+                        ''append'')'
+                      type: string
                     crossSubnet:
+                      description: CrossSubnet enables Calico's cross-subnet mode
+                        when set to true
                       type: boolean
                     ipipMode:
                       description: IPIPMode is mode for CALICO_IPV4POOL_IPIP

pkg/apis/kops/networking.go

@@ -100,7 +100,12 @@ type FlannelNetworkingSpec struct {
 
 // CalicoNetworkingSpec declares that we want Calico networking
 type CalicoNetworkingSpec struct {
-	CrossSubnet bool `json:"crossSubnet,omitempty"` // Enables Calico's cross-subnet mode when set to true
+	// ChainInsertMode controls whether Felix inserts rules to the top of iptables chains, or
+	// appends to the bottom. Leaving the default option is safest to prevent accidentally
+	// breaking connectivity. Default: 'insert' (other options: 'append')
+	ChainInsertMode string `json:"chainInsertMode,omitempty"`
+	// CrossSubnet enables Calico's cross-subnet mode when set to true
+	CrossSubnet bool `json:"crossSubnet,omitempty"`
 	// LogSeverityScreen lets us set the desired log level. (Default: info)
 	LogSeverityScreen string `json:"logSeverityScreen,omitempty"`
 	// MTU to be set in the cni-network-config for calico.

pkg/apis/kops/v1alpha2/networking.go

@@ -100,7 +100,12 @@ type FlannelNetworkingSpec struct {
 
 // CalicoNetworkingSpec declares that we want Calico networking
 type CalicoNetworkingSpec struct {
-	CrossSubnet bool `json:"crossSubnet,omitempty"` // Enables Calico's cross-subnet mode when set to true
+	// ChainInsertMode controls whether Felix inserts rules to the top of iptables chains, or
+	// appends to the bottom. Leaving the default option is safest to prevent accidentally
+	// breaking connectivity. Default: 'insert' (other options: 'append')
+	ChainInsertMode string `json:"chainInsertMode,omitempty"`
+	// CrossSubnet enables Calico's cross-subnet mode when set to true
+	CrossSubnet bool `json:"crossSubnet,omitempty"`
 	// LogSeverityScreen lets us set the desired log level. (Default: info)
 	LogSeverityScreen string `json:"logSeverityScreen,omitempty"`
 	// MTU to be set in the cni-network-config for calico.

tests/integration/create_cluster/complex/expected-v1alpha2.yaml

@@ -19,12 +19,14 @@ spec:
       name: a
     memoryRequest: 100Mi
     name: main
+    version: 3.2.24
   - cpuRequest: 100m
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: events
+    version: 3.2.24
   iam:
     allowContainerRegistry: true
     legacy: false
@@ -36,7 +38,8 @@ spec:
   masterPublicName: api.complex.example.com
   networkCIDR: 172.20.0.0/16
   networking:
-    kubenet: {}
+    calico:
+      majorVersion: v3
   nonMasqueradeCIDR: 100.64.0.0/10
   sshAccess:
   - 1.2.3.4/32

tests/integration/create_cluster/ha/expected-v1alpha2.yaml

@@ -23,6 +23,7 @@ spec:
       name: c
     memoryRequest: 100Mi
     name: main
+    version: 3.2.24
   - cpuRequest: 100m
     etcdMembers:
     - instanceGroup: master-us-test-1a
@@ -33,6 +34,7 @@ spec:
       name: c
     memoryRequest: 100Mi
     name: events
+    version: 3.2.24
   iam:
     allowContainerRegistry: true
     legacy: false
@@ -44,7 +46,8 @@ spec:
   masterPublicName: api.ha.example.com
   networkCIDR: 172.20.0.0/16
   networking:
-    kubenet: {}
+    calico:
+      majorVersion: v3
   nonMasqueradeCIDR: 100.64.0.0/10
   sshAccess:
   - 0.0.0.0/0

tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml

@@ -26,6 +26,7 @@ spec:
       name: c
     memoryRequest: 100Mi
     name: main
+    version: 3.2.24
   - cpuRequest: 100m
     etcdMembers:
     - encryptedVolume: true
@@ -39,6 +40,7 @@ spec:
       name: c
     memoryRequest: 100Mi
     name: events
+    version: 3.2.24
   iam:
     allowContainerRegistry: true
     legacy: false
@@ -50,7 +52,8 @@ spec:
   masterPublicName: api.ha.example.com
   networkCIDR: 172.20.0.0/16
   networking:
-    kubenet: {}
+    calico:
+      majorVersion: v3
   nonMasqueradeCIDR: 100.64.0.0/10
   sshAccess:
   - 0.0.0.0/0

tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml

@@ -23,6 +23,7 @@ spec:
       name: c
     memoryRequest: 100Mi
     name: main
+    version: 3.2.24
   - cpuRequest: 100m
     etcdMembers:
     - instanceGroup: master-us-test1-a
@@ -33,6 +34,7 @@ spec:
       name: c
     memoryRequest: 100Mi
     name: events
+    version: 3.2.24
   iam:
     allowContainerRegistry: true
     legacy: false
@@ -43,7 +45,8 @@ spec:
   kubernetesVersion: v1.15.6-beta.1
   masterPublicName: api.ha-gce.example.com
   networking:
-    kubenet: {}
+    calico:
+      majorVersion: v3
   nonMasqueradeCIDR: 100.64.0.0/10
   project: testproject
   sshAccess:

tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml

@@ -27,6 +27,7 @@ spec:
       name: a-3
     memoryRequest: 100Mi
     name: main
+    version: 3.2.24
   - cpuRequest: 100m
     etcdMembers:
     - instanceGroup: master-us-test-1a-1
@@ -41,6 +42,7 @@ spec:
       name: a-3
     memoryRequest: 100Mi
     name: events
+    version: 3.2.24
   iam:
     allowContainerRegistry: true
     legacy: false
@@ -52,7 +54,8 @@ spec:
   masterPublicName: api.ha.example.com
   networkCIDR: 172.20.0.0/16
   networking:
-    kubenet: {}
+    calico:
+      majorVersion: v3
   nonMasqueradeCIDR: 100.64.0.0/10
   sshAccess:
   - 0.0.0.0/0

tests/integration/create_cluster/minimal/expected-v1alpha2.yaml

@@ -19,12 +19,14 @@ spec:
       name: a
     memoryRequest: 100Mi
     name: main
+    version: 3.2.24
   - cpuRequest: 100m
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: events
+    version: 3.2.24
   iam:
     allowContainerRegistry: true
     legacy: false
@@ -36,7 +38,8 @@ spec:
   masterPublicName: api.minimal.example.com
   networkCIDR: 172.20.0.0/16
   networking:
-    kubenet: {}
+    calico:
+      majorVersion: v3
   nonMasqueradeCIDR: 100.64.0.0/10
   sshAccess:
   - 0.0.0.0/0

tests/integration/create_cluster/overrides/expected-v1alpha2.yaml

@@ -19,12 +19,14 @@ spec:
       name: a
     memoryRequest: 100Mi
     name: main
+    version: 3.2.24
   - cpuRequest: 100m
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: events
+    version: 3.2.24
   iam:
     allowContainerRegistry: true
     legacy: false
@@ -36,7 +38,8 @@ spec:
   masterPublicName: api.overrides.example.com
   networkCIDR: 172.20.0.0/16
   networking:
-    kubenet: {}
+    calico:
+      majorVersion: v3
   nodePortAccess:
   - 1.2.3.4/32
   - 10.20.30.0/24

tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml

@@ -19,12 +19,14 @@ spec:
       name: a
     memoryRequest: 100Mi
     name: main
+    version: 3.2.24
   - cpuRequest: 100m
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: events
+    version: 3.2.24
   iam:
     allowContainerRegistry: true
     legacy: false
@@ -37,7 +39,8 @@ spec:
   networkCIDR: 10.0.0.0/12
   networkID: vpc-12345678
   networking:
-    kubenet: {}
+    calico:
+      majorVersion: v3
   nonMasqueradeCIDR: 100.64.0.0/10
   sshAccess:
   - 0.0.0.0/0

tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml

@@ -19,12 +19,14 @@ spec:
       name: a
     memoryRequest: 100Mi
     name: main
+    version: 3.2.24
   - cpuRequest: 100m
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: events
+    version: 3.2.24
   iam:
     allowContainerRegistry: true
     legacy: false
@@ -37,7 +39,8 @@ spec:
   networkCIDR: 10.0.0.0/12
   networkID: vpc-12345678
   networking:
-    kubenet: {}
+    calico:
+      majorVersion: v3
   nonMasqueradeCIDR: 100.64.0.0/10
   sshAccess:
   - 0.0.0.0/0

tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml

@@ -19,12 +19,14 @@ spec:
       name: a
     memoryRequest: 100Mi
     name: main
+    version: 3.2.24
   - cpuRequest: 100m
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: a
     memoryRequest: 100Mi
     name: events
+    version: 3.2.24
   iam:
     allowContainerRegistry: true
     legacy: false
@@ -37,7 +39,8 @@ spec:
   networkCIDR: 10.0.0.0/12
   networkID: vpc-12345678
   networking:
-    kubenet: {}
+    calico:
+      majorVersion: v3
   nonMasqueradeCIDR: 100.64.0.0/10
   sshAccess:
   - 0.0.0.0/0

upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template

@@ -839,6 +839,9 @@ spec:
               value: "true"
 
             # kops additions
+            # Controls whether Felix inserts rules to the top of iptables chains, or appends to the bottom
+            - name: FELIX_CHAININSERTMODE
+              value: "{{- or .Networking.Calico.ChainInsertMode "append" }}"
             # Set Felix iptables binary variant, Legacy or NFT
             - name: FELIX_IPTABLESBACKEND
               value: "{{- or .Networking.Calico.IptablesBackend "Auto" }}"

upup/pkg/fi/cloudup/bootstrapchannelbuilder.go

@@ -703,7 +703,7 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons {
 			"k8s-1.7":    "2.6.12-kops.1",
 			"k8s-1.7-v3": "3.8.0-kops.2",
 			"k8s-1.12":   "3.9.5-kops.1",
-			"k8s-1.16":   "3.12.0-kops.1",
+			"k8s-1.16":   "3.12.0-kops.2",
 		}
 
 		{