feat(networking): add network cni plugin (kubernetes-sigs#4)

* feat(networking): add custom calico cni

* feat(networking): add custom multus

cluster.yml

@@ -100,7 +100,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kubespray-defaults }
-    - { role: network_plugin/calico/rr, tags: ['network', 'calico_rr'] }
+    - { role: network_plugin/bd-calico/rr, tags: ['network', 'calico_rr'] }
 
 - hosts: kube-master[0]
   gather_facts: False

inventory/deploy-cluster/group_vars/k8s-cluster/k8s-cluster.yml

@@ -92,11 +92,14 @@ kube_users:
 
 # Choose network plugin (cilium, calico, contiv, weave or flannel. Use cni for generic cni plugin)
 # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
-kube_network_plugin: calico
+kube_network_plugin: bd-calico
 
 # Setting multi_networking to true will install Multus: https://github.com/intel/multus-cni
 kube_network_plugin_multus: false
 
+# used to bd-multus
+kube_network_plugin_bd_multus: true
+
 # Kubernetes internal network for services, unused block of space.
 kube_service_addresses: 10.233.0.0/18
 
@@ -351,3 +354,26 @@ kube_apiserver_enable_admission_plugins:
 kube_apiserver_bind_address: "{{ ip | default(fallback_ips[inventory_hostname]) }}"
 kube_controller_manager_bind_address: "{{ ip | default(fallback_ips[inventory_hostname]) }}"
 kube_scheduler_bind_address: "{{ ip | default(fallback_ips[inventory_hostname]) }}"
+
+# set calico mode: vxlan, bird, none, default is bird.
+calico_mode: "vxlan"
+
+# fixedip-ipam server config
+fixedip_ipam_enabled: false
+fixedip_ipam_server_service: ""
+fixedip_ipam_server_port: 6109
+
+fixedip_ipam_version: "v1.5.0-alpha.1"
+
+fixedip_ipam_client_image_repo: "{{ docker_image_repo }}/fixedip-ipam-client"
+fixedip_ipam_client_image_tag: "{{ fixedip_ipam_version }}"
+fixedip_ipam_server_image_repo: "{{ docker_image_repo }}/fixedip-ipam-server"
+fixedip_ipam_server_image_tag: "{{ fixedip_ipam_version }}"
+
+# bridge vlan info
+bridge_vlan_image_repo: "{{ docker_image_repo }}/bridge-vlan"
+bridge_vlan_image_tag: "v1.5.0-alpha.2"
+
+# network-agent
+network_agent_image_repo: "{{ docker_image_repo }}/network-agent"
+network_agent_image_tag: "v1.5.0-alpha.2"

inventory/deploy-cluster/group_vars/k8s-cluster/k8s-net-calico.yml

@@ -18,7 +18,7 @@
 # calico_pool_cidr: 1.2.3.4/5
 
 # Global as_num (/calico/bgp/v1/global/as_num)
-# global_as_num: "64512"
+global_as_num: "64512"
 
 # You can set MTU value here. If left undefined or empty, it will
 # not be specified in calico CNI config, so Calico will use built-in
@@ -36,7 +36,7 @@
 # calico_advertise_cluster_ips: true
 
 # Choose data store type for calico: "etcd" or "kdd" (kubernetes datastore)
-# calico_datastore: "etcd"
+calico_datastore: "kdd"
 
 # Choose Calico iptables backend: "Legacy", "Auto" or "NFT"
 # calico_iptables_backend: "Legacy"

roles/kubernetes-apps/network_plugin/bd-multus/tasks/main.yml

@@ -7,5 +7,10 @@
     resource: "{{ item.item.type }}"
     filename: "{{ kube_config_dir }}/{{ item.item.file }}"
     state: "latest"
+    wait: true
+  register: result
+  until: result is success
+  retries: 5
+  delay: 2
   with_items: "{{ multus_manifest_1.results }} + {{ multus_manifest_2.results }}"
   when: inventory_hostname == groups['kube-master'][0] and not item is skipped

roles/kubernetes-apps/network_plugin/meta/main.yml

@@ -49,3 +49,13 @@ dependencies:
     when: kube_network_plugin_multus
     tags:
       - multus
+
+  - role: kubernetes-apps/network_plugin/bd-calico
+    when: kube_network_plugin == 'bd-calico'
+    tags:
+      - bd-calico
+
+  - role: kubernetes-apps/network_plugin/bd-multus
+    when: kube_network_plugin_bd_multus
+    tags:
+      - bd-multus

roles/kubernetes/preinstall/tasks/0020-verify-settings.yml

@@ -22,7 +22,7 @@
 
 - name: Stop if unknown network plugin
   assert:
-    that: kube_network_plugin in ['calico', 'canal', 'flannel', 'weave', 'cloud', 'cilium', 'cni', 'contiv', 'ovn4nfv','kube-ovn', 'kube-router', 'macvlan']
+    that: kube_network_plugin in ['calico', 'canal', 'flannel', 'weave', 'cloud', 'cilium', 'cni', 'contiv', 'ovn4nfv','kube-ovn', 'kube-router', 'macvlan', 'bd-calico']
     msg: "{{ kube_network_plugin }} is not supported"
   when:
     - kube_network_plugin is defined

roles/network_plugin/bd-calico/defaults/main.yml

@@ -91,3 +91,11 @@ typha_max_connections_lower_limit: 300
 typha_secure: false
 
 calico_feature_control: {}
+
+# set calico mode: vxlan, bird, none, default is bird.
+calico_mode: "vxlan"
+
+# fixedip-ipam server config
+fixedip_ipam_enabled: false
+fixedip_ipam_server_service: ""
+fixedip_ipam_server_port: 6109

roles/network_plugin/bd-calico/templates/calico-config.yml.j2

@@ -19,7 +19,7 @@ data:
   cluster_type: "kubespray"
   calico_backend: "{{ calico_network_backend }}"
 {% else %}
-  cluster_type: "kubespray,bgp"
+  cluster_type: "k8s,bgp"
   calico_backend: "bird"
 {% endif %}
 {% if inventory_hostname in groups['k8s-cluster'] and peer_with_router|default(false) %}

roles/network_plugin/bd-calico/templates/calico-node.yml.j2

@@ -399,5 +399,5 @@ spec:
 {% endif %}
   updateStrategy:
     rollingUpdate:
-      maxUnavailable: {{ serial | default('20%') }}
+      maxUnavailable: 1
     type: RollingUpdate

roles/network_plugin/bd-calico/templates/cni-calico.conflist.j2

@@ -1,5 +1,5 @@
 {
-  "name": "cni0",
+  "name": "k8s-pod-network",
   "cniVersion":"0.3.1",
   "plugins":[
     {
@@ -14,6 +14,7 @@
     {% endif %}
 {% endif %}
       "type": "calico",
+      "mode": "{{ calico_mode }}",
       "log_level": "info",
 {% if calico_datastore == "etcd" %}
       "etcd_endpoints": "{{ etcd_access_addresses }}",
@@ -26,6 +27,11 @@
         "type": "host-local",
         "subnet": "usePodCidr"
       },
+{% elif fixedip_ipam_enabled %}
+      "ipam": {
+        "type": "fixedip-ipam",
+        "server": "http://{{ fixedip_ipam_server_service }}:{{ fixedip_ipam_server_port }}"
+      },
 {% else %}
       "ipam": {
         "type": "calico-ipam",

roles/network_plugin/bd-calico/templates/kdd-crds.yml.j2

@@ -33,6 +33,16 @@ spec:
           spec:
             description: FelixConfigurationSpec contains the values of the Felix configuration.
             properties:
+              allowIPIPPacketsFromWorkloads:
+                description: 'AllowIPIPPacketsFromWorkloads controls whether Felix
+                  will add a rule to drop IPIP encapsulated traffic from workloads
+                  [Default: false]'
+                type: boolean
+              allowVXLANPacketsFromWorkloads:
+                description: 'AllowVXLANPacketsFromWorkloads controls whether Felix
+                  will add a rule to drop VXLAN encapsulated traffic from workloads
+                  [Default: false]'
+                type: boolean
               awsSrcDstCheck:
                 description: 'Set source-destination-check on AWS EC2 instances. Accepted
                   value must be one of "DoNothing", "Enabled" or "Disabled". [Default:
@@ -204,6 +214,13 @@ spec:
                   - protocol
                   type: object
                 type: array
+              featureDetectOverride:
+                description: FeatureDetectOverride is used to override the feature
+                  detection. Values are specified in a comma separated list with no
+                  spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=".
+                  "true" or "false" will force the feature, empty or omitted values
+                  are auto-detected.
+                type: string
               genericXDPEnabled:
                 description: 'GenericXDPEnabled enables Generic XDP so network cards
                   that don''t support XDP offload or driver modes can use XDP. This
@@ -235,6 +252,11 @@ spec:
                   integrations set the ‘cali’ value, and our OpenStack integration
                   sets the ‘tap’ value. [Default: cali]'
                 type: string
+              interfaceRefreshInterval:
+                description: InterfaceRefreshInterval is the period at which Felix
+                  rescans local interfaces to verify their state. The rescan can be
+                  disabled by setting the interval to 0.
+                type: string
               ipipEnabled:
                 type: boolean
               ipipMTU:
@@ -506,8 +528,6 @@ spec:
                   Calico''s BPF maps or attached programs. Set to 0 to disable XDP
                   refresh. [Default: 90s]'
                 type: string
-            required:
-            - bpfLogLevel
             type: object
         type: object
     served: true
@@ -845,6 +865,32 @@ spec:
                   64512]'
                 format: int32
                 type: integer
+              communities:
+                description: Communities is a list of BGP community values and their
+                  arbitrary names for tagging routes.
+                items:
+                  description: Community contains standard or large community value
+                    and its name.
+                  properties:
+                    name:
+                      description: Name given to community value.
+                      type: string
+                    value:
+                      description: Value must be of format `aa:nn` or `aa:nn:mm`.
+                        For standard community use `aa:nn` format, where `aa` and
+                        `nn` are 16 bit number. For large community use `aa:nn:mm`
+                        format, where `aa`, `nn` and `mm` are 32 bit number. Where,
+                        `aa` is an AS Number, `nn` and `mm` are per-AS identifier.
+                      pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$
+                      type: string
+                  type: object
+                type: array
+              listenPort:
+                description: ListenPort is the port where BGP protocol should listen.
+                  Defaults to 179
+                maximum: 65535
+                minimum: 1
+                type: integer
               logSeverityScreen:
                 description: 'LogSeverityScreen is the log severity above which logs
                   are sent to the stdout. [Default: INFO]'
@@ -853,13 +899,36 @@ spec:
                 description: 'NodeToNodeMeshEnabled sets whether full node to node
                   BGP mesh is enabled. [Default: true]'
                 type: boolean
+              prefixAdvertisements:
+                description: PrefixAdvertisements contains per-prefix advertisement
+                  configuration.
+                items:
+                  description: PrefixAdvertisement configures advertisement properties
+                    for the specified CIDR.
+                  properties:
+                    cidr:
+                      description: CIDR for which properties should be advertised.
+                      type: string
+                    communities:
+                      description: Communities can be list of either community names
+                        already defined in `Specs.Communities` or community value
+                        of format `aa:nn` or `aa:nn:mm`. For standard community use
+                        `aa:nn` format, where `aa` and `nn` are 16 bit number. For
+                        large community use `aa:nn:mm` format, where `aa`, `nn` and
+                        `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and
+                        `mm` are per-AS identifier.
+                      items:
+                        type: string
+                      type: array
+                  type: object
+                type: array
               serviceClusterIPs:
                 description: ServiceClusterIPs are the CIDR blocks from which service
                   cluster IPs are allocated. If specified, Calico will advertise these
                   blocks, as well as any cluster IPs within them.
                 items:
-                  description: ServiceClusterIPBlock represents a single whitelisted
-                    CIDR block for ClusterIPs.
+                  description: ServiceClusterIPBlock represents a single allowed ClusterIP
+                    CIDR block.
                   properties:
                     cidr:
                       type: string
@@ -870,8 +939,8 @@ spec:
                   Service External IPs. Kubernetes Service ExternalIPs will only be
                   advertised if they are within one of these blocks.
                 items:
-                  description: ServiceExternalIPBlock represents a single whitelisted
-                    CIDR External IP block.
+                  description: ServiceExternalIPBlock represents a single allowed
+                    External IP CIDR block.
                   properties:
                     cidr:
                       type: string
@@ -946,7 +1015,7 @@ spec:
                 type: object
               ipipMode:
                 description: Contains configuration for IPIP tunneling for this pool.
-                  If not specified, then this is defaulted to "Never" (i.e. IPIP tunelling
+                  If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling
                   is disabled).
                 type: string
               nat-outgoing:
@@ -966,7 +1035,7 @@ spec:
               vxlanMode:
                 description: Contains configuration for VXLAN tunneling for this pool.
                   If not specified, then this is defaulted to "Never" (i.e. VXLAN
-                  tunelling is disabled).
+                  tunneling is disabled).
                 type: string
             required:
             - cidr
@@ -1341,7 +1410,7 @@ spec:
                         code:
                           description: Match on a specific ICMP code.  If specified,
                             the Type value must also be specified. This is a technical
-                            limitation imposed by the kernel’s iptables firewall,
+                            limitation imposed by the kernel's iptables firewall,
                             which Calico uses to enforce the rule.
                           type: integer
                         type:
@@ -1370,7 +1439,7 @@ spec:
                         code:
                           description: Match on a specific ICMP code.  If specified,
                             the Type value must also be specified. This is a technical
-                            limitation imposed by the kernel’s iptables firewall,
+                            limitation imposed by the kernel's iptables firewall,
                             which Calico uses to enforce the rule.
                           type: integer
                         type:
@@ -1672,7 +1741,7 @@ spec:
                         code:
                           description: Match on a specific ICMP code.  If specified,
                             the Type value must also be specified. This is a technical
-                            limitation imposed by the kernel’s iptables firewall,
+                            limitation imposed by the kernel's iptables firewall,
                             which Calico uses to enforce the rule.
                           type: integer
                         type:
@@ -1701,7 +1770,7 @@ spec:
                         code:
                           description: Match on a specific ICMP code.  If specified,
                             the Type value must also be specified. This is a technical
-                            limitation imposed by the kernel’s iptables firewall,
+                            limitation imposed by the kernel's iptables firewall,
                             which Calico uses to enforce the rule.
                           type: integer
                         type:
@@ -2142,7 +2211,7 @@ spec:
                         code:
                           description: Match on a specific ICMP code.  If specified,
                             the Type value must also be specified. This is a technical
-                            limitation imposed by the kernel’s iptables firewall,
+                            limitation imposed by the kernel's iptables firewall,
                             which Calico uses to enforce the rule.
                           type: integer
                         type:
@@ -2171,7 +2240,7 @@ spec:
                         code:
                           description: Match on a specific ICMP code.  If specified,
                             the Type value must also be specified. This is a technical
-                            limitation imposed by the kernel’s iptables firewall,
+                            limitation imposed by the kernel's iptables firewall,
                             which Calico uses to enforce the rule.
                           type: integer
                         type:
@@ -2473,7 +2542,7 @@ spec:
                         code:
                           description: Match on a specific ICMP code.  If specified,
                             the Type value must also be specified. This is a technical
-                            limitation imposed by the kernel’s iptables firewall,
+                            limitation imposed by the kernel's iptables firewall,
                             which Calico uses to enforce the rule.
                           type: integer
                         type:
@@ -2502,7 +2571,7 @@ spec:
                         code:
                           description: Match on a specific ICMP code.  If specified,
                             the Type value must also be specified. This is a technical
-                            limitation imposed by the kernel’s iptables firewall,
+                            limitation imposed by the kernel's iptables firewall,
                             which Calico uses to enforce the rule.
                           type: integer
                         type: