feat: update CRDs to accomodate multiple pod nics (#2717)

* feat: update CRDs to accomodate multiple pod nics

* fix: device type in mtpnc status, rename device types

* fix: change podnetworks to podnetworkconfigs

crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go

@@ -19,10 +19,6 @@ import (
 // +kubebuilder:printcolumn:name="PodNetworkInstance",type=string,JSONPath=`.spec.podNetworkInstance`
 // +kubebuilder:printcolumn:name="PodNetwork",type=string,JSONPath=`.spec.podNetwork`
 // +kubebuilder:printcolumn:name="PodName",type=string,JSONPath=`.spec.podName`
-// +kubebuilder:printcolumn:name="NCID",type=string,JSONPath=`.status.ncID`
-// +kubebuilder:printcolumn:name="PrimaryIP",type=string,JSONPath=`.status.primaryIP`
-// +kubebuilder:printcolumn:name="MacAddress",type=string,JSONPath=`.status.macAddress`
-// +kubebuilder:printcolumn:name="GatewayIP",type=string,JSONPath=`.status.gatewayIP`
 type MultitenantPodNetworkConfig struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
@@ -51,16 +47,38 @@ type MultitenantPodNetworkConfigSpec struct {
 	PodName string `json:"podName,omitempty"`
 }
 
+type InterfaceInfo struct {
+	// NCID is the network container id
+	NCID string `json:"ncID,omitempty"`
+	// PrimaryIP is the ip allocated to the network container
+	// +kubebuilder:validation:Optional
+	PrimaryIP string `json:"primaryIP,omitempty"`
+	// MacAddress is the MAC Address of the VM's NIC which this network container was created for
+	MacAddress string `json:"macAddress,omitempty"`
+	// GatewayIP is the gateway ip of the injected subnet
+	// +kubebuilder:validation:Optional
+	GatewayIP string `json:"gatewayIP,omitempty"`
+	// DeviceType is the device type that this NC was created for
+	DeviceType DeviceType `json:"deviceType,omitempty"`
+}
+
 // MultitenantPodNetworkConfigStatus defines the observed state of PodNetworkConfig
 type MultitenantPodNetworkConfigStatus struct {
-	// network container id
+	// Deprecated - use InterfaceInfos
+	// +kubebuilder:validation:Optional
 	NCID string `json:"ncID,omitempty"`
-	// ip allocated to the network container
+	// Deprecated - use InterfaceInfos
+	// +kubebuilder:validation:Optional
 	PrimaryIP string `json:"primaryIP,omitempty"`
-	// maps to the NIC to be injected for the network container
+	// Deprecated - use InterfaceInfos
+	// +kubebuilder:validation:Optional
 	MacAddress string `json:"macAddress,omitempty"`
-	// Gateway IP
+	// Deprecated - use InterfaceInfos
+	// +kubebuilder:validation:Optional
 	GatewayIP string `json:"gatewayIP,omitempty"`
+	// InterfaceInfos describes all of the network container goal state for this Pod
+	// +kubebuilder:validation:Optional
+	InterfaceInfos []InterfaceInfo `json:"interfaceInfos,omitempty"`
 }
 
 func init() {

crd/multitenancy/api/v1alpha1/podnetwork.go

@@ -16,9 +16,10 @@ import (
 // +kubebuilder:subresource:status
 // +kubebuilder:printcolumn:name="Status",type=string,priority=1,JSONPath=`.status.status`
 // +kubebuilder:printcolumn:name="Address Prefixes",type=string,priority=1,JSONPath=`.status.addressPrefixes`
-// +kubebuilder:printcolumn:name="Network",type=string,priority=1,JSONPath=`.spec.vnetGUID`
+// +kubebuilder:printcolumn:name="Network",type=string,priority=1,JSONPath=`.spec.networkID`
 // +kubebuilder:printcolumn:name="Subnet",type=string,priority=1,JSONPath=`.spec.subnetResourceID`
 // +kubebuilder:printcolumn:name="SubnetGUID",type=string,priority=1,JSONPath=`.spec.subnetGUID`
+// +kubebuilder:printcolumn:name="DeviceType",type=string,priority=1,JSONPath=`.spec.subnetGUID`
 type PodNetwork struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
@@ -36,15 +37,31 @@ type PodNetworkList struct {
 	Items           []PodNetwork `json:"items"`
 }
 
+// +kubebuilder:validation:Enum=acn.azure.com/vnet-nic;acn.azure.com/infiniband-nic
+type DeviceType string
+
+const (
+	DeviceTypeVnetNIC       DeviceType = "acn.azure.com/vnet-nic"
+	DeviceTypeInfiniBandNIC DeviceType = "acn.azure.com/infiniband-nic"
+)
+
 // PodNetworkSpec defines the desired state of PodNetwork
 type PodNetworkSpec struct {
+	// NetworkID is the identifier for the network, e.g. vnet guid or IB network ID
 	// +kubebuilder:validation:Optional
-	// customer vnet guid
-	VnetGUID string `json:"vnetGUID,omitempty"`
+	NetworkID string `json:"networkID,omitempty"`
+	// DeviceType is the device type that is required by this network
+	// +kubebuilder:validation:Optional
+	DeviceType DeviceType `json:"deviceType,omitempty"`
 	// customer subnet id
+	// +kubebuilder:validation:Optional
 	SubnetResourceID string `json:"subnetResourceID,omitempty"`
 	// customer subnet guid
+	// +kubebuilder:validation:Optional
 	SubnetGUID string `json:"subnetGUID,omitempty"`
+	// Deprecated - Use NetworkID
+	// +kubebuilder:validation:Optional
+	VnetGUID string `json:"vnetGUID,omitempty"`
 }
 
 // Status indicates the status of PN

crd/multitenancy/api/v1alpha1/podnetworkinstance.go

@@ -16,9 +16,8 @@ import (
 // +kubebuilder:subresource:status
 // +kubebuilder:metadata:labels=managed=
 // +kubebuilder:metadata:labels=owner=
-// +kubebuilder:printcolumn:name="Pod IPs",type=string,priority=1,JSONPath=`.status.podIPAddresses`
-// +kubebuilder:printcolumn:name="PodNetwork",type=string,priority=1,JSONPath=`.spec.podNetwork`
-// +kubebuilder:printcolumn:name="PodIPReservationSize",type=string,priority=1,JSONPath=`.spec.podIPReservationSize`
+// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.status`
+// +kubebuilder:printcolumn:name="PodNetworks",priority=1,type=string,JSONPath=`.spec.podNetworks`
 type PodNetworkInstance struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
@@ -36,19 +35,35 @@ type PodNetworkInstanceList struct {
 	Items           []PodNetworkInstance `json:"items"`
 }
 
+// PodNetworkConfig describes a template for how to attach a PodNetwork to a Pod
+type PodNetworkConfig struct {
+	// PodNetwork is the name of a PodNetwork resource
+	PodNetwork string `json:"podNetwork"`
+	// PodIPReservationSize is the number of IP address to statically reserve
+	// +kubebuilder:default=0
+	PodIPReservationSize int `json:"podIPReservationSize,omitempty"`
+}
+
 // PodNetworkInstanceSpec defines the desired state of PodNetworkInstance
 type PodNetworkInstanceSpec struct {
-	// pod network resource object name
-	PodNetwork string `json:"podnetwork"`
-	// number of backend IP address to reserve for running pods
+	// Deprecated - use PodNetworks
+	// +kubebuilder:validation:Optional
+	PodNetwork string `json:"podnetwork,omitempty"`
+	// Deprecated - use PodNetworks
 	// +kubebuilder:default=0
-	PodIPReservationSize int `json:"podIPReservationSize"`
+	PodIPReservationSize int `json:"podIPReservationSize,omitempty"`
+	// PodNetworkConfigs describes each PodNetwork to attach to a single Pod
+	// optional for now in case orchestrator uses the deprecated fields
+	// +kubebuilder:validation:Optional
+	PodNetworkConfigs []PodNetworkConfig `json:"podNetworkConfigs"`
 }
 
 // PodNetworkInstanceStatus defines the observed state of PodNetworkInstance
 type PodNetworkInstanceStatus struct {
-	PodIPAddresses []string  `json:"podIPAddresses,omitempty"`
-	Status         PNIStatus `json:"status,omitempty"`
+	// +kubebuilder:validation:Optional
+	PodIPAddresses     []string             `json:"podIPAddresses,omitempty"`
+	Status             PNIStatus            `json:"status,omitempty"`
+	PodNetworkStatuses map[string]PNIStatus `json:"podNetworkStatuses,omitempty"`
 }
 
 // PNIStatus indicates the status of PNI

crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml

@@ -29,18 +29,6 @@ spec:
     - jsonPath: .spec.podName
       name: PodName
       type: string
-    - jsonPath: .status.ncID
-      name: NCID
-      type: string
-    - jsonPath: .status.primaryIP
-      name: PrimaryIP
-      type: string
-    - jsonPath: .status.macAddress
-      name: MacAddress
-      type: string
-    - jsonPath: .status.gatewayIP
-      name: GatewayIP
-      type: string
     name: v1alpha1
     schema:
       openAPIV3Schema:
@@ -80,16 +68,43 @@ spec:
               of PodNetworkConfig
             properties:
               gatewayIP:
-                description: Gateway IP
+                description: Deprecated - use InterfaceInfos
                 type: string
+              interfaceInfos:
+                description: InterfaceInfos describes all of the network container
+                  goal state for this Pod
+                items:
+                  properties:
+                    deviceType:
+                      description: DeviceType is the device type that this NC was
+                        created for
+                      enum:
+                      - acn.azure.com/vnet-nic
+                      - acn.azure.com/infiniband-nic
+                      type: string
+                    gatewayIP:
+                      description: GatewayIP is the gateway ip of the injected subnet
+                      type: string
+                    macAddress:
+                      description: MacAddress is the MAC Address of the VM's NIC which
+                        this network container was created for
+                      type: string
+                    ncID:
+                      description: NCID is the network container id
+                      type: string
+                    primaryIP:
+                      description: PrimaryIP is the ip allocated to the network container
+                      type: string
+                  type: object
+                type: array
               macAddress:
-                description: maps to the NIC to be injected for the network container
+                description: Deprecated - use InterfaceInfos
                 type: string
               ncID:
-                description: network container id
+                description: Deprecated - use InterfaceInfos
                 type: string
               primaryIP:
-                description: ip allocated to the network container
+                description: Deprecated - use InterfaceInfos
                 type: string
             type: object
         type: object

crd/multitenancy/manifests/multitenancy.acn.azure.com_podnetworkinstances.yaml

@@ -20,16 +20,11 @@ spec:
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
-    - jsonPath: .status.podIPAddresses
-      name: Pod IPs
-      priority: 1
-      type: string
-    - jsonPath: .spec.podNetwork
-      name: PodNetwork
-      priority: 1
+    - jsonPath: .status.status
+      name: Status
       type: string
-    - jsonPath: .spec.podIPReservationSize
-      name: PodIPReservationSize
+    - jsonPath: .spec.podNetworks
+      name: PodNetworks
       priority: 1
       type: string
     name: v1alpha1
@@ -55,14 +50,31 @@ spec:
             properties:
               podIPReservationSize:
                 default: 0
-                description: number of backend IP address to reserve for running pods
+                description: Deprecated - use PodNetworks
                 type: integer
+              podNetworkConfigs:
+                description: PodNetworkConfigs describes each PodNetwork to attach
+                  to a single Pod optional for now in case orchestrator uses the deprecated
+                  fields
+                items:
+                  description: PodNetworkConfig describes a template for how to attach
+                    a PodNetwork to a Pod
+                  properties:
+                    podIPReservationSize:
+                      default: 0
+                      description: PodIPReservationSize is the number of IP address
+                        to statically reserve
+                      type: integer
+                    podNetwork:
+                      description: PodNetwork is the name of a PodNetwork resource
+                      type: string
+                  required:
+                  - podNetwork
+                  type: object
+                type: array
               podnetwork:
-                description: pod network resource object name
+                description: Deprecated - use PodNetworks
                 type: string
-            required:
-            - podIPReservationSize
-            - podnetwork
             type: object
           status:
             description: PodNetworkInstanceStatus defines the observed state of PodNetworkInstance
@@ -71,6 +83,16 @@ spec:
                 items:
                   type: string
                 type: array
+              podNetworkStatuses:
+                additionalProperties:
+                  description: PNIStatus indicates the status of PNI
+                  enum:
+                  - Ready
+                  - CreateReservationSetError
+                  - PodNetworkNotReady
+                  - InsufficientIPAddressesOnSubnet
+                  type: string
+                type: object
               status:
                 description: PNIStatus indicates the status of PNI
                 enum: