Skip to content

feat(controller): support vmnetcfg disabled mode #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 22, 2024
Merged

feat(controller): support vmnetcfg disabled mode #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 17 additions & 6 deletions pkg/apis/network.harvesterhci.io/v1alpha1/virtualmachinenetworkconfig.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,18 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
AllocatedState NetworkConfigState = "Allocated"
PendingState NetworkConfigState = "Pending"
)

var (
Allocated condition.Cond = "Allocated"
Disabled condition.Cond = "Disabled"
)

type NetworkConfigState string

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +kubebuilder:resource:shortName=vmnetcfg;vmnetcfgs,scope=Namespaced
Expand All @@ -29,25 +36,29 @@ type VirtualMachineNetworkConfig struct {
type VirtualMachineNetworkConfigSpec struct {
VMName string `json:"vmName,omitempty"`
NetworkConfig []NetworkConfig `json:"networkConfig,omitempty"`

// +optional
Paused *bool `json:"paused,omitempty"`
}

type NetworkConfig struct {
NetworkName string `json:"networkName,omitempty"`
MACAddress string `json:"macAddress,omitempty"`

// +optional
IPAddress *string `json:"ipAddress,omitempty"`
}

type VirtualMachineNetworkConfigStatus struct {
NetworkConfig []NetworkConfigStatus `json:"networkConfig,omitempty"`
// Conditions is a list of Wrangler conditions that describe the state
// of the VirtualMachineNetworkConfigStatus.

// +optional
Conditions []genericcondition.GenericCondition `json:"conditions,omitempty"`
}

type NetworkConfigStatus struct {
AllocatedIPAddress string `json:"allocatedIPAddress,omitempty"`
MACAddress string `json:"macAddress,omitempty"`
NetworkName string `json:"networkName,omitempty"`
Status string `json:"status,omitempty"`
AllocatedIPAddress string `json:"allocatedIPAddress,omitempty"`
MACAddress string `json:"macAddress,omitempty"`
NetworkName string `json:"networkName,omitempty"`
State NetworkConfigState `json:"state,omitempty"`
}
102 changes: 83 additions & 19 deletions pkg/controller/vmnetcfg/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,14 +59,50 @@ func Register(ctx context.Context, management *config.Management) error {
handler.Allocate,
)

vmnetcfgs.OnChange(ctx, controllerName, handler.OnChange)
vmnetcfgs.OnRemove(ctx, controllerName, handler.OnRemove)

return nil
}

func (h *Handler) OnChange(key string, vmNetCfg *networkv1.VirtualMachineNetworkConfig) (*networkv1.VirtualMachineNetworkConfig, error) {
if vmNetCfg == nil || vmNetCfg.DeletionTimestamp != nil {
return nil, nil
}

logrus.Debugf("(vmnetcfg.OnChange) vmnetcfg configuration %s has been changed: %+v", key, vmNetCfg.Spec.NetworkConfig)

vmNetCfgCpy := vmNetCfg.DeepCopy()

// Check if the VirtualMachineNetworkConfig is administratively disabled
if vmNetCfg.Spec.Paused != nil && *vmNetCfg.Spec.Paused {
logrus.Infof("(vmnetcfg.OnChange) try to cleanup ipam and cache, and update ippool status for vmnetcfg %s", key)
if err := h.cleanup(vmNetCfg); err != nil {
return vmNetCfg, err
}
networkv1.Disabled.True(vmNetCfgCpy)
updateAllNetworkConfigState(vmNetCfgCpy.Status.NetworkConfig)
if !reflect.DeepEqual(vmNetCfgCpy, vmNetCfg) {
return h.vmnetcfgClient.UpdateStatus(vmNetCfgCpy)
}
return vmNetCfg, nil
}
networkv1.Disabled.False(vmNetCfgCpy)

if !reflect.DeepEqual(vmNetCfgCpy, vmNetCfg) {
return h.vmnetcfgClient.UpdateStatus(vmNetCfgCpy)
}

return vmNetCfg, nil
}

func (h *Handler) Allocate(vmNetCfg *networkv1.VirtualMachineNetworkConfig, status networkv1.VirtualMachineNetworkConfigStatus) (networkv1.VirtualMachineNetworkConfigStatus, error) {
logrus.Debugf("(vmnetcfg.Allocate) allocate ip for vmnetcfg %s/%s", vmNetCfg.Namespace, vmNetCfg.Name)

if vmNetCfg.Spec.Paused != nil && *vmNetCfg.Spec.Paused {
return status, fmt.Errorf("vmnetcfg %s/%s was administratively disabled", vmNetCfg.Namespace, vmNetCfg.Name)
}

var ncStatuses []networkv1.NetworkConfigStatus
for _, nc := range vmNetCfg.Spec.NetworkConfig {
exists, err := h.cacheAllocator.HasMAC(nc.NetworkName, nc.MACAddress)
Expand All @@ -75,6 +111,7 @@ func (h *Handler) Allocate(vmNetCfg *networkv1.VirtualMachineNetworkConfig, stat
}
if exists {
// Recover IP from cache

ip, err := h.cacheAllocator.GetIPByMAC(nc.NetworkName, nc.MACAddress)
if err != nil {
return status, err
Expand All @@ -85,7 +122,7 @@ func (h *Handler) Allocate(vmNetCfg *networkv1.VirtualMachineNetworkConfig, stat
AllocatedIPAddress: ip,
MACAddress: nc.MACAddress,
NetworkName: nc.NetworkName,
Status: "Allocated",
State: networkv1.AllocatedState,
}
ncStatuses = append(ncStatuses, ncStatus)

Expand All @@ -95,7 +132,7 @@ func (h *Handler) Allocate(vmNetCfg *networkv1.VirtualMachineNetworkConfig, stat
ncStatus.NetworkName,
ncStatus.MACAddress,
ncStatus.AllocatedIPAddress,
ncStatus.Status,
string(ncStatus.State),
)

// Update IPPool status
Expand Down Expand Up @@ -139,10 +176,15 @@ func (h *Handler) Allocate(vmNetCfg *networkv1.VirtualMachineNetworkConfig, stat
}

// Allocate new IP

dIP := util.UnspecifiedIPAddress
if nc.IPAddress != nil {
dIP = *nc.IPAddress
}
// Recover IP from status (resume from paused state)
if oIP, err := findIPAddressFromNetworkConfigStatusByMACAddress(vmNetCfg.Status.NetworkConfig, nc.MACAddress); err == nil {
dIP = oIP
}

ip, err := h.ipAllocator.AllocateIP(nc.NetworkName, dIP)
if err != nil {
Expand All @@ -158,7 +200,7 @@ func (h *Handler) Allocate(vmNetCfg *networkv1.VirtualMachineNetworkConfig, stat
AllocatedIPAddress: ip,
MACAddress: nc.MACAddress,
NetworkName: nc.NetworkName,
Status: "Allocated",
State: networkv1.AllocatedState,
}
ncStatuses = append(ncStatuses, ncStatus)

Expand All @@ -168,7 +210,7 @@ func (h *Handler) Allocate(vmNetCfg *networkv1.VirtualMachineNetworkConfig, stat
ncStatus.NetworkName,
ncStatus.MACAddress,
ncStatus.AllocatedIPAddress,
ncStatus.Status,
string(ncStatus.State),
)

// Update IPPool status
Expand Down Expand Up @@ -221,32 +263,40 @@ func (h *Handler) OnRemove(key string, vmNetCfg *networkv1.VirtualMachineNetwork

logrus.Debugf("(vmnetcfg.OnRemove) vmnetcfg configuration %s/%s has been removed", vmNetCfg.Namespace, vmNetCfg.Name)

h.metricsAllocator.DeleteVmNetCfgStatus(key)
if err := h.cleanup(vmNetCfg); err != nil {
return vmNetCfg, err
}

return vmNetCfg, nil
}

func (h *Handler) cleanup(vmNetCfg *networkv1.VirtualMachineNetworkConfig) error {
h.metricsAllocator.DeleteVmNetCfgStatus(vmNetCfg.Namespace + "/" + vmNetCfg.Name)

for _, nc := range vmNetCfg.Status.NetworkConfig {
for _, ncStatus := range vmNetCfg.Status.NetworkConfig {
// Deallocate IP address from IPAM
isAllocated, err := h.ipAllocator.IsAllocated(nc.NetworkName, nc.AllocatedIPAddress)
isAllocated, err := h.ipAllocator.IsAllocated(ncStatus.NetworkName, ncStatus.AllocatedIPAddress)
if err != nil {
return vmNetCfg, err
return err
}
if isAllocated {
if err := h.ipAllocator.DeallocateIP(nc.NetworkName, nc.AllocatedIPAddress); err != nil {
return vmNetCfg, err
if err := h.ipAllocator.DeallocateIP(ncStatus.NetworkName, ncStatus.AllocatedIPAddress); err != nil {
return err
}
}

// Remove entry from cache
exists, err := h.cacheAllocator.HasMAC(nc.NetworkName, nc.MACAddress)
exists, err := h.cacheAllocator.HasMAC(ncStatus.NetworkName, ncStatus.MACAddress)
if err != nil {
return vmNetCfg, err
return err
}
if exists {
if err := h.cacheAllocator.DeleteMAC(nc.NetworkName, nc.MACAddress); err != nil {
return vmNetCfg, err
if err := h.cacheAllocator.DeleteMAC(ncStatus.NetworkName, ncStatus.MACAddress); err != nil {
return err
}
}

ipPoolNamespace, ipPoolName := kv.RSplit(nc.NetworkName, "/")
ipPoolNamespace, ipPoolName := kv.RSplit(ncStatus.NetworkName, "/")
if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
ipPool, err := h.ippoolCache.Get(ipPoolNamespace, ipPoolName)
if err != nil {
Expand All @@ -256,20 +306,34 @@ func (h *Handler) OnRemove(key string, vmNetCfg *networkv1.VirtualMachineNetwork
ipPoolCpy := ipPool.DeepCopy()

// Remove record in IPPool status
delete(ipPoolCpy.Status.IPv4.Allocated, nc.AllocatedIPAddress)
delete(ipPoolCpy.Status.IPv4.Allocated, ncStatus.AllocatedIPAddress)

if !reflect.DeepEqual(ipPoolCpy, ipPool) {
logrus.Infof("(vmnetcfg.OnRemove) update ippool %s/%s", ipPool.Namespace, ipPool.Name)
logrus.Infof("(vmnetcfg.cleanup) update ippool %s/%s", ipPool.Namespace, ipPool.Name)
ipPoolCpy.Status.LastUpdate = metav1.Now()
_, err := h.ippoolClient.UpdateStatus(ipPoolCpy)
return err
}

return nil
}); err != nil {
return vmNetCfg, err
return err
}
}
return nil
}

return vmNetCfg, nil
func findIPAddressFromNetworkConfigStatusByMACAddress(ncStatuses []networkv1.NetworkConfigStatus, macAddress string) (ipAddress string, err error) {
for _, ncStatus := range ncStatuses {
if ncStatus.MACAddress == macAddress && ncStatus.AllocatedIPAddress != "" {
return ncStatus.AllocatedIPAddress, nil
}
}
return util.UnspecifiedIPAddress, fmt.Errorf("could not find allocated ip for mac %s", macAddress)
}

func updateAllNetworkConfigState(ncStatuses []networkv1.NetworkConfigStatus) {
for i := range ncStatuses {
ncStatuses[i].State = networkv1.PendingState
}
}
8 changes: 4 additions & 4 deletions pkg/metrics/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ var (
LabelVmNetCfgName = "vmnetcfg"
LabelMACAddress = "mac"
LabelIPAddress = "ip"
LabelStatus = "status"
LabelState = "state"
)

type MetricsAllocator struct {
Expand Down Expand Up @@ -60,7 +60,7 @@ func NewMetricsAllocator() *MetricsAllocator {
LabelNetworkName,
LabelMACAddress,
LabelIPAddress,
LabelStatus,
LabelState,
},
),
}
Expand Down Expand Up @@ -103,13 +103,13 @@ func (a *MetricsAllocator) DeleteIPPool(name string, cidr string, networkName st
})
}

func (a *MetricsAllocator) UpdateVmNetCfgStatus(name, networkName, macAddress, ipAddress, status string) {
func (a *MetricsAllocator) UpdateVmNetCfgStatus(name, networkName, macAddress, ipAddress, state string) {
a.vmNetCfgStatus.With(prometheus.Labels{
LabelVmNetCfgName: name,
LabelNetworkName: networkName,
LabelMACAddress: macAddress,
LabelIPAddress: ipAddress,
LabelStatus: status,
LabelState: state,
}).Set(float64(1))
}

Expand Down