Cache govmomi Clients during reconciliation

This should put less strain on the vCenters.

esx/check.go

@@ -94,6 +94,9 @@ func CheckForMaintenance(ctx context.Context, params CheckParameters) (Maintenan
 		if task.Info.Name == "EnterMaintenanceMode_Task" {
 			// do not care about status queued and error
 			// success should already be handled by checking for Runtime.InMaintenanceMode
+			// also recent tasks retains completed tasks for a while
+			// so checking for success could result in returning in-maintenance while the ESX
+			// is actually running again.
 			if task.Info.State == types.TaskInfoStateRunning {
 				return InMaintenance, nil
 			}

esx/config.go

@@ -55,10 +55,12 @@ type Credential struct {
 type VCenters struct {
 	// URL to regional vCenters with the availability zone replaced by AvailabilityZoneReplacer.
 	Template string `config:"templateUrl" validate:"required"`
-	// If true the vCenters certificates are not validated
+	// If true the vCenters certificates are not validated.
 	Insecure bool `config:"insecure"`
 	// Pair of credentials per availability zone.
 	Credentials map[string]Credential `config:"credentials" validate:"required"`
+	// Cache of vCenter clients per AZ.
+	cache map[string]*govmomi.Client `config:",ignore"`
 }
 
 // Gets an URL to connect to a vCenters in a specific availability zone.
@@ -78,6 +80,22 @@ func (vc *VCenters) URL(availabilityZone string) (*url.URL, error) {
 
 // Returns a ready to use vCenter client for the given availability zone.
 func (vc *VCenters) Client(ctx context.Context, availabilityZone string) (*govmomi.Client, error) {
+	if vc.cache == nil {
+		vc.cache = make(map[string]*govmomi.Client)
+	}
+	client, ok := vc.cache[availabilityZone]
+	if ok {
+		return client, nil
+	}
+	client, err := vc.makeClient(ctx, availabilityZone)
+	if err != nil {
+		return nil, err
+	}
+	vc.cache[availabilityZone] = client
+	return client, nil
+}
+
+func (vc *VCenters) makeClient(ctx context.Context, availabilityZone string) (*govmomi.Client, error) {
 	url, err := vc.URL(availabilityZone)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to render vCenter URL: %w", err)
@@ -88,3 +106,11 @@ func (vc *VCenters) Client(ctx context.Context, availabilityZone string) (*govmo
 	}
 	return client, nil
 }
+
+func (vc *VCenters) ClearCache(ctx context.Context) {
+	for _, client := range vc.cache {
+		// try logout, which should clean some resources on the vCenter
+		_ = client.Logout(ctx)
+	}
+	vc.cache = make(map[string]*govmomi.Client)
+}

esx/runnable.go

@@ -116,6 +116,7 @@ func (r *Runnable) Reconcile(ctx context.Context, conf Config) {
 			continue
 		}
 	}
+	conf.VCenters.ClearCache(ctx)
 }
 
 // Checks the maintenance mode of the given ESX and attaches the according Maintenance label.