Implement ESX watch controller

The controller fetches the maintenance state of an ESX host, which a
Kubernetes node may run on, and attaches it as a label.

esx/check.go

@@ -0,0 +1,121 @@
+/*******************************************************************************
+*
+* Copyright 2020 SAP SE
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You should have received a copy of the License along with this
+* program. If not, you may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+*******************************************************************************/
+
+package esx
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/vmware/govmomi/property"
+	"github.com/vmware/govmomi/view"
+	"github.com/vmware/govmomi/vim25/mo"
+)
+
+type ESXMaintenance string
+
+const NoMaintenance ESXMaintenance = "false"
+
+const InMaintenance ESXMaintenance = "true"
+
+const NotRequired ESXMaintenance = "not-required"
+
+// Timestamps tracks the last time esx hosts haven been checked.
+type Timestamps struct {
+	// Specifies how often the vCenter is queried for a specific esx host.
+	Interval time.Duration
+	// Maps an ESX Hosts to the time it was checked
+	lastChecks map[string]time.Time
+}
+
+func NewTimestamps() Timestamps {
+	return Timestamps{
+		Interval:   1 * time.Minute,
+		lastChecks: make(map[string]time.Time),
+	}
+}
+
+// Returns true if an esx host needs to be checked for maintenance.
+func (t *Timestamps) CheckRequired(host string) bool {
+	t.clean()
+	lastCheck, ok := t.lastChecks[host]
+	if !ok {
+		return true
+	}
+	return time.Since(lastCheck) > t.Interval
+}
+
+// Sets the time the given esx host was checked to time.Now().
+func (t *Timestamps) MarkChecked(host string) {
+	t.lastChecks[host] = time.Now()
+}
+
+// Cleanup not recently checked esx hosts to avoid "leaking" memory, if esx hosts get removed.
+func (t *Timestamps) clean() {
+	for host, stamp := range t.lastChecks {
+		if time.Since(stamp) > t.Interval {
+			delete(t.lastChecks, host)
+		}
+	}
+}
+
+// Describes an ESX host within an availability zone.
+type Host struct {
+	Name             string
+	AvailabilityZone string
+}
+
+type CheckParameters struct {
+	VCenters   *VCenters
+	Timestamps *Timestamps
+	Host       Host
+}
+
+// Performs a check for the specified host if allowed by timestamps.
+func CheckForMaintenance(ctx context.Context, params CheckParameters) (ESXMaintenance, error) {
+	if !params.Timestamps.CheckRequired(params.Host.Name) {
+		return NotRequired, nil
+	}
+	// Do the check
+	client, err := params.VCenters.Client(ctx, params.Host.AvailabilityZone)
+	if err != nil {
+		return NoMaintenance, fmt.Errorf("Failed to check for esx host maintenance state: %w", err)
+	}
+	mgr := view.NewManager(client.Client)
+	view, err := mgr.CreateContainerView(context.Background(), client.ServiceContent.RootFolder,
+		[]string{"HostSystem"}, true)
+	if err != nil {
+		return NoMaintenance, fmt.Errorf("Failed to create container view: %w", err)
+	}
+	var hss []mo.HostSystem
+	err = view.RetrieveWithFilter(context.Background(), []string{"HostSystem"}, []string{"runtime"},
+		&hss, property.Filter{"name": params.Host.Name})
+	if err != nil {
+		return NoMaintenance, fmt.Errorf("Failed to fetch runtime information for esx host %v: %w", params.Host.Name, err)
+	}
+	if len(hss) != 1 {
+		return NoMaintenance, fmt.Errorf("Expected to retrieve 1 esx host from vCenter, but got %v", len(hss))
+	}
+	params.Timestamps.MarkChecked(params.Host.Name)
+	if hss[0].Runtime.InMaintenanceMode {
+		return InMaintenance, nil
+	}
+	return NoMaintenance, nil
+}

esx/check_test.go

@@ -0,0 +1,137 @@
+/*******************************************************************************
+*
+* Copyright 2020 SAP SE
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You should have received a copy of the License along with this
+* program. If not, you may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+*******************************************************************************/
+
+package esx
+
+import (
+	"context"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vmware/govmomi/object"
+	"github.com/vmware/govmomi/vim25/types"
+)
+
+var _ = Describe("Timestamps", func() {
+
+	It("should pass if time since last timestamp > Interval", func() {
+		timestamps := NewTimestamps()
+		timestamps.lastChecks["host"] = time.Now().Add(-2 * timestamps.Interval)
+		result := timestamps.CheckRequired("host")
+		Expect(result).To(BeTrue())
+	})
+
+	It("should not pass if time since last timestamp < Interval", func() {
+		timestamps := NewTimestamps()
+		timestamps.lastChecks["host"] = time.Now().Add(timestamps.Interval / -2)
+		result := timestamps.CheckRequired("host")
+		Expect(result).To(BeFalse())
+	})
+
+	It("should pass if the host is not tracked", func() {
+		timestamps := NewTimestamps()
+		result := timestamps.CheckRequired("host")
+		Expect(result).To(BeTrue())
+	})
+
+	It("marks hosts as cheked", func() {
+		timestamps := NewTimestamps()
+		timestamps.MarkChecked("host")
+		Expect(timestamps.lastChecks).To(HaveKey("host"))
+	})
+
+})
+
+const HostSystemName string = "DC0_H0"
+
+var _ = Describe("Do", func() {
+	var vCenters *VCenters
+
+	BeforeEach(func() {
+		vCenters = &VCenters{
+			Template: "http://" + AvailabilityZoneReplacer,
+			Credentials: map[string]Credential{
+				vcServer.URL.Host: {
+					Username: "user",
+					Password: "pass",
+				},
+			},
+		}
+
+		// set host out of maintenance
+		client, err := vCenters.Client(context.Background(), vcServer.URL.Host)
+		Expect(err).To(Succeed())
+		host := object.NewHostSystem(client.Client, types.ManagedObjectReference{
+			Type:  "HostSystem",
+			Value: "host-21",
+		})
+		task, err := host.ExitMaintenanceMode(context.Background(), 1000)
+		Expect(err).To(Succeed())
+		err = task.Wait(context.Background())
+		Expect(err).To(Succeed())
+	})
+
+	It("should return NoMaintenance if the host is not in maintenance", func() {
+		timestamps := NewTimestamps()
+		result, err := CheckForMaintenance(context.Background(), CheckParameters{vCenters, &timestamps, Host{
+			AvailabilityZone: vcServer.URL.Host,
+			Name:             HostSystemName,
+		}})
+		Expect(err).To(Succeed())
+		Expect(result).To(Equal(NoMaintenance))
+		Expect(timestamps.lastChecks).To(HaveKey(HostSystemName))
+	})
+
+	It("should return InMaintenance if the host is in maintenance", func() {
+		client, err := vCenters.Client(context.Background(), vcServer.URL.Host)
+		Expect(err).To(Succeed())
+
+		// set host in maintenance
+		host := object.NewHostSystem(client.Client, types.ManagedObjectReference{
+			Type:  "HostSystem",
+			Value: "host-21",
+		})
+		task, err := host.EnterMaintenanceMode(context.Background(), 1000, false, &types.HostMaintenanceSpec{})
+		Expect(err).To(Succeed())
+		err = task.Wait(context.Background())
+		Expect(err).To(Succeed())
+
+		timestamps := NewTimestamps()
+		result, err := CheckForMaintenance(context.Background(), CheckParameters{vCenters, &timestamps, Host{
+			AvailabilityZone: vcServer.URL.Host,
+			Name:             HostSystemName,
+		}})
+		Expect(err).To(Succeed())
+		Expect(result).To(Equal(InMaintenance))
+		Expect(timestamps.lastChecks).To(HaveKey(HostSystemName))
+	})
+
+	It("should respect the check interval", func() {
+		timestamps := NewTimestamps()
+		timestamps.lastChecks[HostSystemName] = time.Now()
+		result, err := CheckForMaintenance(context.Background(), CheckParameters{vCenters, &timestamps, Host{
+			AvailabilityZone: vcServer.URL.Host,
+			Name:             HostSystemName,
+		}})
+		Expect(err).To(Succeed())
+		Expect(result).To(Equal(NotRequired))
+	})
+
+})

esx/config.go

@@ -0,0 +1,82 @@
+/*******************************************************************************
+*
+* Copyright 2020 SAP SE
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You should have received a copy of the License along with this
+* program. If not, you may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+*******************************************************************************/
+
+package esx
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/vmware/govmomi"
+)
+
+// Specifies the string in a vCenter URL, which is replaced by the availability zone.
+const AvailabilityZoneReplacer string = "$AZ"
+
+type Config struct {
+	Intervals struct {
+		Node time.Duration `config:"node" validate:"required"`
+		ESX  time.Duration `config:"esx" validate:"required"`
+	} `config:"intervals" validate:"required"`
+	VCenters VCenters `config:"vCenters" validate:"required"`
+}
+
+type Credential struct {
+	Username string `config:"username" validate:"required"`
+	Password string `config:"password" validate:"required"`
+}
+
+// VCenters contains connection information to regional vCenters.
+type VCenters struct {
+	// URL to regional vCenters with the availability zone replaced by AvailabilityZoneReplacer.
+	Template string `config:"templateUrl" validate:"required"`
+	// Pair of credentials per availability zone.
+	Credentials map[string]Credential `config:"credentials" validate:"required"`
+}
+
+// Gets an URL to connect to a vCenters in a specific availability zone.
+func (vc *VCenters) URL(availabilityZone string) (*url.URL, error) {
+	withAZ := strings.ReplaceAll(vc.Template+"/sdk", AvailabilityZoneReplacer, availabilityZone)
+	vCenterURL, err := url.Parse(withAZ)
+	if err != nil {
+		return nil, err
+	}
+	cred, ok := vc.Credentials[availabilityZone]
+	if !ok {
+		return nil, fmt.Errorf("No vCenter credentials have been provided for availability zone %v", availabilityZone)
+	}
+	vCenterURL.User = url.UserPassword(cred.Username, cred.Password)
+	return vCenterURL, nil
+}
+
+// Returns a ready to use vCenter client for the given availability zone.
+func (vc *VCenters) Client(ctx context.Context, availabilityZone string) (*govmomi.Client, error) {
+	url, err := vc.URL(availabilityZone)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to render vCenter URL: %w", err)
+	}
+	client, err := govmomi.NewClient(ctx, url, false)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to create vCenter client: %w", err)
+	}
+	return client, nil
+}

esx/config_test.go

@@ -0,0 +1,53 @@
+/*******************************************************************************
+*
+* Copyright 2020 SAP SE
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You should have received a copy of the License along with this
+* program. If not, you may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+*******************************************************************************/
+
+package esx
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("VCenters", func() {
+
+	It("should create valid vCenter URLs", func() {
+		vCenters := VCenters{
+			Template: "https://region-$AZ.local",
+			Credentials: map[string]Credential{
+				"a": {
+					Username: "user1",
+					Password: "pw1",
+				},
+				"b": {
+					Username: "user2",
+					Password: "pw2",
+				},
+			},
+		}
+		urlA, err := vCenters.URL("a")
+		Expect(err).To(Succeed())
+		Expect(urlA.String()).To(Equal("https://user1:pw1@region-a.local/sdk"))
+		urlB, err := vCenters.URL("b")
+		Expect(err).To(Succeed())
+		Expect(urlB.String()).To(Equal("https://user2:pw2@region-b.local/sdk"))
+		_, err = vCenters.URL("c")
+		Expect(err).To(HaveOccurred())
+	})
+
+})