Merge pull request #9 from arangodb/pod-monitoring

Pod monitoring

Author: ewoutp

main.go

@@ -95,6 +95,7 @@ func main() {
 
 // Run the operator
 func cmdMainRun(cmd *cobra.Command, args []string) {
+	goflag.CommandLine.Parse([]string{"-logtostderr"})
 	var err error
 	logService, err = logging.NewService(logLevel)
 	if err != nil {

pkg/apis/arangodb/v1alpha/conditions.go

@@ -0,0 +1,131 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package v1alpha
+
+import (
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// ConditionType is a strongly typed condition name
+type ConditionType string
+
+const (
+	// ConditionTypeReady indicates that the member or entire deployment is ready and running normally.
+	ConditionTypeReady ConditionType = "Ready"
+)
+
+// Condition represents one current condition of a deployment or deployment member.
+// A condition might not show up if it is not happening.
+// For example, if a cluster is not upgrading, the Upgrading condition would not show up.
+type Condition struct {
+	// Type of  condition.
+	Type ConditionType `json:"type"`
+	// Status of the condition, one of True, False, Unknown.
+	Status v1.ConditionStatus `json:"status"`
+	// The last time this condition was updated.
+	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
+	// Last time the condition transitioned from one status to another.
+	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
+	// The reason for the condition's last transition.
+	Reason string `json:"reason,omitempty"`
+	// A human readable message indicating details about the transition.
+	Message string `json:"message,omitempty"`
+}
+
+// ConditionList is a list of conditions.
+// Each type is allowed only once.
+type ConditionList []Condition
+
+// IsTrue return true when a condition with given type exists and its status is `True`.
+func (list ConditionList) IsTrue(conditionType ConditionType) bool {
+	c, found := list.Get(conditionType)
+	return found && c.Status == v1.ConditionTrue
+}
+
+// Get a condition by type.
+// Returns true if found, false if not found.
+func (list ConditionList) Get(conditionType ConditionType) (Condition, bool) {
+	for _, x := range list {
+		if x.Type == conditionType {
+			return x, true
+		}
+	}
+	// Not found
+	return Condition{}, false
+}
+
+// Update the condition, replacing an old condition with same type (if any)
+// Returns true when changes were made, false otherwise.
+func (list *ConditionList) Update(conditionType ConditionType, status bool, reason, message string) bool {
+	src := *list
+	statusX := v1.ConditionFalse
+	if status {
+		statusX = v1.ConditionTrue
+	}
+	for i, x := range src {
+		if x.Type == conditionType {
+			if x.Status != statusX {
+				// Transition to another status
+				src[i].Status = statusX
+				now := metav1.Now()
+				src[i].LastTransitionTime = now
+				src[i].LastUpdateTime = now
+				src[i].Reason = reason
+				src[i].Message = message
+			} else if x.Reason != reason || x.Message != message {
+				src[i].LastUpdateTime = metav1.Now()
+				src[i].Reason = reason
+				src[i].Message = message
+			} else {
+				return false
+			}
+			return true
+		}
+	}
+	// Not found
+	now := metav1.Now()
+	*list = append(src, Condition{
+		Type:               conditionType,
+		LastUpdateTime:     now,
+		LastTransitionTime: now,
+		Status:             statusX,
+		Reason:             reason,
+		Message:            message,
+	})
+	return true
+}
+
+// Remove the condition with given type.
+// Returns true if removed, or false if not found.
+func (list *ConditionList) Remove(conditionType ConditionType) bool {
+	src := *list
+	for i, x := range src {
+		if x.Type == conditionType {
+			*list = append(src[:i], src[i+1:]...)
+			return true
+		}
+	}
+	// Not found
+	return false
+}

pkg/apis/arangodb/v1alpha/deployment_status.go

@@ -22,7 +22,11 @@
 
 package v1alpha
 
-import "github.com/pkg/errors"
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+)
 
 // DeploymentState is a strongly typed state of a deployment
 type DeploymentState string
@@ -64,6 +68,9 @@ type DeploymentStatus struct {
 
 	// Members holds the status for all members in all server groups
 	Members DeploymentStatusMembers `json:"members"`
+
+	// Conditions specific to the entire deployment
+	Conditions ConditionList `json:"conditions,omitempty"`
 }
 
 // DeploymentStatusMembers holds the member status of all server groups
@@ -86,6 +93,95 @@ func (ds DeploymentStatusMembers) ContainsID(id string) bool {
 		ds.SyncWorkers.ContainsID(id)
 }
 
+// ForeachServerGroup calls the given callback for all server groups.
+// If the callback returns an error, this error is returned and the callback is
+// not called for the remaining groups.
+func (ds DeploymentStatusMembers) ForeachServerGroup(cb func(group ServerGroup, list *MemberStatusList) error) error {
+	if err := cb(ServerGroupSingle, &ds.Single); err != nil {
+		return maskAny(err)
+	}
+	if err := cb(ServerGroupAgents, &ds.Agents); err != nil {
+		return maskAny(err)
+	}
+	if err := cb(ServerGroupDBServers, &ds.DBServers); err != nil {
+		return maskAny(err)
+	}
+	if err := cb(ServerGroupCoordinators, &ds.Coordinators); err != nil {
+		return maskAny(err)
+	}
+	if err := cb(ServerGroupSyncMasters, &ds.SyncMasters); err != nil {
+		return maskAny(err)
+	}
+	if err := cb(ServerGroupSyncWorkers, &ds.SyncWorkers); err != nil {
+		return maskAny(err)
+	}
+	return nil
+}
+
+// MemberStatusByPodName returns a reference to the element in the given set of lists that has the given pod name.
+// If no such element exists, nil is returned.
+func (ds DeploymentStatusMembers) MemberStatusByPodName(podName string) (MemberStatus, ServerGroup, bool) {
+	if result, found := ds.Single.ElementByPodName(podName); found {
+		return result, ServerGroupSingle, true
+	}
+	if result, found := ds.Agents.ElementByPodName(podName); found {
+		return result, ServerGroupAgents, true
+	}
+	if result, found := ds.DBServers.ElementByPodName(podName); found {
+		return result, ServerGroupDBServers, true
+	}
+	if result, found := ds.Coordinators.ElementByPodName(podName); found {
+		return result, ServerGroupCoordinators, true
+	}
+	if result, found := ds.SyncMasters.ElementByPodName(podName); found {
+		return result, ServerGroupSyncMasters, true
+	}
+	if result, found := ds.SyncWorkers.ElementByPodName(podName); found {
+		return result, ServerGroupSyncWorkers, true
+	}
+	return MemberStatus{}, 0, false
+}
+
+// UpdateMemberStatus updates the given status in the given group.
+func (ds *DeploymentStatusMembers) UpdateMemberStatus(status MemberStatus, group ServerGroup) error {
+	var err error
+	switch group {
+	case ServerGroupSingle:
+		err = ds.Single.Update(status)
+	case ServerGroupAgents:
+		err = ds.Agents.Update(status)
+	case ServerGroupDBServers:
+		err = ds.DBServers.Update(status)
+	case ServerGroupCoordinators:
+		err = ds.Coordinators.Update(status)
+	case ServerGroupSyncMasters:
+		err = ds.SyncMasters.Update(status)
+	case ServerGroupSyncWorkers:
+		err = ds.SyncWorkers.Update(status)
+	default:
+		return maskAny(errors.Wrapf(NotFoundError, "ServerGroup %d is not known", group))
+	}
+	if err != nil {
+		return maskAny(err)
+	}
+	return nil
+}
+
+// AllMembersReady returns true when all members are in the Ready state.
+func (ds DeploymentStatusMembers) AllMembersReady() bool {
+	if err := ds.ForeachServerGroup(func(group ServerGroup, list *MemberStatusList) error {
+		for _, x := range *list {
+			if !x.Conditions.IsTrue(ConditionTypeReady) {
+				return fmt.Errorf("not ready")
+			}
+		}
+		return nil
+	}); err != nil {
+		return false
+	}
+	return true
+}
+
 // MemberStatusList is a list of MemberStatus entries
 type MemberStatusList []MemberStatus
 
@@ -99,6 +195,17 @@ func (l MemberStatusList) ContainsID(id string) bool {
 	return false
 }
 
+// ElementByPodName returns the element in the given list that has the given pod name and true.
+// If no such element exists, an empty element and false is returned.
+func (l MemberStatusList) ElementByPodName(podName string) (MemberStatus, bool) {
+	for i, x := range l {
+		if x.PodName == podName {
+			return l[i], true
+		}
+	}
+	return MemberStatus{}, false
+}
+
 // Add a member to the list.
 // Returns an AlreadyExistsError if the ID of the given member already exists.
 func (l *MemberStatusList) Add(m MemberStatus) error {
@@ -143,10 +250,8 @@ type MemberState string
 const (
 	// MemberStateNone indicates that the state is not set yet
 	MemberStateNone MemberState = ""
-	// MemberStateCreating indicates that the member is in the process of being created
-	MemberStateCreating MemberState = "Creating"
-	// MemberStateReady indicates that the member is running and reachable
-	MemberStateReady MemberState = "Ready"
+	// MemberStateCreated indicates that all resources needed for the member have been created
+	MemberStateCreated MemberState = "Created"
 	// MemberStateCleanOut indicates that a dbserver is in the process of being cleaned out
 	MemberStateCleanOut MemberState = "CleanOut"
 	// MemberStateShuttingDown indicates that a member is shutting down
@@ -164,4 +269,6 @@ type MemberStatus struct {
 	PersistentVolumeClaimName string `json:"persistentVolumeClaimName,omitempty"`
 	// PodName holds the name of the Pod that currently runs this member
 	PodName string `json:"podName,omitempty"`
+	// Conditions specific to this member
+	Conditions ConditionList `json:"conditions,omitempty"`
 }