Adding chaos-monkey for deployments

Makefile

@@ -65,6 +65,10 @@ ifndef ENTERPRISEIMAGE
 	ENTERPRISEIMAGE := $(DEFAULTENTERPRISEIMAGE)
 endif
 
+ifndef ALLOWCHAOS
+	ALLOWCHAOS := true
+endif
+
 BINNAME := $(PROJECT)
 BIN := $(BINDIR)/$(BINNAME)
 TESTBINNAME := $(PROJECT)_test
@@ -200,7 +204,8 @@ manifests: $(GOBUILDDIR)
 		--output-suffix=$(MANIFESTSUFFIX) \
 		--image=$(OPERATORIMAGE) \
 		--image-sha256=$(IMAGESHA256) \
-		--namespace=$(DEPLOYMENTNAMESPACE)
+		--namespace=$(DEPLOYMENTNAMESPACE) \
+		--allow-chaos=$(ALLOWCHAOS)
 
 # Testing
 

main.go

@@ -79,7 +79,9 @@ var (
 	operatorOptions struct {
 		enableDeployment bool // Run deployment operator
 		enableStorage    bool // Run deployment operator
-		createCRD        bool
+	}
+	chaosOptions struct {
+		allowed bool
 	}
 	deploymentProbe probe.Probe
 	storageProbe    probe.Probe
@@ -92,7 +94,7 @@ func init() {
 	f.StringVar(&logLevel, "log.level", defaultLogLevel, "Set initial log level")
 	f.BoolVar(&operatorOptions.enableDeployment, "operator.deployment", false, "Enable to run the ArangoDeployment operator")
 	f.BoolVar(&operatorOptions.enableStorage, "operator.storage", false, "Enable to run the ArangoLocalStorage operator")
-	f.BoolVar(&operatorOptions.createCRD, "operator.create-crd", true, "Disable to avoid create the custom resource definition")
+	f.BoolVar(&chaosOptions.allowed, "chaos.allowed", false, "Set to allow chaos in deployments. Only activated when allowed and enabled in deployment")
 }
 
 func main() {
@@ -188,7 +190,7 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper
 		ServiceAccount:   serviceAccount,
 		EnableDeployment: operatorOptions.enableDeployment,
 		EnableStorage:    operatorOptions.enableStorage,
-		CreateCRD:        operatorOptions.createCRD,
+		AllowChaos:       chaosOptions.allowed,
 	}
 	deps := operator.Dependencies{
 		LogService:      logService,

manifests/templates/deployment/deployment.yaml

@@ -19,6 +19,7 @@ spec:
         image: {{ .Image }}
         args:
           - --operator.deployment
+          - --chaos.allowed={{ .Deployment.AllowChaos }}
         env:
         - name: MY_POD_NAMESPACE
           valueFrom:

pkg/apis/deployment/v1alpha/chaos_spec.go

@@ -0,0 +1,91 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package v1alpha
+
+import (
+	time "time"
+
+	"github.com/arangodb/kube-arangodb/pkg/util"
+	"github.com/pkg/errors"
+)
+
+// ChaosSpec holds configuration for the deployment chaos monkey.
+type ChaosSpec struct {
+	// Enabled switches the chaos monkey for a deployment on or off.
+	Enabled *bool `json:"enabled,omitempty"`
+	// Interval is the time between events
+	Interval *time.Duration `json:"interval,omitempty"`
+	// KillPodProbability is the chance of a pod being killed during an event
+	KillPodProbability *Percent `json:"kill-pod-probability,omitempty"`
+}
+
+// IsEnabled returns the value of enabled.
+func (s ChaosSpec) IsEnabled() bool {
+	return util.BoolOrDefault(s.Enabled)
+}
+
+// GetInterval returns the value of interval.
+func (s ChaosSpec) GetInterval() time.Duration {
+	return util.DurationOrDefault(s.Interval)
+}
+
+// GetKillPodProbability returns the value of kill-pod-probability.
+func (s ChaosSpec) GetKillPodProbability() Percent {
+	return PercentOrDefault(s.KillPodProbability)
+}
+
+// Validate the given spec
+func (s ChaosSpec) Validate() error {
+	if s.IsEnabled() {
+		if s.GetInterval() <= 0 {
+			return maskAny(errors.Wrapf(ValidationError, "Interval must be > 0"))
+		}
+		if err := s.GetKillPodProbability().Validate(); err != nil {
+			return maskAny(err)
+		}
+	}
+	return nil
+}
+
+// SetDefaults fills in missing defaults
+func (s *ChaosSpec) SetDefaults() {
+	if s.GetInterval() == 0 {
+		s.Interval = util.NewDuration(time.Minute)
+	}
+	if s.GetKillPodProbability() == 0 {
+		s.KillPodProbability = NewPercent(50)
+	}
+}
+
+// SetDefaultsFrom fills unspecified fields with a value from given source spec.
+func (s *ChaosSpec) SetDefaultsFrom(source ChaosSpec) {
+	if s.Enabled == nil {
+		s.Enabled = util.NewBoolOrNil(source.Enabled)
+	}
+	if s.Interval == nil {
+		s.Interval = util.NewDurationOrNil(source.Interval)
+	}
+	if s.KillPodProbability == nil {
+		s.KillPodProbability = NewPercentOrNil(source.KillPodProbability)
+	}
+}

pkg/apis/deployment/v1alpha/deployment_spec.go

@@ -62,6 +62,8 @@ type DeploymentSpec struct {
 	Coordinators ServerGroupSpec `json:"coordinators"`
 	SyncMasters  ServerGroupSpec `json:"syncmasters"`
 	SyncWorkers  ServerGroupSpec `json:"syncworkers"`
+
+	Chaos ChaosSpec `json:"chaos"`
 }
 
 // GetMode returns the value of mode.
@@ -147,6 +149,7 @@ func (s *DeploymentSpec) SetDefaults(deploymentName string) {
 	s.Coordinators.SetDefaults(ServerGroupCoordinators, s.GetMode().HasCoordinators(), s.GetMode())
 	s.SyncMasters.SetDefaults(ServerGroupSyncMasters, s.Sync.IsEnabled(), s.GetMode())
 	s.SyncWorkers.SetDefaults(ServerGroupSyncWorkers, s.Sync.IsEnabled(), s.GetMode())
+	s.Chaos.SetDefaults()
 }
 
 // SetDefaultsFrom fills unspecified fields with a value from given source spec.
@@ -176,6 +179,7 @@ func (s *DeploymentSpec) SetDefaultsFrom(source DeploymentSpec) {
 	s.Coordinators.SetDefaultsFrom(source.Coordinators)
 	s.SyncMasters.SetDefaultsFrom(source.SyncMasters)
 	s.SyncWorkers.SetDefaultsFrom(source.SyncWorkers)
+	s.Chaos.SetDefaultsFrom(source.Chaos)
 }
 
 // Validate the specification.
@@ -226,6 +230,9 @@ func (s *DeploymentSpec) Validate() error {
 	if err := s.SyncWorkers.Validate(ServerGroupSyncWorkers, s.Sync.IsEnabled(), s.GetMode(), s.GetEnvironment()); err != nil {
 		return maskAny(err)
 	}
+	if err := s.Chaos.Validate(); err != nil {
+		return maskAny(errors.Wrap(err, "spec.chaos"))
+	}
 	return nil
 }
 

pkg/apis/deployment/v1alpha/percent.go

@@ -0,0 +1,62 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package v1alpha
+
+import (
+	"github.com/pkg/errors"
+)
+
+// Percent is a percentage between 0 and 100.
+type Percent int
+
+// Validate the given percentage.
+func (p Percent) Validate() error {
+	if p < 0 || p > 100 {
+		return maskAny(errors.Wrapf(ValidationError, "Percentage must be between 0 and 100, got %d", int(p)))
+	}
+	return nil
+}
+
+// NewPercent returns a reference to a percent with given value.
+func NewPercent(input Percent) *Percent {
+	return &input
+}
+
+// NewPercentOrNil returns nil if input is nil, otherwise returns a clone of the given value.
+func NewPercentOrNil(input *Percent) *Percent {
+	if input == nil {
+		return nil
+	}
+	return NewPercent(*input)
+}
+
+// PercentOrDefault returns the default value or 0 if input is nil, otherwise returns the referenced value.
+func PercentOrDefault(input *Percent, defaultValue ...Percent) Percent {
+	if input == nil {
+		if len(defaultValue) > 0 {
+			return defaultValue[0]
+		}
+		return 0
+	}
+	return *input
+}

pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go

@@ -144,6 +144,49 @@ func (in *AuthenticationSpec) DeepCopy() *AuthenticationSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ChaosSpec) DeepCopyInto(out *ChaosSpec) {
+	*out = *in
+	if in.Enabled != nil {
+		in, out := &in.Enabled, &out.Enabled
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(bool)
+			**out = **in
+		}
+	}
+	if in.Interval != nil {
+		in, out := &in.Interval, &out.Interval
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(time.Duration)
+			**out = **in
+		}
+	}
+	if in.KillPodProbability != nil {
+		in, out := &in.KillPodProbability, &out.KillPodProbability
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(Percent)
+			**out = **in
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChaosSpec.
+func (in *ChaosSpec) DeepCopy() *ChaosSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(ChaosSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Condition) DeepCopyInto(out *Condition) {
 	*out = *in
@@ -220,6 +263,7 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 	in.Coordinators.DeepCopyInto(&out.Coordinators)
 	in.SyncMasters.DeepCopyInto(&out.SyncMasters)
 	in.SyncWorkers.DeepCopyInto(&out.SyncWorkers)
+	in.Chaos.DeepCopyInto(&out.Chaos)
 	return
 }
 

pkg/deployment/chaos/context.go

@@ -0,0 +1,40 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package chaos
+
+import (
+	"k8s.io/api/core/v1"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
+)
+
+// Context provides methods to the chaos package.
+type Context interface {
+	// GetSpec returns the current specification of the deployment
+	GetSpec() api.DeploymentSpec
+	// DeletePod deletes a pod with given name in the namespace
+	// of the deployment. If the pod does not exist, the error is ignored.
+	DeletePod(podName string) error
+	// GetOwnedPods returns a list of all pods owned by the deployment.
+	GetOwnedPods() ([]v1.Pod, error)
+}

pkg/deployment/chaos/errors.go

@@ -0,0 +1,29 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package chaos
+
+import "github.com/pkg/errors"
+
+var (
+	maskAny = errors.WithStack
+)