Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow to suspend chaoskube at certain weekdays #56

Merged
merged 7 commits into from Jan 22, 2018
Merged

Allow to suspend chaoskube at certain weekdays #56

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
898c04d
feat: allow to suspend chaoskube at certain weekdays
Jan 9, 2018
0507097
fix: make weekdays detection aware of time zones
Jan 16, 2018
8f88321
ref: move weekday parsing to utils package
Jan 17, 2018
66f2cc1
fix: some tests only care a about the number of terminated pods
Jan 22, 2018
66b24b8
fix: change default timezone to UTC
Jan 22, 2018
415336d
docs: document weekday filter flag
Jan 22, 2018
439bb00
docs: update comment for New b/c it takes more params
linki Jan 22, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
7 changes: 5 additions & 2 deletions chaoskube/chaoskube.go
View file
Open in desktop
Expand Up @@ -27,6 +27,8 @@ type Chaoskube struct {
Namespaces labels.Selector
// a list of weekdays when termination is suspended
ExcludedWeekdays []time.Weekday
// the timezone to apply when detecting the current weekday
Timezone *time.Location
// an instance of logrus.StdLogger to write log messages to
Logger log.StdLogger
// dry run will not allow any pod terminations
Expand All @@ -49,13 +51,14 @@ var msgWeekdayExcluded = "This day of the week is excluded from chaos."
// New returns a new instance of Chaoskube. It expects a kubernetes client, a
// label and namespace selector to reduce the amount of affected pods as well as
// whether to enable dryRun mode and a seed to seed the randomizer with.
func New(client kubernetes.Interface, labels, annotations, namespaces labels.Selector, excludedWeekdays []time.Weekday, logger log.StdLogger, dryRun bool, seed int64) *Chaoskube {
func New(client kubernetes.Interface, labels, annotations, namespaces labels.Selector, excludedWeekdays []time.Weekday, timezone *time.Location, logger log.StdLogger, dryRun bool, seed int64) *Chaoskube {
c := &Chaoskube{
Client: client,
Labels: labels,
Annotations: annotations,
Namespaces: namespaces,
ExcludedWeekdays: excludedWeekdays,
Timezone: timezone,
Logger: logger,
DryRun: dryRun,
Seed: seed,
Expand Down Expand Up @@ -120,7 +123,7 @@ func (c *Chaoskube) DeletePod(victim v1.Pod) error {
// TerminateVictim picks and deletes a victim if found.
func (c *Chaoskube) TerminateVictim() error {
for _, wd := range c.ExcludedWeekdays {
if wd == c.Now().Weekday() {
if wd == c.Now().In(c.Timezone).Weekday() {
c.Logger.Printf(msgWeekdayExcluded)
return nil
}
Expand Down
69 changes: 47 additions & 22 deletions chaoskube/chaoskube_test.go
View file
Open in desktop
Expand Up @@ -25,7 +25,7 @@ func TestNew(t *testing.T) {
namespaces, _ := labels.Parse("qux")
excludedWeekdays := []time.Weekday{time.Friday}

chaoskube := New(client, labelSelector, annotations, namespaces, excludedWeekdays, logger, false, 42)
chaoskube := New(client, labelSelector, annotations, namespaces, excludedWeekdays, time.UTC, logger, false, 42)

if chaoskube == nil {
t.Errorf("expected Chaoskube but got nothing")
Expand Down Expand Up @@ -55,6 +55,10 @@ func TestNew(t *testing.T) {
t.Errorf("expected %s, got %s", time.Friday.String(), chaoskube.ExcludedWeekdays[0].String())
}

if chaoskube.Timezone != time.UTC {
t.Errorf("expected %#v, got %#v", time.UTC, chaoskube.Timezone)
}

if chaoskube.Logger != logger {
t.Errorf("expected %#v, got %#v", logger, chaoskube.Logger)
}
Expand All @@ -70,7 +74,7 @@ func TestNew(t *testing.T) {

// TestCandidates tests the set of pods available for termination
func TestCandidates(t *testing.T) {
chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{}, false, 0)
chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{}, time.UTC, false, 0)

validateCandidates(t, chaoskube, []map[string]string{
{"namespace": "default", "name": "foo"},
Expand All @@ -86,7 +90,7 @@ func TestCandidatesLabelSelector(t *testing.T) {
t.Fatal(err)
}

chaoskube := setup(t, selector, labels.Everything(), labels.Everything(), []time.Weekday{}, false, 0)
chaoskube := setup(t, selector, labels.Everything(), labels.Everything(), []time.Weekday{}, time.UTC, false, 0)

validateCandidates(t, chaoskube, []map[string]string{
{"namespace": "default", "name": "foo"},
Expand All @@ -100,7 +104,7 @@ func TestCandidatesExcludingLabelSelector(t *testing.T) {
t.Fatal(err)
}

chaoskube := setup(t, selector, labels.Everything(), labels.Everything(), []time.Weekday{}, false, 0)
chaoskube := setup(t, selector, labels.Everything(), labels.Everything(), []time.Weekday{}, time.UTC, false, 0)

validateCandidates(t, chaoskube, []map[string]string{
{"namespace": "testing", "name": "bar"},
Expand All @@ -115,7 +119,7 @@ func TestCandidatesAnnotationSelector(t *testing.T) {
t.Fatal(err)
}

chaoskube := setup(t, labels.Everything(), selector, labels.Everything(), []time.Weekday{}, false, 0)
chaoskube := setup(t, labels.Everything(), selector, labels.Everything(), []time.Weekday{}, time.UTC, false, 0)

validateCandidates(t, chaoskube, []map[string]string{
{"namespace": "default", "name": "foo"},
Expand All @@ -129,7 +133,7 @@ func TestCandidatesExcludingAnnotationSelector(t *testing.T) {
t.Fatal(err)
}

chaoskube := setup(t, labels.Everything(), selector, labels.Everything(), []time.Weekday{}, false, 0)
chaoskube := setup(t, labels.Everything(), selector, labels.Everything(), []time.Weekday{}, time.UTC, false, 0)

validateCandidates(t, chaoskube, []map[string]string{
{"namespace": "testing", "name": "bar"},
Expand Down Expand Up @@ -159,15 +163,15 @@ func TestCandidatesNamespaces(t *testing.T) {
t.Fatal(err)
}

chaoskube := setup(t, labels.Everything(), labels.Everything(), namespaces, []time.Weekday{}, false, 0)
chaoskube := setup(t, labels.Everything(), labels.Everything(), namespaces, []time.Weekday{}, time.UTC, false, 0)

validateCandidates(t, chaoskube, test.pods)
}
}

// TestVictim tests that a pod is chosen from the candidates
func TestVictim(t *testing.T) {
chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{}, false, 2000)
chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{}, time.UTC, false, 2000)

validateVictim(t, chaoskube, map[string]string{
"namespace": "default", "name": "foo",
Expand All @@ -176,7 +180,7 @@ func TestVictim(t *testing.T) {

// TestAnotherVictim tests that the chosen victim is different for another seed
func TestAnotherVictim(t *testing.T) {
chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{}, false, 4000)
chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{}, time.UTC, false, 4000)

validateVictim(t, chaoskube, map[string]string{
"namespace": "testing", "name": "bar",
Expand All @@ -191,7 +195,7 @@ func TestAnotherVictimRespectsLabelSelector(t *testing.T) {
t.Fatal(err)
}

chaoskube := setup(t, selector, labels.Everything(), labels.Everything(), []time.Weekday{}, false, 4000)
chaoskube := setup(t, selector, labels.Everything(), labels.Everything(), []time.Weekday{}, time.UTC, false, 4000)

validateVictim(t, chaoskube, map[string]string{
"namespace": "default", "name": "foo",
Expand All @@ -200,7 +204,7 @@ func TestAnotherVictimRespectsLabelSelector(t *testing.T) {

// TestNoVictimReturnsError tests that on missing victim it returns a known error
func TestNoVictimReturnsError(t *testing.T) {
chaoskube := New(fake.NewSimpleClientset(), labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{}, logger, false, 2000)
chaoskube := New(fake.NewSimpleClientset(), labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{}, time.UTC, logger, false, 2000)

if _, err := chaoskube.Victim(); err != ErrPodNotFound {
t.Errorf("expected %#v, got %#v", ErrPodNotFound, err)
Expand All @@ -209,7 +213,7 @@ func TestNoVictimReturnsError(t *testing.T) {

// TestDeletePod tests deleting a particular pod
func TestDeletePod(t *testing.T) {
chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{}, false, 0)
chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{}, time.UTC, false, 0)

victim := util.NewPod("default", "foo")

Expand All @@ -226,7 +230,7 @@ func TestDeletePod(t *testing.T) {

// TestDeletePodDryRun tests that enabled dry run doesn't delete the pod
func TestDeletePodDryRun(t *testing.T) {
chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{}, true, 0)
chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{}, time.UTC, true, 0)

victim := util.NewPod("default", "foo")

Expand All @@ -242,7 +246,7 @@ func TestDeletePodDryRun(t *testing.T) {

// TestTerminateVictim tests that the correct victim pod is chosen and deleted
func TestTerminateVictim(t *testing.T) {
chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{}, false, 2000)
chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{}, time.UTC, false, 2000)

if err := chaoskube.TerminateVictim(); err != nil {
t.Fatal(err)
Expand All @@ -255,9 +259,9 @@ func TestTerminateVictim(t *testing.T) {

// TestTerminateVictimRespectsExcludedWeekday tests that no victim is terminated when the current weekday is excluded.
func TestTerminateVictimRespectsExcludedWeekdays(t *testing.T) {
chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{time.Friday}, false, 2000)
chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{time.Friday}, time.UTC, false, 2000)

// simulate that it's a Friday in our test.
// simulate that it's a Friday in our test (UTC).
chaoskube.Now = ThankGodItsFriday{}.Now

if err := chaoskube.TerminateVictim(); err != nil {
Expand All @@ -274,9 +278,9 @@ func TestTerminateVictimRespectsExcludedWeekdays(t *testing.T) {

// TestTerminateVictimOnNonExcludedWeekdays tests that victim is terminated when weekday filter doesn't match.
func TestTerminateVictimOnNonExcludedWeekdays(t *testing.T) {
chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{time.Friday}, false, 2000)
chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{time.Friday}, time.UTC, false, 2000)

// simulate that it's a Saturday in our test.
// simulate that it's a Saturday in our test (UTC).
chaoskube.Now = func() time.Time { return ThankGodItsFriday{}.Now().Add(24 * time.Hour) }

if err := chaoskube.TerminateVictim(); err != nil {
Expand All @@ -288,10 +292,31 @@ func TestTerminateVictimOnNonExcludedWeekdays(t *testing.T) {
})
}

// TestTerminateVictimRespectsTimezone tests that victim is terminated when weekday filter doesn't match due to different timezone.
func TestTerminateVictimRespectsTimezone(t *testing.T) {
timezone, err := time.LoadLocation("Australia/Brisbane")
if err != nil {
t.Fatal(err)
}

chaoskube := setup(t, labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{time.Friday}, timezone, false, 2000)

// simulate that it's a Friday in our test (UTC). However, in Australia it's already Saturday.
chaoskube.Now = ThankGodItsFriday{}.Now

if err := chaoskube.TerminateVictim(); err != nil {
t.Fatal(err)
}

validateCandidates(t, chaoskube, []map[string]string{
{"namespace": "testing", "name": "bar"},
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How do you know which pod will be left if the selection is "random"? Is it because, by testing ahead of time with the same Seed, you know it always terminates the same victim? If so, I get that it works, but doesn't "smell" right. (What if logic in the rand package changes?) I would consider simply asserting that the number of pods available was reduced by 1 would also be valid.

Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point. I was thinking, that by providing a seed I can control what's the "random" outcome. I used it to test that pods not matching the filters are really ignored (by testing that without the filter and the same seed it gets deleted).

However, for this test it doesn't matter which pod is killed. I'll change it as you suggested.

Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@twildeboer Where applicable, I removed testing for particular pods in favour of testing just the number of surviving pods. Also removed defining a seed where it doesn't influence the outcome.

})
}

// TestTerminateNoVictimLogsInfo tests that missing victim prints a log message
func TestTerminateNoVictimLogsInfo(t *testing.T) {
logOutput.Reset()
chaoskube := New(fake.NewSimpleClientset(), labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{}, logger, false, 0)
chaoskube := New(fake.NewSimpleClientset(), labels.Everything(), labels.Everything(), labels.Everything(), []time.Weekday{}, time.UTC, logger, false, 0)

if err := chaoskube.TerminateVictim(); err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -346,7 +371,7 @@ func validateLog(t *testing.T, msg string) {
}
}

func setup(t *testing.T, labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, dryRun bool, seed int64) *Chaoskube {
func setup(t *testing.T, labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, timezone *time.Location, dryRun bool, seed int64) *Chaoskube {
pods := []v1.Pod{
util.NewPod("default", "foo"),
util.NewPod("testing", "bar"),
Expand All @@ -362,14 +387,14 @@ func setup(t *testing.T, labelSelector labels.Selector, annotations labels.Selec

logOutput.Reset()

return New(client, labelSelector, annotations, namespaces, excludedWeekdays, logger, dryRun, seed)
return New(client, labelSelector, annotations, namespaces, excludedWeekdays, timezone, logger, dryRun, seed)
}

// ThankGodItsFriday is a helper struct that contains a Now() function that always returns a Friday.
type ThankGodItsFriday struct{}

// Now returns a particular Friday.
func (t ThankGodItsFriday) Now() time.Time {
blackFriday, _ := time.Parse(time.RFC1123, "Fri, 24 Sep 1869 15:04:05 EST")
blackFriday, _ := time.Parse(time.RFC1123, "Fri, 24 Sep 1869 15:04:05 UTC")
return blackFriday
}
10 changes: 10 additions & 0 deletions main.go
View file
Open in desktop
Expand Up @@ -21,6 +21,7 @@ var (
annString string
nsString string
excludedWeekdays string
timezone string
master string
kubeconfig string
interval time.Duration
Expand All @@ -35,6 +36,7 @@ func init() {
kingpin.Flag("annotations", "A set of annotations to restrict the list of affected pods. Defaults to everything.").StringVar(&annString)
kingpin.Flag("namespaces", "A set of namespaces to restrict the list of affected pods. Defaults to everything.").StringVar(&nsString)
kingpin.Flag("excluded-weekdays", "A list of weekdays when termination is suspended, e.g. sat,sun").StringVar(&excludedWeekdays)
kingpin.Flag("timezone", "The timezone to apply when detecting the current weekday, e.g. Local, UTC, Europe/Berlin. Defaults to Local.").Default(time.Local.String()).StringVar(&timezone)
kingpin.Flag("master", "The address of the Kubernetes cluster to target").StringVar(&master)
kingpin.Flag("kubeconfig", "Path to a kubeconfig file").StringVar(&kubeconfig)
kingpin.Flag("interval", "Interval between Pod terminations").Default("10m").DurationVar(&interval)
Expand Down Expand Up @@ -86,6 +88,13 @@ func main() {
log.Infof("Filtering pods by namespaces: %s", namespaces.String())
}

parsedTimezone, err := time.LoadLocation(timezone)
if err != nil {
log.Fatal(err)
}
timezoneName, _ := time.Now().In(parsedTimezone).Zone()
log.Infof("Using time zone: %s (%s)", parsedTimezone.String(), timezoneName)

parsedWeekdays := parseWeekdays(excludedWeekdays)
if len(parsedWeekdays) > 0 {
log.Infof("Excluding weekdays: %s", parsedWeekdays)
Expand All @@ -97,6 +106,7 @@ func main() {
annotations,
namespaces,
parsedWeekdays,
parsedTimezone,
log.StandardLogger(),
dryRun,
time.Now().UTC().UnixNano(),
Expand Down