Skip to content
Permalink
Browse files

Implemented an e2e test reproducing the traffic shifting bug

This commit implements an e2e test that reproduces the problem with
traffic label shifting observed in shipper 0.7. The scenario it replays:
	1. Create an application
	2. Create first release with 0 replicas and set it to full on
	3. Create second release with 2 replicas and set it to full-on
	4. Inspect the active pods: only 1 pod will contain
	traffic:enabled label, but both pods are expected to be active

Signed-off-by: Oleg Sidorov <oleg.sidorov@booking.com>
  • Loading branch information
icanhazbroccoli authored and parhamdoustdar committed Nov 19, 2019
1 parent bc31d42 commit 2b8f2419831cc7d430102a5bd30a8ba54cb5952e
Showing with 74 additions and 0 deletions.
  1. +56 −0 test/e2e/e2e_test.go
  2. +18 −0 test/e2e/e2e_util.go
@@ -1435,6 +1435,62 @@ func TestDeletedDeploymentsAreReinstalled(t *testing.T) {
f.checkPods(relName, targetReplicas)
}

func TestConsistentTrafficBalanceOnStraightFullOn(t *testing.T) {
if !*runEndToEnd {
t.Skip("skipping end-to-end tests: --e2e is false")
}
t.Parallel()

ns, err := setupNamespace(t.Name())
f := newFixture(ns.GetName(), t)
if err != nil {
t.Fatalf("could not create namespace %s: %q", ns.GetName(), err)
}
defer func() {
if *inspectFailed && t.Failed() {
return
}
teardownNamespace(ns.GetName())
}()

app := newApplication(ns.GetName(), appName, &allIn)
app.Spec.Template.Chart.Name = "test-nginx"
app.Spec.Template.Chart.Version = "0.0.1"
app.Spec.Template.Values = &shipper.ChartValues{"replicaCount": 0}

_, err = shipperClient.ShipperV1alpha1().Applications(ns.GetName()).Create(app)
if err != nil {
t.Fatalf("could not create application %q: %q", appName, err)
}

t.Logf("waiting for a new release for new application %q", appName)
incumbent := f.waitForRelease(appName, 0)
incumbentName := incumbent.GetName()
t.Logf("waiting for incumbent release %q to complete", incumbentName)
f.waitForComplete(incumbent.GetName())
t.Logf("checking that incumbent release %q has %d pods (strategy step 0 -- finished)", incumbentName, 0)
f.checkPods(incumbentName, 0)

app, err = shipperClient.ShipperV1alpha1().Applications(ns.GetName()).Get(app.GetName(), metav1.GetOptions{})
if err != nil {
t.Fatalf("could not fetch application %q: %q", appName, err)
}

app.Spec.Template.Strategy = &vanguard
app.Spec.Template.Values = &shipper.ChartValues{"replicaCount": 2}
if _, err := shipperClient.ShipperV1alpha1().Applications(ns.GetName()).Update(app); err != nil {
t.Fatalf("could not update app %q: %q", app.GetName(), err)
}

contender := f.waitForRelease(appName, 1)
contenderName := contender.GetName()
f.targetStep(2, contenderName)
f.waitForComplete(contender.GetName())
f.checkPods(contenderName, 2)

f.checkEndpointActivePods("test-nginx-prod", 2)
}

// TODO(btyler): cover a variety of broken chart cases as soon as we report
// those outcomes somewhere other than stderr.

@@ -71,6 +71,24 @@ func (f *fixture) checkPods(relName string, expectedCount int) {
}
}

func (f *fixture) checkEndpointActivePods(epName string, podCnt int) {
ep, err := kubeClient.CoreV1().Endpoints(f.namespace).Get(epName, metav1.GetOptions{})
if err != nil {
f.t.Fatalf("could not fetch endpoint %q: %q", epName, err)
}

ips := make([]string, 0, podCnt)
for _, subset := range ep.Subsets {
for _, addr := range subset.Addresses {
ips = append(ips, addr.IP)
}
}

if len(ips) != podCnt {
f.t.Fatalf("wrong IP address count in the endpoint %q: want: %d, got: %d", epName, len(ips), podCnt)
}
}

func (f *fixture) waitForRelease(appName string, historyIndex int) *shipper.Release {
var newRelease *shipper.Release
start := time.Now()

0 comments on commit 2b8f241

Please sign in to comment.
You can’t perform that action at this time.