Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[examples] k8petstore: connection timeouts + allow blips. #21621

Merged
merged 1 commit into from
Feb 26, 2016
Merged

[examples] k8petstore: connection timeouts + allow blips. #21621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions examples/k8petstore/k8petstore-nodeport.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ function pollfor {
do
### Just testing that the front end comes up. Not sure how to test total entries etc... (yet)
echo "Trying curl frontend:3000 via $TEST_IP:$NODE_PORT, attempt ${i}. Expect a few failures while pulling images... "
curl --max-time 1 "$TEST_IP:$NODE_PORT" > result
curl --max-time 1 --connect-timeout 3 "$TEST_IP:$NODE_PORT" > result
cat result
cat result | grep -q "k8-bps"
if [ $? -eq 0 ]; then
Expand All @@ -296,7 +296,7 @@ function tests {
for i in `seq 1 $_SECONDS`;
do
echo "curl : $TEST_IP:$NODE_PORT , $i of $_SECONDS"
curr_cnt="`curl --max-time 1 "$TEST_IP:$NODE_PORT/llen"`"
curr_cnt="`curl --max-time 1 --connect-timeout 3 "$TEST_IP:$NODE_PORT/llen"`"
### Write CSV File of # of trials / total transcations.
echo "$i $curr_cnt" >> result
echo "total transactions so far : $curr_cnt"
Expand Down
66 changes: 34 additions & 32 deletions test/e2e/example_k8petstore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,16 @@ import (
)

const (
k8bpsContainerVersion = "r.2.8.19" // Container version, see the examples/k8petstore dockerfiles for details.
k8bpsThroughputDummy = "0" // Polling time = 0, since we poll in ginkgo rather than using the shell script tests.
k8bpsRedisSlaves = "1" // Number of redis slaves.
k8bpsDontRunTest = "0" // Don't bother embedded test.
k8bpsStartupTimeout = 30 * time.Second // Amount of elapsed time before petstore transactions are being stored.
k8bpsContainerVersion = "r.2.8.19" // Container version, see the examples/k8petstore dockerfiles for details.
k8bpsThroughputDummy = "0" // Polling time = 0, since we poll in ginkgo rather than using the shell script tests.
k8bpsRedisSlaves = "1" // Number of redis slaves.
k8bpsDontRunTest = "0" // Don't bother embedded test.
k8bpsStartupTimeout = 30 * time.Second // Amount of elapsed time before petstore transactions are being stored.
k8bpsMinTransactionsOnStartup = 3 // Amount of transactions we expect we should have before data generator starts.

// Constants for the first test. We can make this a hashmap once we add scale tests to it.
k8bpsSmokeTestTransactions = 50
k8bpsSmokeTestTimeout = 60 * time.Second
k8bpsSmokeTestFinalTransactions = 50
k8bpsSmokeTestTimeout = 60 * time.Second
)

// readTransactions reads # of transactions from the k8petstore web server endpoint.
Expand All @@ -64,8 +65,8 @@ func readTransactions(c *client.Client, ns string) (error, int) {
}

// runK8petstore runs the k8petstore application, bound to external nodeport, and
// polls until minExpected transactions are acquired, in a maximum of maxSeconds.
func runK8petstore(restServers int, loadGenerators int, c *client.Client, ns string, minExpected int, maxTime time.Duration) {
// polls until finalTransactionsExpected transactions are acquired, in a maximum of maxSeconds.
func runK8petstore(restServers int, loadGenerators int, c *client.Client, ns string, finalTransactionsExpected int, maxTime time.Duration) {

var err error = nil
k8bpsScriptLocation := filepath.Join(testContext.RepoRoot, "examples/k8petstore/k8petstore-nodeport.sh")
Expand Down Expand Up @@ -118,37 +119,38 @@ T:
for {
select {
case <-transactionsCompleteTimeout:
Logf("Timeout %v reached, transactions not complete. Breaking!", tick)
Logf("Completion timeout %v reached, %v transactions not complete. Breaking!", time.Duration(maxTime), finalTransactionsExpected)
break T
case <-startupTimeout:
err, totalTransactions = readTransactions(c, ns)
Logf("Timeout %v reached. Checking if transactions have occured.", startupTimeout)
// If we don't have 3 transactions, fail the test.
if err != nil {
Logf("Failed : Error %v", err)
break T
}
if totalTransactions < 3 {
break T
}
ready = true
case <-tick:
// Pass if we've collected enough transactions
// Don't fail if there's an error. We expect a few failures might happen in the cloud.
err, totalTransactions = readTransactions(c, ns)
if ready {
Expect(err).NotTo(HaveOccurred())
if err == nil {
Logf("PetStore : Time: %v, %v = total petstore transactions stored into redis.", time.Now(), totalTransactions)
if totalTransactions >= k8bpsMinTransactionsOnStartup {
ready = true
}
if totalTransactions >= finalTransactionsExpected {
break T
}
} else {
if ready {
Logf("Blip: during polling: %v", err)
} else {
Logf("Not ready yet: %v", err)
}
}
if totalTransactions > minExpected {
case <-startupTimeout:
if !ready {
Logf("Startup Timeout %v reached: Its been too long and we still haven't started accumulating %v transactions!", startupTimeout, k8bpsMinTransactionsOnStartup)
break T
}
Logf("Time: %v, %v = total petstore transactions stored into redis.", time.Now(), totalTransactions)
}
}

// We should have exceeded the minExpected num of transactions.
// We should have exceeded the finalTransactionsExpected num of transactions.
// If this fails, but there are transactions being created, we may need to recalibrate
// the minExpected value - or else - your cluster is broken/slow !
Ω(totalTransactions).Should(BeNumerically(">", minExpected))
// the finalTransactionsExpected value - or else - your cluster is broken/slow !
Ω(totalTransactions).Should(BeNumerically(">", finalTransactionsExpected))
}

var _ = Describe("Pet Store [Feature:Example]", func() {
Expand All @@ -157,14 +159,14 @@ var _ = Describe("Pet Store [Feature:Example]", func() {
var nodeCount int
f := NewFramework("petstore")

It(fmt.Sprintf("should scale to persist a nominal number ( %v ) of transactions in %v seconds", k8bpsSmokeTestTransactions, k8bpsSmokeTestTimeout), func() {
It(fmt.Sprintf("should scale to persist a nominal number ( %v ) of transactions in %v seconds", k8bpsSmokeTestFinalTransactions, k8bpsSmokeTestTimeout), func() {
nodes := ListSchedulableNodesOrDie(f.Client)
nodeCount = len(nodes.Items)

loadGenerators := nodeCount
restServers := nodeCount
fmt.Printf("load generators / rest servers [ %v / %v ] ", loadGenerators, restServers)
runK8petstore(restServers, loadGenerators, f.Client, f.Namespace.Name, k8bpsSmokeTestTransactions, k8bpsSmokeTestTimeout)
runK8petstore(restServers, loadGenerators, f.Client, f.Namespace.Name, k8bpsSmokeTestFinalTransactions, k8bpsSmokeTestTimeout)
})

})