forked from kubernetes/kubernetes
-
Notifications
You must be signed in to change notification settings - Fork 1
/
sd_soak.go
100 lines (84 loc) · 3.38 KB
/
sd_soak.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"fmt"
"math"
"time"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
)
const (
// maxAllowedLostFraction is the fraction of lost logs considered acceptable.
maxAllowedLostFraction = 0.01
// maxAllowedRestartsPerHour is the number of fluentd container restarts
// considered acceptable. Once per hour is fine for now, as long as it
// doesn't loose too much logs.
maxAllowedRestartsPerHour = 1.0
// lastPodIngestionSlack is the amount of time to wait for the last pod's
// logs to be ingested by the logging agent.
lastPodIngestionSlack = 5 * time.Minute
)
var _ = framework.KubeDescribe("Cluster level logging implemented by Stackdriver [Feature:StackdriverLogging] [Soak]", func() {
f := framework.NewDefaultFramework("sd-logging-load")
It("should ingest logs from applications running for a prolonged amount of time", func() {
sdLogsProvider, err := newSdLogsProvider(f)
framework.ExpectNoError(err, "Failed to create Stackdriver logs provider")
err = sdLogsProvider.Init()
defer sdLogsProvider.Cleanup()
framework.ExpectNoError(err, "Failed to init Stackdriver logs provider")
nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items
maxPodCount := 10
jobDuration := 1 * time.Hour
linesPerPodPerSecond := 100
testDuration := 21 * time.Hour
ingestionTimeout := testDuration + 30*time.Minute
allowedRestarts := int(math.Ceil(float64(testDuration) /
float64(time.Hour) * maxAllowedRestartsPerHour))
podRunDelay := time.Duration(int64(jobDuration) / int64(maxPodCount))
podRunCount := maxPodCount*(int(testDuration/jobDuration)-1) + 1
linesPerPod := linesPerPodPerSecond * int(jobDuration.Seconds())
pods := []*loggingPod{}
for runIdx := 0; runIdx < podRunCount; runIdx++ {
for nodeIdx, node := range nodes {
podName := fmt.Sprintf("job-logs-generator-%d-%d-%d-%d", maxPodCount, linesPerPod, runIdx, nodeIdx)
pods = append(pods, newLoggingPod(podName, node.Name, linesPerPod, jobDuration))
}
}
By("Running short-living pods")
go func() {
for _, pod := range pods {
pod.Start(f)
time.Sleep(podRunDelay)
defer f.PodClient().Delete(pod.Name, &meta_v1.DeleteOptions{})
}
// Waiting until the last pod has completed
time.Sleep(jobDuration - podRunDelay + lastPodIngestionSlack)
}()
By("Waiting for all log lines to be ingested")
config := &loggingTestConfig{
LogsProvider: sdLogsProvider,
Pods: pods,
IngestionTimeout: ingestionTimeout,
MaxAllowedLostFraction: maxAllowedLostFraction,
MaxAllowedFluentdRestarts: allowedRestarts,
}
err = waitForFullLogsIngestion(f, config)
if err != nil {
framework.Failf("Failed to ingest logs: %v", err)
} else {
framework.Logf("Successfully ingested all logs")
}
})
})