-
Notifications
You must be signed in to change notification settings - Fork 5.1k
/
fixture.go
320 lines (256 loc) · 10.3 KB
/
fixture.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
package utils
import (
"context"
"encoding/json"
"fmt"
"os"
"regexp"
"strings"
"sync"
"testing"
"time"
log "github.com/sirupsen/logrus"
"github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1"
"k8s.io/apimachinery/pkg/api/equality"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
appclientset "github.com/argoproj/argo-cd/v2/pkg/client/clientset/versioned"
)
const (
// ArgoCDNamespace is the namespace into which Argo CD and ApplicationSet controller are deployed,
// and in which Application resources should be created.
ArgoCDNamespace = "argocd-e2e"
// ApplicationSetNamespace is the namespace into which temporary resources (such as Deployments/Pods/etc)
// can be deployed, such as using it as the target namespace in an Application resource.
// Note: this is NOT the namespace the ApplicationSet controller is deployed to; see ArgoCDNamespace.
ApplicationSetNamespace = "applicationset-e2e"
TmpDir = "/tmp/applicationset-e2e"
TestingLabel = "e2e.argoproj.io"
)
var (
id string
// call GetClientVars() to retrieve the Kubernetes client data for E2E test fixtures
clientInitialized sync.Once
internalClientVars *E2EFixtureK8sClient
)
// E2EFixtureK8sClient contains Kubernetes clients initialized from local k8s configuration
type E2EFixtureK8sClient struct {
KubeClientset kubernetes.Interface
DynamicClientset dynamic.Interface
AppClientset appclientset.Interface
AppSetClientset dynamic.ResourceInterface
}
// GetE2EFixtureK8sClient initializes the Kubernetes clients (if needed), and returns the most recently initalized value.
// Note: this requires a local Kubernetes configuration (for example, while running the E2E tests).
func GetE2EFixtureK8sClient() *E2EFixtureK8sClient {
// Initialize the Kubernetes clients only on first use
clientInitialized.Do(func() {
// set-up variables
config := getKubeConfig("", clientcmd.ConfigOverrides{})
internalClientVars = &E2EFixtureK8sClient{
AppClientset: appclientset.NewForConfigOrDie(config),
DynamicClientset: dynamic.NewForConfigOrDie(config),
KubeClientset: kubernetes.NewForConfigOrDie(config),
}
internalClientVars.AppSetClientset = internalClientVars.DynamicClientset.Resource(v1alpha1.GroupVersion.WithResource("applicationsets")).Namespace(ArgoCDNamespace)
})
return internalClientVars
}
// EnsureCleanSlate ensures that the Kubernetes resources on the cluster are are in a 'clean' state, before a test is run.
func EnsureCleanState(t *testing.T) {
start := time.Now()
fixtureClient := GetE2EFixtureK8sClient()
policy := v1.DeletePropagationForeground
// Delete the applicationset-e2e namespace, if it exists
err := fixtureClient.KubeClientset.CoreV1().Namespaces().Delete(context.Background(), ApplicationSetNamespace, v1.DeleteOptions{PropagationPolicy: &policy})
if err != nil && !strings.Contains(err.Error(), "not found") { // 'not found' error is expected
CheckError(err)
}
// delete resources
// kubectl delete applicationsets --all
CheckError(fixtureClient.AppSetClientset.DeleteCollection(context.Background(), v1.DeleteOptions{PropagationPolicy: &policy}, v1.ListOptions{}))
// kubectl delete apps --all
CheckError(fixtureClient.AppClientset.ArgoprojV1alpha1().Applications(ArgoCDNamespace).DeleteCollection(context.Background(), v1.DeleteOptions{PropagationPolicy: &policy}, v1.ListOptions{}))
// kubectl delete secrets -l e2e.argoproj.io=true
CheckError(fixtureClient.KubeClientset.CoreV1().Secrets(ArgoCDNamespace).DeleteCollection(context.Background(),
v1.DeleteOptions{PropagationPolicy: &policy}, v1.ListOptions{LabelSelector: TestingLabel + "=true"}))
// First we wait up to 30 seconds for all the ApplicationSets to delete, but we don't fail if they don't.
// Why? We want to give Argo CD time to delete the Application's child resources, before we remove the finalizers below.
_ = waitForSuccess(func() error {
list, err := fixtureClient.AppSetClientset.List(context.Background(), v1.ListOptions{})
if err != nil {
return err
}
if list != nil && len(list.Items) > 0 {
// Fail
return fmt.Errorf("Waiting for list of ApplicationSets to be size zero: %d", len(list.Items))
}
return nil // Pass
}, time.Now().Add(30*time.Second))
// Remove finalizers from Argo CD Application resources in the namespace
err = waitForSuccess(func() error {
appList, err := fixtureClient.AppClientset.ArgoprojV1alpha1().Applications(ArgoCDNamespace).List(context.Background(), v1.ListOptions{})
if err != nil {
return err
}
for _, app := range appList.Items {
t.Log("Removing finalizer for: ", app.Name)
app.Finalizers = []string{}
_, err := fixtureClient.AppClientset.ArgoprojV1alpha1().Applications(ArgoCDNamespace).Update(context.TODO(), &app, v1.UpdateOptions{})
if err != nil {
return err
}
}
return nil
}, time.Now().Add(120*time.Second))
CheckError(err)
CheckError(waitForExpectedClusterState())
// remove tmp dir
CheckError(os.RemoveAll(TmpDir))
// create tmp dir
FailOnErr(Run("", "mkdir", "-p", TmpDir))
log.WithFields(log.Fields{"duration": time.Since(start), "name": t.Name(), "id": id, "username": "admin", "password": "password"}).Info("clean state")
}
func waitForExpectedClusterState() error {
fixtureClient := GetE2EFixtureK8sClient()
// Wait up to 60 seconds for all the ApplicationSets to delete
if err := waitForSuccess(func() error {
list, err := fixtureClient.AppSetClientset.List(context.Background(), v1.ListOptions{})
if err != nil {
return err
}
if list != nil && len(list.Items) > 0 {
// Fail
return fmt.Errorf("Waiting for list of ApplicationSets to be size zero: %d", len(list.Items))
}
return nil // Pass
}, time.Now().Add(60*time.Second)); err != nil {
return err
}
// Wait up to 60 seconds for all the Applications to delete
if err := waitForSuccess(func() error {
appList, err := fixtureClient.AppClientset.ArgoprojV1alpha1().Applications(ArgoCDNamespace).List(context.Background(), v1.ListOptions{})
if err != nil {
return err
}
if appList != nil && len(appList.Items) > 0 {
// Fail
return fmt.Errorf("Waiting for list of Applications to be size zero: %d", len(appList.Items))
}
return nil // Pass
}, time.Now().Add(60*time.Second)); err != nil {
return err
}
// Wait up to 120 seconds for namespace to not exist
if err := waitForSuccess(func() error {
_, err := fixtureClient.KubeClientset.CoreV1().Namespaces().Get(context.Background(), ApplicationSetNamespace, v1.GetOptions{})
msg := ""
if err == nil {
msg = fmt.Sprintf("namespace '%s' still exists, after delete", ApplicationSetNamespace)
}
if msg == "" && err != nil && strings.Contains(err.Error(), "not found") {
// Success is an error containing 'applicationset-e2e' not found.
return nil
}
if msg == "" {
msg = err.Error()
}
return fmt.Errorf(msg)
}, time.Now().Add(120*time.Second)); err != nil {
return err
}
return nil
}
// waitForSuccess waits for the condition to return a non-error value.
// Returns if condition returns nil, or the expireTime has elapsed (in which
// case the last error will be returned)
func waitForSuccess(condition func() error, expireTime time.Time) error {
var mostRecentError error
for {
if time.Now().After(expireTime) {
break
}
conditionErr := condition()
if conditionErr != nil {
// Fail!
mostRecentError = conditionErr
} else {
// Pass!
mostRecentError = nil
break
}
// Wait 0.5 seconds on fail
time.Sleep(500 * time.Millisecond)
}
return mostRecentError
}
// getKubeConfig creates new kubernetes client config using specified config path and config overrides variables
func getKubeConfig(configPath string, overrides clientcmd.ConfigOverrides) *rest.Config {
loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
loadingRules.ExplicitPath = configPath
clientConfig := clientcmd.NewInteractiveDeferredLoadingClientConfig(loadingRules, &overrides, os.Stdin)
restConfig, err := clientConfig.ClientConfig()
CheckError(err)
return restConfig
}
// creates e2e tests fixture: ensures that Application CRD is installed, creates temporal namespace, starts repo and api server,
// configure currently available cluster.
func init() {
// ensure we log all shell execs
log.SetLevel(log.DebugLevel)
}
// PrettyPrintJson is a utility function for debugging purposes
func PrettyPrintJson(obj interface{}) string {
bytes, err := json.MarshalIndent(obj, "", " ")
if err != nil {
return err.Error()
}
return string(bytes)
}
// returns dns friends string which is no longer than 63 characters and has specified postfix at the end
func DnsFriendly(str string, postfix string) string {
matchFirstCap := regexp.MustCompile("(.)([A-Z][a-z]+)")
matchAllCap := regexp.MustCompile("([a-z0-9])([A-Z])")
str = matchFirstCap.ReplaceAllString(str, "${1}-${2}")
str = matchAllCap.ReplaceAllString(str, "${1}-${2}")
str = strings.ToLower(str)
if diff := len(str) + len(postfix) - 63; diff > 0 {
str = str[:len(str)-diff]
}
return str + postfix
}
func MustToUnstructured(obj interface{}) *unstructured.Unstructured {
uObj, err := ToUnstructured(obj)
if err != nil {
panic(err)
}
return uObj
}
// ToUnstructured converts a concrete K8s API type to an unstructured object
func ToUnstructured(obj interface{}) (*unstructured.Unstructured, error) {
uObj, err := runtime.NewTestUnstructuredConverter(equality.Semantic).ToUnstructured(obj)
if err != nil {
return nil, err
}
return &unstructured.Unstructured{Object: uObj}, nil
}
// IsGitHubSkippedTest returns true if the test should be skipped because it requires a GitHub API Token
// and one has not been provided.
// Unfortunately, GitHub Actions cannot use repository secrets, so we need to skip these tests for PRs.
//
// Tests that call this function require a GITHUB_TOKEN to be present, otherwise they will fail, due to
// GitHub's rate limiting on anonymous API requests.
//
// Note: This only applies to tests that use the GitHub API (different from GitHub's Git service)
func IsGitHubAPISkippedTest(t *testing.T) bool {
if strings.TrimSpace(os.Getenv("GITHUB_TOKEN")) == "" {
t.Skip("Skipping this test, as the GITHUB_TOKEN is not set. Please ensure this test passes locally, with your own GITHUB_TOKEN.")
return true
}
return false
}