-
Notifications
You must be signed in to change notification settings - Fork 96
/
tests.go
195 lines (184 loc) · 8.6 KB
/
tests.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
package common
import (
"github.com/integr8ly/integreatly-operator/apis/v1alpha1"
)
// All tests are be linked[1] to the integreatly-test-cases[2] repo by using the same ID
// 1. https://gitlab.cee.redhat.com/integreatly-qe/integreatly-test-cases#how-to-automate-a-test-case-and-link-it-back
// 2. https://gitlab.cee.redhat.com/integreatly-qe/integreatly-test-cases
var (
ALL_TESTS = []TestSuite{
{
[]TestCase{
// Add all tests that can be executed prior to a completed installation here
{"Verify RHMI CRD Exists", TestIntegreatlyCRDExists},
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeManagedApi, v1alpha1.InstallationTypeMultitenantManagedApi},
},
}
HAPPY_PATH_TESTS = []TestSuite{
//Add all happy path tests to be executed after RHMI installation is completed here
{
[]TestCase{
{"A32 - Validate SSO config", TestSSOconfig},
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeManagedApi},
},
{
[]TestCase{
{"E10 - Verify Customer Grafana Route is accessible", TestCustomerGrafanaExternalRouteAccessible},
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeManagedApi, v1alpha1.InstallationTypeMultitenantManagedApi},
},
{
[]TestCase{
// Keep test as first on the list, as it ensures that all products are reported as complete
{"A01 - Verify that all stages in the integreatly-operator CR report completed", TestIntegreatlyStagesStatus},
{"A03 - Verify all namespaces have been created with the correct name", TestNamespaceCreated},
{"A05 - Verify product operator version", TestProductOperatorVersions},
{"A07 - Verify product versions", TestProductVersions},
{"A08 - Verify all products routes are created", TestIntegreatlyRoutesExist},
{"A09 - Verify Subscription Install Plan Strategy", TestSubscriptionInstallPlanType},
{"A10 - Verify CRO Postgres CRs Successful", TestCROPostgresSuccessfulState},
{"A11 - Verify CRO Redis CRs Successful", TestCRORedisSuccessfulState},
{"A13 - Verify Deployment resources have the expected replicas", TestDeploymentExpectedReplicas},
{"A14 - Verify Deployment Config resources have the expected replicas", TestDeploymentConfigExpectedReplicas},
{"A15 - Verify Stateful Set resources have the expected replicas", TestStatefulSetsExpectedReplicas},
{"A26 - Verify Sendgrid Credentials Are Configured Properly", TestSendgridCredentialsAreValid},
{"C01 - Verify Alerts are not pending or firing apart from DeadMansSwitch", TestIntegreatlyAlertsPendingOrFiring},
{"F02 - Verify PodDisruptionBudgets exist", TestIntegreatlyPodDisruptionBudgetsExist},
{"A27 + A28 - Verify pod priority class is created and set", TestPriorityClass},
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeManagedApi, v1alpha1.InstallationTypeMultitenantManagedApi},
},
{
[]TestCase{
{"M01 - Verify multitenancy works as expected", TestMultitenancy},
//{"MT02 - Performance test simulate parallel Tenants creation", TestMultitenancyPerformance},
// MT02 test will be used for manual Performance verification Only. Not include in Test suite!
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeMultitenantManagedApi},
},
{
[]TestCase{
{"Validate resource requirements are set", ValidateResourceRequirements},
{"Verify addon instance status conditions", TestStatusConditions},
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeManagedApi},
},
}
OBSERVABILITY_TESTS = []TestSuite{
//Observability related tests are kept separate from HAPPY_PATH because we skip these tests when IN_PROW is true.
//This is because the prow checks run on OCP clusters and OCP clusters don't install OBO by default. See MGDAPI-5783.
{
[]TestCase{
{"Test if cluster package is available", TestClusterPackageAvailable},
{"Test RHMI installation CR metric", TestRHMICRMetrics},
{"A06 - Verify PVC", TestPVClaims},
{"C04 - Verify Alerts exist", TestIntegreatlyAlertsExist},
{"C10B - Verify Prometheus blackbox targets", TestAdditionalBlackboxTargets},
{"C08B - Verify alert links to SOPs", TestSOPUrls},
{"Verify Alerts are not firing during or after installation apart from DeadMansSwitch", TestIntegreatlyAlertsFiring},
{"Verify prometheus metrics scrapped", TestMetricsScrappedByPrometheus},
{"E09 - Verify customer dashboards exist", TestIntegreatlyCustomerDashboardsExist},
{"Verify ClusterObjectTemplates ready state", TestClusterObjectTemplateState},
{"Verify package operator resource stability", TestPackageOperatorResourceStability},
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeManagedApi, v1alpha1.InstallationTypeMultitenantManagedApi},
},
{
[]TestCase{
{"M02B - Verify RHOAM version metric is exposed in Prometheus", TestRhoamVersionMetricExposed},
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeManagedApi},
},
}
//Threescale cluster scoped test suite to be used when Threescale becomes cluster scoped.
THREESCALE_CLUSTER_SCOPED_TESTS = []TestSuite{
{
[]TestCase{
{"H29 - Verify that backend can be created via backend CR", Test3scaleBackendViaCR},
{"H30 - Verify that product can be created via product CR", Test3scaleProductViaCR},
{"H31 - Verify that tenant can be created via tenant CR", Test3scaleTenantViaCr},
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeManagedApi},
},
}
IDP_BASED_TESTS = []TestSuite{
{
[]TestCase{
{"B06 - Verify users with no email get default email", TestDefaultUserEmail},
{"Verify Network Policy allows cross NS access to SVC", TestNetworkPolicyAccessNSToSVC},
{"C19 - Validate creation of invalid username triggers alert", TestInvalidUserNameAlert},
{"H34 - Verify 3scale custom SMTP full config", Test3ScaleCustomSMTPFullConfig},
{"H35 - Verify 3scale custom SMTP partial config", Test3ScaleCustomSMTPPartialConfig},
{"H24 - Verify selfmanaged Apicast", TestSelfmanagedApicast},
{"A33 - Verify console links", TestConsoleLinks},
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeManagedApi, v1alpha1.InstallationTypeMultitenantManagedApi},
},
{
[]TestCase{
{"B01B - Verify users can login to products", TestProductLogins},
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeManagedApi},
},
{
[]TestCase{
{"A16 - Custom first broker login flow", TestAuthDelayFirstBrokerLogin},
{"B03 - Verify RHMI Developer User Permissions are Correct", TestRHMIDeveloperUserPermissions},
{"B04 - Verify Dedicated Admin User Permissions are Correct", TestDedicatedAdminUserPermissions},
{"B09 - Verify dedicated admin users are synced with User SSO", TestDedicatedAdminUsersSyncedSSO},
{"H03 - Verify 3scale CRUDL permissions", Test3ScaleCrudlPermissions},
{"H07 - ThreeScale User Promotion", Test3ScaleUserPromotion},
// Keep H11 as last 3scale IDP Test as test causes 3scale deployments to be rescaled at the end of test
// Can potentially cause subsequent tests be flaky due to waiting for 3scale deployments to complete
{"H11 - Verify 3scale SMTP config", Test3ScaleSMTPConfig},
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeManagedApi},
},
}
SCALABILITY_TESTS = []TestSuite{
{
[]TestCase{
{"F05 - Verify Replicas Scale correctly in Threescale", TestReplicasInThreescale},
{"F08 - Verify Replicas Scale correctly in RHSSO", TestReplicasInRHSSO},
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeManagedApi, v1alpha1.InstallationTypeMultitenantManagedApi},
},
{
[]TestCase{
{"F08 - Verify Replicas Scale correctly in User SSO", TestReplicasInUserSSO},
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeManagedApi},
},
{
[]TestCase{
{"A34 - Verify Quota values", TestQuotaValues},
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeManagedApi, v1alpha1.InstallationTypeMultitenantManagedApi},
},
}
FAILURE_TESTS = []TestCase{
{"C03 - Verify that alerting mechanism works", TestIntegreatlyAlertsMechanism},
}
DESTRUCTIVE_TESTS = []TestCase{
// Add all destructive tests here that should not be executed as part of the happy path tests
{"J03 - Verify namespaces restored when deleted", TestNamespaceRestoration},
}
GCP_TESTS = []TestSuite{
{
[]TestCase{
{"N01 - Verify that Noobaa can be created via product CR", TestNoobaaViaCR},
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeManagedApi},
},
}
AWS_SPECIFIC_TESTS = []TestSuite{
{
[]TestCase{
{"A12 - Verify CRO BlobStorage CRs Successful", TestCROBlobStorageSuccessfulState},
},
[]v1alpha1.InstallationType{v1alpha1.InstallationTypeManagedApi, v1alpha1.InstallationTypeMultitenantManagedApi},
},
}
)