/
istio.go
345 lines (288 loc) · 13.3 KB
/
istio.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
// Copyright 2018-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package k8sTest
import (
"context"
"fmt"
"runtime"
"time"
. "github.com/cilium/cilium/test/ginkgo-ext"
"github.com/cilium/cilium/test/helpers"
. "github.com/onsi/gomega"
)
// This tests the Istio integration, following the configuration
// instructions specified in the Istio Getting Started Guide in
// Documentation/gettingstarted/istio.rst.
var _ = Describe("K8sIstioTest", func() {
var (
// istioSystemNamespace is the default namespace into which Istio is
// installed.
istioSystemNamespace = "istio-system"
istioVersion = "1.5.7"
// Modifiers for pre-release testing, normally empty
prerelease = "" // "-beta.1"
istioctlParams = ""
// Keeping these here in comments serve multiple purposes:
// - remind how to test with prerelease images in future
// - cause CI infra to prepull these images so that they do not
// need to be pulled on demand during the test
// " --set values.pilot.image=docker.io/cilium/istio_pilot:1.5.7" +
// " --set values.global.proxy.image=docker.io/cilium/istio_proxy:1.5.7" +
// " --set values.global.proxy_init.image=docker.io/cilium/istio_proxy:1.5.7"
ciliumOptions = map[string]string{
// "global.proxy.sidecarImageRegex": "jrajahalme/istio_proxy",
}
// Map of tested runtimes for cilium-istioctl
ciliumIstioctlOSes = map[string]string{
"darwin": "osx",
"linux": "linux",
}
// istioServiceNames is the set of Istio services needed for the tests
istioServiceNames = []string{
"istio-ingressgateway",
"istio-pilot",
}
// wgetCommand is the command used in this test because the Istio apps
// do not provide curl.
wgetCommand = fmt.Sprintf("wget --tries=2 --connect-timeout %d", helpers.CurlConnectTimeout)
kubectl *helpers.Kubectl
uptimeCancel context.CancelFunc
teardownTimeout = 10 * time.Minute
ciliumFilename string
)
BeforeAll(func() {
k8sVersion := helpers.GetCurrentK8SEnv()
switch k8sVersion {
case "1.7", "1.8", "1.9", "1.10", "1.11", "1.12", "1.13":
Skip(fmt.Sprintf("Istio %s doesn't support K8S %s", istioVersion, k8sVersion))
}
kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)
By("Downloading cilium-istioctl")
os := "linux"
if kubectl.IsLocal() {
// Use Ginkgo runtime OS instead when commands are executed in the local Ginkgo host
os = ciliumIstioctlOSes[runtime.GOOS]
}
ciliumIstioctlURL := "https://github.com/cilium/istio/releases/download/" + istioVersion + prerelease + "/cilium-istioctl-" + istioVersion + "-" + os + ".tar.gz"
res := kubectl.Exec(fmt.Sprintf("curl --retry 5 -L %s | tar xz", ciliumIstioctlURL))
res.ExpectSuccess("unable to download %s", ciliumIstioctlURL)
res = kubectl.ExecShort("./cilium-istioctl version")
res.ExpectSuccess("unable to execute cilium-istioctl")
ciliumFilename = helpers.TimestampFilename("cilium.yaml")
DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, ciliumOptions)
By("Labeling default namespace for sidecar injection")
res = kubectl.NamespaceLabel(helpers.DefaultNamespace, "istio-injection=enabled")
res.ExpectSuccess("unable to label namespace %q", helpers.DefaultNamespace)
By("Deploying Istio")
res = kubectl.Exec("./cilium-istioctl manifest apply -y" + istioctlParams)
res.ExpectSuccess("unable to deploy Istio")
})
AfterAll(func() {
By("Deleting default namespace sidecar injection label")
_ = kubectl.NamespaceLabel(helpers.DefaultNamespace, "istio-injection-")
By("Deleting the Istio resources")
_ = kubectl.Exec(fmt.Sprintf("./cilium-istioctl manifest generate | %s delete -f -", helpers.KubectlCmd))
By("Waiting all terminating PODs to disappear")
err := kubectl.WaitCleanAllTerminatingPods(teardownTimeout)
ExpectWithOffset(1, err).To(BeNil(), "terminating Istio PODs are not deleted after timeout")
By("Deleting the istio-system namespace")
_ = kubectl.NamespaceDelete(istioSystemNamespace)
kubectl.CloseSSHClient()
})
JustBeforeEach(func() {
var err error
uptimeCancel, err = kubectl.BackgroundReport("uptime")
Expect(err).To(BeNil(), "Cannot start background report process")
})
JustAfterEach(func() {
uptimeCancel()
kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
})
AfterFailed(func() {
kubectl.CiliumReport(helpers.CiliumNamespace,
"cilium endpoint list",
"cilium bpf proxy list")
})
// This is defined as a separate function to be called from the test below
// so that we properly capture test artifacts if any of the assertions fail
// (see https://github.com/cilium/cilium/pull/8508).
waitIstioReady := func() {
// Ignore one-time jobs and Prometheus. All other pods in the
// namespaces have an "istio" label.
By("Waiting for Istio pods to be ready")
// First wait for at least one POD to get into running state so that WaitforPods
// below does not succeed if there are no PODs with the "istio" label.
err := kubectl.WaitforNPodsRunning(istioSystemNamespace, "-l istio", 1, helpers.HelperTimeout)
ExpectWithOffset(1, err).To(BeNil(),
"No Istio POD is Running after timeout in namespace %q", istioSystemNamespace)
// Then wait for all the Istio PODs to get Ready
// Note that this succeeds if there are no PODs matching the filter (-l istio -n istio-system).
err = kubectl.WaitforPods(istioSystemNamespace, "-l istio", helpers.HelperTimeout)
ExpectWithOffset(1, err).To(BeNil(),
"Istio pods are not ready after timeout in namespace %q", istioSystemNamespace)
for _, name := range istioServiceNames {
By("Waiting for Istio service %q to be ready", name)
err = kubectl.WaitForServiceEndpoints(
istioSystemNamespace, "", name, helpers.HelperTimeout)
ExpectWithOffset(1, err).Should(BeNil(), "Service %q is not ready after timeout", name)
}
for _, name := range istioServiceNames {
By("Waiting for DNS to resolve Istio service %q", name)
err = kubectl.WaitForKubeDNSEntry(name, istioSystemNamespace)
ExpectWithOffset(1, err).To(BeNil(), "DNS entry is not ready after timeout")
}
}
// This is a subset of Services's "Bookinfo Demo" test suite, with the pods
// injected with Istio sidecar proxies and Istio mTLS enabled.
SkipContextIf(func() bool { return ciliumIstioctlOSes[runtime.GOOS] == "" }, "Istio Bookinfo Demo", func() {
var (
resourceYAMLPaths []string
policyPaths []string
)
AfterEach(func() {
for _, resourcePath := range resourceYAMLPaths {
By("Deleting resource in file %q", resourcePath)
// Explicitly do not check result to avoid having assertions in AfterEach.
_ = kubectl.Delete(resourcePath)
}
for _, policyPath := range policyPaths {
By("Deleting policy in file %q", policyPath)
// Explicitly do not check result to avoid having assertions in AfterEach.
_ = kubectl.Delete(policyPath)
}
})
// shouldConnect checks that srcPod can connect to dstURI.
shouldConnect := func(srcPod, srcContainer, dstURI string) bool {
By("Checking that %q can connect to %q", srcPod, dstURI)
res := kubectl.ExecPodContainerCmd(
helpers.DefaultNamespace, srcPod, srcContainer, fmt.Sprintf("%s %s", wgetCommand, dstURI))
if !res.WasSuccessful() {
GinkgoPrint("Unable to connect from %q to %q: %s", srcPod, dstURI, res.OutputPrettyPrint())
return false
}
return true
}
// shouldNotConnect checks that srcPod cannot connect to dstURI.
shouldNotConnect := func(srcPod, srcContainer, dstURI string) bool {
By("Checking that %q cannot connect to %q", srcPod, dstURI)
res := kubectl.ExecPodContainerCmd(
helpers.DefaultNamespace, srcPod, srcContainer, fmt.Sprintf("%s %s", wgetCommand, dstURI))
if res.WasSuccessful() {
GinkgoPrint("Was able to connect from %q to %q, but expected no connection: %s", srcPod, dstURI, res.OutputPrettyPrint())
return false
}
return true
}
// formatLabelArgument formats the provided key-value pairs as labels for use in
// querying Kubernetes.
formatLabelArgument := func(firstKey, firstValue string, nextLabels ...string) string {
baseString := fmt.Sprintf("-l %s=%s", firstKey, firstValue)
if nextLabels == nil {
return baseString
} else if len(nextLabels)%2 != 0 {
Fail("must provide even number of arguments for label key-value pairings")
} else {
for i := 0; i < len(nextLabels); i += 2 {
baseString = fmt.Sprintf("%s,%s=%s", baseString, nextLabels[i], nextLabels[i+1])
}
}
return baseString
}
// formatAPI is a helper function which formats a URI to access.
formatAPI := func(service, port, resource string) string {
target := fmt.Sprintf(
"%s.%s.svc.cluster.local:%s",
service, helpers.DefaultNamespace, port)
if resource != "" {
return fmt.Sprintf("%s/%s", target, resource)
}
return target
}
It("Tests bookinfo inter-service connectivity", func() {
var err error
version := "version"
v1 := "v1"
productPage := "productpage"
reviews := "reviews"
ratings := "ratings"
details := "details"
dnsChecks := []string{productPage, reviews, ratings, details}
app := "app"
health := "health"
ratingsPath := "ratings/0"
apiPort := "9080"
podNameFilter := "{.items[*].metadata.name}"
bookinfoV1YAML := helpers.ManifestGet(kubectl.BasePath(), "bookinfo-v1.yaml")
bookinfoV2YAML := helpers.ManifestGet(kubectl.BasePath(), "bookinfo-v2.yaml")
l7PolicyPath := helpers.ManifestGet(kubectl.BasePath(), "cnp-specs.yaml")
waitIstioReady()
// Create the L7 policy before creating the pods, in order to test
// that the sidecar proxy mode doesn't deadlock on endpoint
// creation in this case.
policyPaths = []string{l7PolicyPath}
for _, policyPath := range policyPaths {
By("Creating policy in file %q", policyPath)
_, err := kubectl.CiliumPolicyAction(helpers.DefaultNamespace, policyPath, helpers.KubectlApply, helpers.HelperTimeout)
Expect(err).Should(BeNil(), "Unable to create policy %q", policyPath)
}
resourceYAMLPaths = []string{bookinfoV2YAML, bookinfoV1YAML}
for _, resourcePath := range resourceYAMLPaths {
By("Creating resources in file %q", resourcePath)
res := kubectl.Create(resourcePath)
res.ExpectSuccess("Unable to create resource %q", resourcePath)
}
// Wait for pods and endpoints to be ready before creating the
// next resources to reduce the load on the next pod creations,
// in order to reduce the probability of regeneration timeout.
By("Waiting for Bookinfo pods to be ready")
err = kubectl.WaitforPods(helpers.DefaultNamespace, "-l zgroup=bookinfo", helpers.HelperTimeout)
Expect(err).Should(BeNil(), "Pods are not ready after timeout")
By("Waiting for Bookinfo endpoints to be ready")
err = kubectl.CiliumEndpointWaitReady()
Expect(err).Should(BeNil(), "Endpoints are not ready after timeout")
for _, service := range []string{details, ratings, reviews, productPage} {
By("Waiting for Bookinfo service %q to be ready", service)
err = kubectl.WaitForServiceEndpoints(
helpers.DefaultNamespace, "", service,
helpers.HelperTimeout)
Expect(err).Should(BeNil(), "Service %q is not ready after timeout", service)
}
for _, name := range dnsChecks {
By("Waiting for DNS to resolve Bookinfo service %q", name)
err = kubectl.WaitForKubeDNSEntry(name, helpers.DefaultNamespace)
Expect(err).To(BeNil(), "DNS entry is not ready after timeout")
}
By("Testing L7 filtering")
reviewsPodV1, err := kubectl.GetPods(helpers.DefaultNamespace, formatLabelArgument(app, reviews, version, v1)).Filter(podNameFilter)
Expect(err).Should(BeNil(), "Cannot get reviewsV1 pods")
productpagePodV1, err := kubectl.GetPods(helpers.DefaultNamespace, formatLabelArgument(app, productPage, version, v1)).Filter(podNameFilter)
Expect(err).Should(BeNil(), "Cannot get productpageV1 pods")
// Connectivity checks often need to be repeated because Pilot
// is eventually consistent, i.e. it may take some time for a
// sidecar proxy to get updated with the configuration for another
// new endpoint and it rejects egress traffic with 503s in the
// meantime.
err = helpers.WithTimeout(func() bool {
allGood := true
allGood = shouldConnect(reviewsPodV1.String(), "reviews", formatAPI(ratings, apiPort, health)) && allGood
allGood = shouldNotConnect(reviewsPodV1.String(), "reviews", formatAPI(ratings, apiPort, ratingsPath)) && allGood
allGood = shouldConnect(productpagePodV1.String(), "productpage", formatAPI(details, apiPort, health)) && allGood
allGood = shouldNotConnect(productpagePodV1.String(), "productpage", formatAPI(ratings, apiPort, health)) && allGood
allGood = shouldNotConnect(productpagePodV1.String(), "productpage", formatAPI(ratings, apiPort, ratingsPath)) && allGood
return allGood
}, "Istio sidecar proxies are not configured", &helpers.TimeoutConfig{Timeout: helpers.HelperTimeout})
Expect(err).Should(BeNil(), "Cannot configure Istio sidecar proxies")
})
})
})