/
gmsa_full.go
423 lines (366 loc) · 14.4 KB
/
gmsa_full.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// This test ensures that the whole GMSA process works as intended.
// However, it does require a pretty heavy weight set up to run correctly;
// in particular, it does make a number of assumptions about the cluster it
// runs against:
// * there exists a Windows worker node with the agentpool=windowsgmsa label on it
// * that node is joined to a working Active Directory domain.
// * a GMSA account has been created in that AD domain, and then installed on that
// same worker.
// * a valid k8s manifest file containing a single CRD definition has been generated using
// https://github.com/kubernetes-sigs/windows-gmsa/blob/master/scripts/GenerateCredentialSpecResource.ps1
// with the credential specs of that GMSA account, or type GMSACredentialSpec and named gmsa-e2e;
// and that manifest file has been written to C:\gmsa\gmsa-cred-spec-gmsa-e2e.yml
// on that same worker node.
// * the API has both MutatingAdmissionWebhook and ValidatingAdmissionWebhook
// admission controllers enabled.
// * the cluster comprises at least one Linux node that accepts workloads - it
// can be the master, but any other Linux node is fine too. This is needed for
// the webhook's pod.
// All these assumptions are fulfilled by an AKS extension when setting up the AKS
// cluster we run daily e2e tests against, but they do make running this test
// outside of that very specific context pretty hard.
package windows
import (
"context"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
"strings"
"time"
v1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
imageutils "k8s.io/kubernetes/test/utils/image"
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
)
const (
// gmsaFullNodeLabel is the label we expect to find on at least one node
// that is then expected to fulfill all the expectations explained above.
gmsaFullNodeLabel = "agentpool=windowsgmsa"
// gmsaCrdManifestPath is where we expect to find the manifest file for
// the GMSA cred spec on that node - see explanations above.
gmsaCrdManifestPath = `C:\gmsa\gmsa-cred-spec-gmsa-e2e.yml`
// gmsaCustomResourceName is the expected name of the GMSA custom resource
// defined at gmsaCrdManifestPath
gmsaCustomResourceName = "gmsa-e2e"
// gmsaWebhookDeployScriptURL is the URL of the deploy script for the GMSA webook
// TODO(wk8): we should pin versions.
gmsaWebhookDeployScriptURL = "https://raw.githubusercontent.com/kubernetes-sigs/windows-gmsa/master/admission-webhook/deploy/deploy-gmsa-webhook.sh"
// output from the nltest /query command should have this in it
expectedQueryOutput = "The command completed successfully"
// The name of the expected domain
gmsaDomain = "k8sgmsa.lan"
)
var _ = SIGDescribe("[Feature:Windows] GMSA Full [Serial] [Slow]", func() {
f := framework.NewDefaultFramework("gmsa-full-test-windows")
ginkgo.Describe("GMSA support", func() {
ginkgo.It("works end to end", func() {
defer ginkgo.GinkgoRecover()
ginkgo.By("finding the worker node that fulfills this test's assumptions")
nodes := findPreconfiguredGmsaNodes(f.ClientSet)
if len(nodes) != 1 {
e2eskipper.Skipf("Expected to find exactly one node with the %q label, found %d", gmsaFullNodeLabel, len(nodes))
}
node := nodes[0]
ginkgo.By("retrieving the contents of the GMSACredentialSpec custom resource manifest from the node")
crdManifestContents := retrieveCRDManifestFileContents(f, node)
ginkgo.By("downloading the GMSA webhook deploy script")
deployScriptPath, err := downloadFile(gmsaWebhookDeployScriptURL)
defer func() { os.Remove(deployScriptPath) }()
if err != nil {
framework.Failf(err.Error())
}
ginkgo.By("deploying the GMSA webhook")
webhookCleanUp, err := deployGmsaWebhook(f, deployScriptPath)
defer webhookCleanUp()
if err != nil {
framework.Failf(err.Error())
}
ginkgo.By("creating the GMSA custom resource")
customResourceCleanup, err := createGmsaCustomResource(f.Namespace.Name, crdManifestContents)
defer customResourceCleanup()
if err != nil {
framework.Failf(err.Error())
}
ginkgo.By("creating an RBAC role to grant use access to that GMSA resource")
rbacRoleName, rbacRoleCleanup, err := createRBACRoleForGmsa(f)
defer rbacRoleCleanup()
if err != nil {
framework.Failf(err.Error())
}
ginkgo.By("creating a service account")
serviceAccountName := createServiceAccount(f)
ginkgo.By("binding the RBAC role to the service account")
bindRBACRoleToServiceAccount(f, serviceAccountName, rbacRoleName)
ginkgo.By("creating a pod using the GMSA cred spec")
podName := createPodWithGmsa(f, serviceAccountName)
// nltest /QUERY will only return successfully if there is a GMSA
// identity configured, _and_ it succeeds in contacting the AD controller
// and authenticating with it.
ginkgo.By("checking that nltest /QUERY returns successfully")
var output string
gomega.Eventually(func() bool {
output, err = runKubectlExecInNamespace(f.Namespace.Name, podName, "nltest", "/QUERY")
if err != nil {
framework.Logf("unable to run command in container via exec: %s", err)
return false
}
if !isValidOutput(output) {
// try repairing the secure channel by running reset command
// https://kubernetes.io/docs/tasks/configure-pod-container/configure-gmsa/#troubleshooting
output, err = runKubectlExecInNamespace(f.Namespace.Name, podName, "nltest", fmt.Sprintf("/sc_reset:%s", gmsaDomain))
if err != nil {
framework.Logf("unable to run command in container via exec: %s", err)
return false
}
framework.Logf("failed to connect to domain; tried resetting the domain, output:\n%s", string(output))
return false
}
return true
}, 1*time.Minute, 1*time.Second).Should(gomega.BeTrue())
})
})
})
func isValidOutput(output string) bool {
return strings.Contains(output, expectedQueryOutput) &&
!strings.Contains(output, "ERROR_NO_LOGON_SERVERS") &&
!strings.Contains(output, "RPC_S_SERVER_UNAVAILABLE")
}
// findPreconfiguredGmsaNode finds node with the gmsaFullNodeLabel label on it.
func findPreconfiguredGmsaNodes(c clientset.Interface) []v1.Node {
nodeOpts := metav1.ListOptions{
LabelSelector: gmsaFullNodeLabel,
}
nodes, err := c.CoreV1().Nodes().List(context.TODO(), nodeOpts)
if err != nil {
framework.Failf("Unable to list nodes: %v", err)
}
return nodes.Items
}
// retrieveCRDManifestFileContents retrieves the contents of the file
// at gmsaCrdManifestPath on node; it does so by scheduling a single pod
// on nodes with the gmsaFullNodeLabel label with that file's directory
// mounted on it, and then exec-ing into that pod to retrieve the file's
// contents.
func retrieveCRDManifestFileContents(f *framework.Framework, node v1.Node) string {
podName := "retrieve-gmsa-crd-contents"
// we can't use filepath.Dir here since the test itself runs on a Linux machine
splitPath := strings.Split(gmsaCrdManifestPath, `\`)
dirPath := strings.Join(splitPath[:len(splitPath)-1], `\`)
volumeName := "retrieve-gmsa-crd-contents-volume"
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Namespace: f.Namespace.Name,
},
Spec: v1.PodSpec{
NodeSelector: node.Labels,
Containers: []v1.Container{
{
Name: podName,
Image: imageutils.GetPauseImageName(),
VolumeMounts: []v1.VolumeMount{
{
Name: volumeName,
MountPath: dirPath,
},
},
},
},
Volumes: []v1.Volume{
{
Name: volumeName,
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: dirPath,
},
},
},
},
},
}
f.PodClient().CreateSync(pod)
output, err := runKubectlExecInNamespace(f.Namespace.Name, podName, "cmd", "/S", "/C", fmt.Sprintf("type %s", gmsaCrdManifestPath))
if err != nil {
framework.Failf("failed to retrieve the contents of %q on node %q: %v", gmsaCrdManifestPath, node.Name, err)
}
// Windows to linux new lines
return strings.ReplaceAll(output, "\r\n", "\n")
}
// deployGmsaWebhook deploys the GMSA webhook, and returns a cleanup function
// to be called when done with testing, that removes the temp files it's created
// on disks as well as the API resources it's created.
func deployGmsaWebhook(f *framework.Framework, deployScriptPath string) (func(), error) {
cleanUpFunc := func() {}
tempDir, err := ioutil.TempDir("", "")
if err != nil {
return cleanUpFunc, fmt.Errorf("unable to create temp dir: %w", err)
}
manifestsFile := path.Join(tempDir, "manifests.yml")
name := "gmsa-webhook"
namespace := f.Namespace.Name + "-webhook"
certsDir := path.Join(tempDir, "certs")
// regardless of whether the deployment succeeded, let's do a best effort at cleanup
cleanUpFunc = func() {
framework.RunKubectl(f.Namespace.Name, "delete", "--filename", manifestsFile)
framework.RunKubectl(f.Namespace.Name, "delete", "CustomResourceDefinition", "gmsacredentialspecs.windows.k8s.io")
framework.RunKubectl(f.Namespace.Name, "delete", "CertificateSigningRequest", fmt.Sprintf("%s.%s", name, namespace))
os.RemoveAll(tempDir)
}
cmd := exec.Command("bash", deployScriptPath,
"--file", manifestsFile,
"--name", name,
"--namespace", namespace,
"--certs-dir", certsDir,
"--tolerate-master")
output, err := cmd.CombinedOutput()
if err == nil {
framework.Logf("GMSA webhook successfully deployed, output:\n%s", string(output))
} else {
err = fmt.Errorf("unable to deploy GMSA webhook, output:\n%s: %w", string(output), err)
}
return cleanUpFunc, err
}
// createGmsaCustomResource creates the GMSA API object from the contents
// of the manifest file retrieved from the worker node.
// It returns a function to clean up both the temp file it creates and
// the API object it creates when done with testing.
func createGmsaCustomResource(ns string, crdManifestContents string) (func(), error) {
cleanUpFunc := func() {}
tempFile, err := ioutil.TempFile("", "")
if err != nil {
return cleanUpFunc, fmt.Errorf("unable to create temp file: %w", err)
}
defer tempFile.Close()
cleanUpFunc = func() {
framework.RunKubectl(ns, "delete", "--filename", tempFile.Name())
os.Remove(tempFile.Name())
}
_, err = tempFile.WriteString(crdManifestContents)
if err != nil {
err = fmt.Errorf("unable to write GMSA contents to %q: %w", tempFile.Name(), err)
return cleanUpFunc, err
}
output, err := framework.RunKubectl(ns, "apply", "--filename", tempFile.Name())
if err != nil {
err = fmt.Errorf("unable to create custom resource, output:\n%s: %w", output, err)
}
return cleanUpFunc, err
}
// createRBACRoleForGmsa creates an RBAC cluster role to grant use
// access to our test credential spec.
// It returns the role's name, as well as a function to delete it when done.
func createRBACRoleForGmsa(f *framework.Framework) (string, func(), error) {
roleName := f.Namespace.Name + "-rbac-role"
role := &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: roleName,
},
Rules: []rbacv1.PolicyRule{
{
APIGroups: []string{"windows.k8s.io"},
Resources: []string{"gmsacredentialspecs"},
Verbs: []string{"use"},
ResourceNames: []string{gmsaCustomResourceName},
},
},
}
cleanUpFunc := func() {
f.ClientSet.RbacV1().ClusterRoles().Delete(context.TODO(), roleName, metav1.DeleteOptions{})
}
_, err := f.ClientSet.RbacV1().ClusterRoles().Create(context.TODO(), role, metav1.CreateOptions{})
if err != nil {
err = fmt.Errorf("unable to create RBAC cluster role %q: %w", roleName, err)
}
return roleName, cleanUpFunc, err
}
// createServiceAccount creates a service account, and returns its name.
func createServiceAccount(f *framework.Framework) string {
accountName := f.Namespace.Name + "-sa"
account := &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: accountName,
Namespace: f.Namespace.Name,
},
}
if _, err := f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name).Create(context.TODO(), account, metav1.CreateOptions{}); err != nil {
framework.Failf("unable to create service account %q: %v", accountName, err)
}
return accountName
}
// bindRBACRoleToServiceAccount binds the given RBAC cluster role to the given service account.
func bindRBACRoleToServiceAccount(f *framework.Framework, serviceAccountName, rbacRoleName string) {
binding := &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: f.Namespace.Name + "-rbac-binding",
Namespace: f.Namespace.Name,
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: serviceAccountName,
Namespace: f.Namespace.Name,
},
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: rbacRoleName,
},
}
f.ClientSet.RbacV1().RoleBindings(f.Namespace.Name).Create(context.TODO(), binding, metav1.CreateOptions{})
}
// createPodWithGmsa creates a pod using the test GMSA cred spec, and returns its name.
func createPodWithGmsa(f *framework.Framework, serviceAccountName string) string {
podName := "pod-with-gmsa"
credSpecName := gmsaCustomResourceName
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Namespace: f.Namespace.Name,
},
Spec: v1.PodSpec{
ServiceAccountName: serviceAccountName,
Containers: []v1.Container{
{
Name: podName,
Image: imageutils.GetE2EImage(imageutils.BusyBox),
Command: []string{
"powershell.exe",
"-Command",
"sleep -Seconds 600",
},
},
},
SecurityContext: &v1.PodSecurityContext{
WindowsOptions: &v1.WindowsSecurityContextOptions{
GMSACredentialSpecName: &credSpecName,
},
},
},
}
f.PodClient().CreateSync(pod)
return podName
}
func runKubectlExecInNamespace(namespace string, args ...string) (string, error) {
namespaceOption := fmt.Sprintf("--namespace=%s", namespace)
return framework.RunKubectl(namespace, append([]string{"exec", namespaceOption}, args...)...)
}