-
Notifications
You must be signed in to change notification settings - Fork 3
/
main.yml
434 lines (390 loc) · 13 KB
/
main.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
---
- name: Check amount of clusters
include_role:
name: install-multi-cockroachdb
tasks_from: pre_validation
- name: Create cockroachdb namespaces for managed clusters
kubernetes.core.k8s:
wait: yes
state: present
context: "{{ item.contexts }}"
definition:
apiVersion: project.openshift.io/v1
description: cockroachdb
displayName: cockroachdb
kind: Project
metadata:
name: cockroachdb
loop: "{{ clusters }}"
- name: Wait for namespace to be created before creating secrets
kubernetes.core.k8s_info:
api_version: project.openshift.io/v1
kind: Project
name: cockroachdb
wait: yes
context: "{{ item.contexts }}"
wait_sleep: 15
wait_timeout: 400
register: namespace_debug
loop: "{{ clusters }}"
- name: Create variable for IR_PASSWORD registry secret
set_fact:
IR_PASSWORD: "{{ lookup('file', '{{ role_path }}/templates/ir_password.yaml') | from_json }}"
- name: Create ir_password secret in each managed cluster
kubernetes.core.k8s:
wait: yes
context: "{{ item.contexts }}"
namespace: cockroachdb
state: present
definition:
apiVersion: v1
kind: Secret
metadata:
name: ir-secret
data:
.dockerconfigjson: "{{ IR_PASSWORD | to_json | b64encode }}"
type: kubernetes.io/dockerconfigjson
loop: "{{ clusters }}"
- name: Create Service Account for cockroachdb sts
kubernetes.core.k8s:
state: present
wait: yes
context: "{{ item.contexts }}"
definition:
apiVersion: v1
kind: ServiceAccount
metadata:
creationTimestamp: null
name: cockroachdb
namespace: cockroachdb
loop: "{{ clusters }}"
- name: Create ClusterRoleBindings for cockroach service account
kubernetes.core.k8s:
wait: yes
state: present
context: "{{ item.contexts }}"
definition:
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
creationTimestamp: null
name: "cockroach-sa-cluster-admin-binding"
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:serviceaccounts:cockroachdb
loop: "{{ clusters }}"
- name: Wait till submariner add on install completes
kubernetes.core.k8s_info:
api_version: addon.open-cluster-management.io/v1alpha1
kind: ManagedClusterAddOn
name: submariner
namespace: "{{ item.name }}"
wait: yes
context: "{{ hub_context }}"
wait_sleep: 15
wait_timeout: 400
wait_condition:
type: RegistrationApplied
status: True
loop: "{{ clusters }}"
- name: Create variables for node crt command for each cluster
set_fact:
node_crt_key_{{ my_idx }}: "*.{{ item.contexts }}.cockroachdb.cockroachdb.svc.clusterset.local"
loop: "{{ clusters }}"
loop_control:
index_var: my_idx
- name: Creat list of node crt key command
ansible.builtin.set_fact:
node_crt_key_command: "{{ node_crt_key_command | default('') + [vars['node_crt_key_' + my_idx|string]] }}"
loop: "{{ clusters }}"
loop_control:
index_var: my_idx
- name: Format node crt key command
ansible.builtin.set_fact:
format_node_crt_key_command: "{{ node_crt_key_command | join (' ') }}"
- name: Dispalying formatted node cert key command
ansible.builtin.debug:
msg: "{{ format_node_crt_key_command }}"
- name: Create directories needed for cockroachdb statefuleset
ansible.builtin.file:
path: "{{ playbook_dir }}/resources/cockroachdb/cluster{{ my_idx }}"
state: directory
loop: "{{ clusters }}"
loop_control:
index_var: my_idx
- name: Run task to format the templated statefulsets
include_tasks: format-template-statefulsets.yml
- name: git push
include_role:
name: ansible-git
tasks_from: push
- name: Create ClusterRoleBinding for cockroachdb Service Account on managed in GCP
kubernetes.core.k8s:
wait: yes
state: present
context: "{{ item.contexts }}"
definition:
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:openshift:scc:anyuid
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:openshift:scc:anyuid
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:serviceaccounts:cockroachdb
when: item.cloud == "gcp"
loop: "{{ clusters }}"
- name: Create cockroachdb namespaces for hub channel
kubernetes.core.k8s:
wait: yes
state: present
context: "{{ hub_context }}"
definition:
apiVersion: project.openshift.io/v1
description: cockroachdb
displayName: cockroachdb
kind: Project
metadata:
name: cockroachdb
- name: Create directories needed for cockroachdb certs
ansible.builtin.file:
path: "{{ role_path }}/files/{{ item }}"
state: directory
loop:
- certs
- my-safe-directory
- name: Create the CA certificate and key pair
ansible.builtin.command: "{{ cockroach }} cert create-ca --certs-dir={{ role_path }}/files/certs/ --ca-key={{ role_path }}/files/my-safe-directory/ca.key"
args:
creates: "{{ role_path }}/files/certs/ca.crt"
- name: Create a client certificate and key pair for the root user
ansible.builtin.command: "{{ cockroach }} cert create-client root --certs-dir={{ role_path }}/files/certs/ --ca-key={{ role_path }}/files/my-safe-directory/ca.key"
args:
creates: "{{ role_path }}/files/certs/client.root.crt"
- name: Fetch created client certificate and key pair
set_fact:
var_{{ item.name }}: "{{ lookup('file', '{{ role_path }}/files/certs/{{ item.file }}') }}"
loop:
- {file: ca.crt, name: ca_crt}
- {file: client.root.crt, name: client_root_crt}
- {file: client.root.key, name: client_root_key}
- name: Create cockroachdb.client.root secret on managed clusters
kubernetes.core.k8s:
state: present
context: "{{ item.contexts }}"
definition:
apiVersion: v1
kind: Secret
data:
ca.crt: "{{ var_ca_crt | b64encode }}"
client.root.crt: "{{ var_client_root_crt | b64encode }}"
client.root.key: "{{ var_client_root_key | b64encode }}"
metadata:
name: cockroachdb.client.root
namespace: cockroachdb
type: Opaque
loop: "{{ clusters }}"
- name: Create the certificate and key pair for your CockroachDB nodes
ansible.builtin.command: "{{ cockroach }} cert create-node localhost 127.0.0.1 cockroachdb-public cockroachdb-public.cockroachdb cockroachdb-public.cockroachdb.svc.cluster.local *.cockroachdb *.cockroachdb.cockroachdb *.cockroachdb.cockroachdb.svc.cluster.local {{ format_node_crt_key_command }} *.cockroachdb.cockroachdb.svc.clusterset.local --certs-dir={{ role_path }}/files/certs --ca-key={{ role_path }}/files/my-safe-directory/ca.key"
args:
creates: "{{ role_path }}/files/certs/node.crt"
- name: Fetch created node client certificate
set_fact:
node_crt: "{{ lookup('file', '{{ role_path }}/files/certs/node.crt') }}"
- name: Fetch created node client key
set_fact:
node_key: "{{ lookup('file', '{{ role_path }}/files/certs/node.key') }}"
- name: Create cockroachdb.node secret on managed item
kubernetes.core.k8s:
state: present
context: "{{ item.contexts }}"
definition:
apiVersion: v1
kind: Secret
data:
ca.crt: "{{ var_ca_crt | b64encode }}"
client.root.crt: "{{ var_client_root_crt | b64encode }}"
client.root.key: "{{ var_client_root_key | b64encode }}"
node.crt: "{{ node_crt | b64encode }}"
node.key: "{{ node_key | b64encode }}"
metadata:
name: cockroachdb.node
namespace: cockroachdb
type: Opaque
loop: "{{ clusters }}"
- name: Get git_url to format for https protocol for ACM channel creation
set_fact:
git_https_format: "{{ git_url | regex_search('git.github.com:(.*)', '\\1') | first }}"
- name: Create app_repo var for channel github https
set_fact:
app_repo: "{{ 'https://github.com/' + git_https_format }}"
- name: Create channel for cockroachdb application in ACM
kubernetes.core.k8s:
wait: yes
state: present
context: "{{ hub_context }}"
definition:
apiVersion: apps.open-cluster-management.io/v1
kind: Channel
metadata:
name: cockroachdb-app-latest
namespace: cockroachdb
spec:
type: GitHub
pathname: "{{ app_repo }}"
- name: Create cockroachdb application in ACM
kubernetes.core.k8s:
wait: yes
state: present
context: "{{ hub_context }}"
definition:
apiVersion: app.k8s.io/v1beta1
kind: Application
metadata:
name: cockroachdb-app
namespace: cockroachdb
spec:
componentKinds:
- group: apps.open-cluster-management.io
kind: Subscription
descriptor: {}
selector:
matchLabels:
app: cockroachdb-app
- name: Create cockroachdb subscription in ACM
kubernetes.core.k8s:
state: present
context: "{{ hub_context }}"
definition:
apiVersion: apps.open-cluster-management.io/v1
kind: Subscription
metadata:
name: cockroachdb-cluster{{ my_idx }}
namespace: cockroachdb
labels:
app: cockroachdb-app
annotations:
apps.open-cluster-management.io/github-path: resources/cockroachdb/cluster{{ my_idx|string }}
apps.open-cluster-management.io/git-branch: main
spec:
channel: cockroachdb/cockroachdb-app-latest
placement:
clusters:
- name: "{{ item.name }}"
loop: "{{ clusters }}"
loop_control:
index_var: my_idx
- name: Create cockroachdb placement rule in ACM
kubernetes.core.k8s:
state: present
context: "{{ hub_context }}"
definition:
apiVersion: apps.open-cluster-management.io/v1
kind: PlacementRule
metadata:
name: cockroachdb-clusters
namespace: cockroachdb
spec:
clusterConditions:
- type: ManagedClusterConditionAvailable
status: "True"
clusterSelector:
matchLabels:
usage: cockroachdb
- name: Pause and wait to create ServiceExport in managed clusters and for Submariner Gateways to running
pause:
seconds: 300
- name: Create the serviceExport for cockroachdb service
kubernetes.core.k8s:
wait: yes
state: present
context: "{{ item.contexts }}"
definition:
apiVersion: multicluster.x-k8s.io/v1alpha1
kind: ServiceExport
metadata:
name: cockroachdb
namespace: cockroachdb
loop: "{{ clusters }}"
- name: Wait till the cockroachdb-0 pod is ready
kubernetes.core.k8s_info:
kind: Pod
name: cockroachdb-0
context: "{{ item.contexts }}"
namespace: cockroachdb
wait: yes
wait_sleep: 15
wait_timeout: 600
wait_condition:
type: Initialized
status: True
loop: "{{ clusters }}"
loop_control:
index_var: my_idx
when: my_idx == 0
- name: get status submariner
kubernetes.core.k8s_info:
api_version: addon.open-cluster-management.io/v1alpha1
kind: ManagedClusterAddOn
name: submariner
context: "{{ hub_context }}"
namespace: "{{ item.name }}"
loop: "{{ clusters }}"
register: submariner_status
- name: get status output
set_fact:
submariner_ready_status: "{{ item | json_query('resources[].status.conditions[].status') | regex_search('False') }}"
loop: "{{ submariner_status.results }}"
loop_control:
index_var: my_idx
- name: check submariner status
debug:
msg: "Submariner status False, recycling gateway pods"
when: submariner_ready_status == false
- name: Recycle Submariner Gateway Pods
ansible.builtin.shell: "{{ kubectl }} delete pods -l app=submariner-gateway -n submariner-operator --context {{ item.name }}"
loop: "{{ clusters }}"
when: submariner_ready_status == false
- name: Pause and wait for pods from StatefulSet to become ready
pause:
seconds: 600
when: submariner_ready_status == false
- name: get status of pod
kubernetes.core.k8s_info:
kind: Pod
name: cockroachdb-0
context: "{{ item.contexts }}"
namespace: cockroachdb
wait_condition:
type: Ready
status: True
loop: "{{ clusters }}"
register: cockroachdb_ready
loop_control:
index_var: my_idx
when: my_idx == 0
- name: get status output
set_fact:
cockroachdb_ready_status: "{{ cockroachdb_ready | json_query('results[].resources[].status.conditions[].status') | regex_search('False') }}"
- name: check status
debug:
msg: "Pod not initialized, will intialize now"
when: cockroachdb_ready_status == false
- name: Run cockroach init to complete the node startup process and have them join together as a cluster
ansible.builtin.shell: "{{ kubectl }} exec --context {{ clusters[0].contexts }} --namespace cockroachdb -it cockroachdb-0 -- /cockroach/cockroach init --certs-dir=/cockroach/cockroach-certs"
loop: "{{ clusters }}"
ignore_errors: yes
loop_control:
index_var: my_idx
when:
- my_idx == 0
- cockroachdb_ready_status == false