-
Notifications
You must be signed in to change notification settings - Fork 53
140 lines (140 loc) · 5.48 KB
/
test-failover.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
---
name: Failover test
on:
workflow_dispatch:
inputs:
image:
description: GCP image for test cluster
required: true
default: almalinux-cloud/almalinux-8
version:
description: PE version to install
required: true
default: 2019.8.12
version_to_upgrade:
description: PE version to upgrade to
required: false
default: 2021.7.9
ssh-debugging:
description: Boolean; whether or not to pause for ssh debugging
required: true
default: 'false'
log_level:
description: Bolt log level
required: false
default: debug
jobs:
test-failover:
name: PE ${{ matrix.version }} ${{ matrix.architecture }} on ${{ matrix.image }}
runs-on: ubuntu-20.04
env:
BOLT_GEM: true
BOLT_DISABLE_ANALYTICS: true
LANG: en_US.UTF-8
strategy:
fail-fast: false
matrix:
version: ['${{ github.event.inputs.version }}']
image: ['${{ github.event.inputs.image }}']
architecture: [extra-large-with-dr]
steps:
- name: Start SSH session
if: ${{ github.event.inputs.ssh-debugging == 'true' }}
uses: luchihoratiu/debug-via-ssh@main
with:
NGROK_AUTH_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }}
SSH_PASS: ${{ secrets.SSH_PASS }}
- name: Checkout Source
uses: actions/checkout@v4
- name: Activate Ruby 2.7
uses: ruby/setup-ruby@v1
with:
ruby-version: '2.7'
bundler-cache: true
- name: Print bundle environment
if: ${{ github.repository_owner == 'puppetlabs' }}
run: |
echo ::group::info:bundler
bundle env
echo ::endgroup::
- name: Provision test cluster (XL with spare replica)
timeout-minutes: 15
run: |
echo ::group::prepare
mkdir -p $HOME/.ssh
echo 'Host *' > $HOME/.ssh/config
echo ' ServerAliveInterval 150' >> $HOME/.ssh/config
echo ' ServerAliveCountMax 2' >> $HOME/.ssh/config
bundle exec rake spec_prep
echo ::endgroup::
echo ::group::provision
bundle exec bolt plan run peadm_spec::provision_test_cluster \
--modulepath spec/fixtures/modules \
provider=provision_service \
image=${{ matrix.image }} \
architecture=${{ matrix.architecture }}-and-spare-replica
echo ::endgroup::
echo ::group::info:request
cat request.json || true; echo
echo ::endgroup::
echo ::group::info:inventory
sed -e 's/password: .*/password: "[redacted]"/' < spec/fixtures/litmus_inventory.yaml || true
echo ::endgroup::
- name: Install PE on test cluster
timeout-minutes: 120
run: |
bundle exec bolt plan run peadm_spec::install_test_cluster --log_level ${{ github.event.inputs.log_level }} \
--inventoryfile spec/fixtures/litmus_inventory.yaml \
--modulepath spec/fixtures/modules \
architecture=${{ matrix.architecture }} \
version=${{ matrix.version }} \
console_password=${{ secrets.CONSOLE_PASSWORD }}
- name: Perform failover
run: |
bundle exec bolt plan run peadm_spec::perform_failover --log_level ${{ github.event.inputs.log_level }} \
--inventoryfile spec/fixtures/litmus_inventory.yaml \
--modulepath spec/fixtures/modules
- name: Wait as long as the file ${HOME}/pause file is present
if: ${{ always() && github.event.inputs.ssh-debugging == 'true' }}
run: |
while [ -f "${HOME}/pause" ] ; do
echo "${HOME}/pause present, sleeping for 60 seconds..."
sleep 60
done
echo "${HOME}/pause absent, continuing workflow."
- name: Set up yq
uses: frenck/action-setup-yq@v1
with:
version: v4.30.5
- name: Update inventory
run: |
# Remove failed primary
yq -i 'del(.groups[].targets[] | select(.vars.role == "primary"))' spec/fixtures/litmus_inventory.yaml
# Swap primary and replica nodes
sed -i.sedbak 's/primary/__tmp__/;s/spare-replica/__tmp2__/;s/replica/primary/;s/__tmp__/replica/;s/__tmp2__/replica/' \
spec/fixtures/litmus_inventory.yaml
echo ::group::info:inventory
sed -e 's/password: .*/password: "[redacted]"/' < spec/fixtures/litmus_inventory.yaml || true
echo ::endgroup::
- name: Upgrade PE on test cluster
if: ${{ success() && github.event.inputs.version_to_upgrade != '' }}
timeout-minutes: 120
run: |
bundle exec bolt plan run peadm_spec::upgrade_test_cluster --log_level ${{ github.event.inputs.log_level }} \
--inventoryfile spec/fixtures/litmus_inventory.yaml \
--modulepath spec/fixtures/modules \
architecture=${{ matrix.architecture }} \
download_mode='direct' \
version=${{ github.event.inputs.version_to_upgrade }}
- name: Tear down test cluster
if: ${{ always() }}
continue-on-error: true
run: |-
if [ -f spec/fixtures/litmus_inventory.yaml ]; then
echo ::group::tear_down
bundle exec rake 'litmus:tear_down'
echo ::endgroup::
echo ::group::info:request
cat request.json || true; echo
echo ::endgroup::
fi