-
Notifications
You must be signed in to change notification settings - Fork 0
/
element_upgrade.py
196 lines (180 loc) · 9 KB
/
element_upgrade.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
from gettext import translation
import json
import logging
import time
from time import sleep
from get_token import GetToken
from log_setup import Logging
from program_data import PDApi, Common
"""
NetApp / SolidFire
CPE
mnode support utility
"""
"""
Element upgrade
"""
# set up logging
logmsg = Logging.logmsg()
class ElemUpgrade():
def __init__(self):
upgrade_id = ""
upgrade_target = ""
def _config_options(self):
""" Config options
"""
option = "string"
value = []
configoptions = {}
logmsg.info("\nSee the following KB for config samples.")
logmsg.info("https://kb.netapp.com/Advice_and_Troubleshooting/Hybrid_Cloud_Infrastructure/NetApp_HCI/Potential_issues_and_workarounds_when_running_storage_upgrades_using_NetApp_Hybrid_Cloud_Control")
configinput = input("\nEnter exact json or press enter to be prompted: ").rstrip()
if configinput == "":
logmsg.info("Enter option names and values. Enter q when done")
while option != "":
userinput = input("Enter the option name: ").rstrip()
if userinput.lower() == 'q':
break
else:
option = userinput
value = input("Enter the option value: ").rstrip()
if ',' in value:
configoptions[option] = value.split(',')
else:
configoptions[option] = value
configjson = configoptions
else:
configjson = Common.test_json_loads(configinput)
return configjson
def upgrade_option(self):
""" Choose upgrade option
"""
userinput = ""
options = ['s','v','p','r','a','q']
while userinput not in options:
userinput = input("\nUpgrade options: Start, View, Pause, Resume, Abort, Quit: s/v/p/r/a/q: ").rstrip()
return userinput
def select_target_cluster(self, repo):
""" Select target cluster for upgrade
"""
self.upgrade_target = Common.select_target_cluster(repo)
def select_version(self, repo):
""" Select Element version
"""
userinput = "none"
pkglist = {}
logmsg.info("\nLooking for a valid upgrade image...")
url = f'{repo.base_url}/storage/1/clusters/{self.upgrade_target}/valid-packages'
json_return = PDApi.send_get_return_json(repo, url, debug=repo.debug)
if json_return is not None:
for package in json_return:
pkglist[package["filename"]] = package["packageId"]
logmsg.info(f'\t{package["filename"]}')
while userinput not in pkglist:
userinput = input("\nEnter the target package from the list: ").rstrip()
upgrade_package = pkglist[userinput]
logmsg.info(f'Selected package {upgrade_package}')
return upgrade_package
else:
logmsg.info("\nNo valid upgrade packages found.\n\tThe selected cluster is at or above available upgrade packages.\n\tUpload a package with the -a elementupload option")
exit(0)
def start_upgrade(self, repo, upgrade_package):
""" Start Upgrade
"""
logmsg.info("\nStarting Element upgrade")
url = (repo.base_url + "/storage/1/upgrades")
userinput = ""
while userinput != 'y' and userinput != 'n':
userinput = input("Do you have any config options to add? (y/n) ").rstrip()
if userinput == 'y':
configjson = self._config_options()
payload = { "config": configjson, "packageId":upgrade_package,"storageId":self.upgrade_target }
else:
payload = { "config": {}, "packageId":upgrade_package,"storageId":self.upgrade_target }
logmsg.debug(f'Sending POST {url} {json.dumps(payload)}')
json_return = PDApi.send_post_return_json(repo, url, payload)
if json_return is not None:
logmsg.info(f'\nUpgrade ID: {json_return["upgradeId"]}')
logmsg.info(f'Upgrade task ID: {json_return["taskId"]}')
self.upgrade_id = json_return["upgradeId"]
def find_upgrade(self, repo):
""" Find Upgrade
"""
logmsg.info("Find active element upgrade(s)")
url = f'{repo.base_url}/storage/1/upgrades?includeCompleted=false'
json_return = PDApi.send_get_return_json(repo, url, debug=repo.debug)
if len(json_return) == 1:
logmsg.info(f'Running upgrade\nUpgrade ID {json_return[0]["upgradeId"]}\n\tState: {json_return[0]["state"]}\n\tStarted: {json_return[0]["dateCreated"]}\n\tStatus: {json_return[0]["status"]["message"]}\n\tAvailable actions: {json_return[0]["status"]["availableActions"]}\n')
self.upgrade_id = json_return[0]["upgradeId"]
elif len(json_return) > 1:
logmsg.info('Running upgrades\n')
for upgrade in json_return:
logmsg.info(f'Upgrade ID: {upgrade["upgradeId"]}\n\tState: {upgrade["state"]}\n\tStarted: {upgrade["dateCreated"]}\n\tStatus: {upgrade["status"]["message"]}\n\tAvailable actions: {upgrade["status"]["availableActions"]}\n')
userinput = input("Enter the upgrade ID to work with or press Enter for new upgrade: ").rstrip()
self.upgrade_id = userinput
else:
logmsg.info("No running upgrades detected")
def check_upgrade(self, repo):
""" Check Upgrade log
"""
# prevent the log from filling up with debug messages in the while loop
logging.getLogger("urllib3").setLevel(logging.WARNING)
percent_complete = 0
status_message = "checkupgrade"
url = f'{repo.base_url}/storage/1/upgrades/{self.upgrade_id}'
logmsg.info("\nWatch upgrade progress. CTRL-C to exit.")
try:
while percent_complete != 100:
GetToken(repo)
json_return = PDApi.send_get_return_json(repo, url, 'no')
if json_return is not None:
if json_return["state"] == 'initializing':
logmsg.info("Upgrade is initializing. Waiting 15 seconds to start")
time.sleep(15)
elif json_return["state"] == 'error':
logmsg.info(f'\n{json_return["state"]}: {json_return["status"]["message"]}\nSee /var/log/mnode-support-util.log for details')
logmsg.debug(json_return)
if "failedHealthChecks" in json_return.keys():
for fail in json_return["status"]["failedHealthChecks"]:
logmsg.info(f'\tPassed: {fail["passed"]}\t{fail["description"]}')
exit(1)
elif status_message not in json_return["status"]["message"]:
status_message = json_return["status"]["message"]
logmsg.info(f'\nUpgrade status: {json_return["status"]["message"]}\nUpgrade Percentage: {str(json_return["status"]["percent"])}\nUpgrade step: {json_return["status"]["step"]}')
percent_complete = json_return["status"]["percent"]
if "availableActions" in json_return.keys():
for action in json_return["status"]["availableActions"]:
logmsg.info(f'Available action: {action}')
if "nodeDetails" in json_return.keys():
for node in json_return["status"]["nodeDetails"]:
logmsg.info(node)
if "failedHealthChecks" in json_return.keys():
for check in json_return["status"]["failedHealthChecks"]:
logmsg.info(check)
except KeyboardInterrupt:
logmsg.info("Received Ctrl-C")
# Set logging back to debug
logging.getLogger("urllib3").setLevel(logging.DEBUG)
def upgrade_action(self, repo, action):
""" Pause , resume, abort Upgrade
"""
logmsg.info(f'{action} upgrade {self.upgrade_id}')
url = f'{repo.base_url}/storage/1/upgrades/{self.upgrade_id}'
payload = { "config": {},"action":action }
if action == 'resume':
userinput = str.lower(input("Do you have any config options to add? (y/n) ").rstrip())
if userinput == 'y':
configjson = self._config_options()
payload = { "config": configjson, "action":action }
else:
pass
logmsg.debug(f'Sending: PUT {url} {json.dumps(payload)}')
json_return = PDApi.send_put_return_json(repo, url, payload)
if 'state' in json_return:
logmsg.info(f'Upgrade state: {json_return["state"]}')
elif 'detail' in json_return:
logmsg.info(json_return['detail'])
else:
logmsg.info("Unknown return: See /var/log/mnode-support-util.log for details")
logmsg.debug(json.dumps(json_return))
exit(1)