Skip to content
Permalink
Browse files

fix: temporary fix for foreman concurrent puts and posts

removed use of semaphores on gets and reduced it to 1 for puts and
posts.
this should be in place until [1] is resolved.
new semaphore for ipmi plus exceptions refactoring.
removed instack generation from M&R in lieue of cron.

[1] https://projects.theforeman.org/issues/27953

Change-Id: Ifcc56ce29a05682f2d48c39a944da9f62602e026
  • Loading branch information...
grafuls committed Oct 2, 2019
1 parent 71abc29 commit 1696676934298a176af3d084a757f257e2613825
Showing with 49 additions and 41 deletions.
  1. +4 −2 bin/quads-cli
  2. +39 −34 quads/tools/foreman.py
  3. +6 −5 quads/tools/move_and_rebuild_hosts.py
@@ -358,7 +358,10 @@ def main(_args):
for result in js['result']:
_clouds[result['new']].append(result)

semaphore = asyncio.Semaphore(6)
# TODO:
# raise the number of semaphores after this is resolved
# https://projects.theforeman.org/issues/27953#change-127120
semaphore = asyncio.Semaphore(1)
for _cloud, results in _clouds.items():
provisioned = True
tasks = []
@@ -415,7 +418,6 @@ def main(_args):
if provisioned:
_new_cloud_obj = Cloud.objects(name=_cloud).first()
_new_cloud_obj.update(provisioned=True)
make_instackenv_json.main()

_old_cloud_obj = Cloud.objects(name=results[0]["current"]).first()
_old_cloud_obj.update(provisioned=False, validated=False, vlan=None, wipe=True)
@@ -34,18 +34,18 @@ def __exit__(self):
async def get(self, endpoint):
logger.debug("GET: %s" % endpoint)
try:
async with self.semaphore:
async with aiohttp.ClientSession(
loop=self.loop
) as session:
async with session.get(
self.url + endpoint,
auth=BasicAuth(self.username, self.password),
verify_ssl=False,
) as response:
result = await response.json(content_type="application/json")
except Exception:
logger.exception("There was something wrong with your request.")
async with aiohttp.ClientSession(
loop=self.loop
) as session:
async with session.get(
self.url + endpoint,
auth=BasicAuth(self.username, self.password),
verify_ssl=False,
) as response:
result = await response.json(content_type="application/json")
except Exception as ex:
logger.debug(ex)
logger.error("There was something wrong with your request.")
return {}
return result

@@ -83,8 +83,9 @@ def __exit__(self):
verify_ssl=False,
) as response:
await response.json(content_type="application/json")
except Exception:
logger.exception("There was something wrong with your request.")
except Exception as ex:
logger.debug(ex)
logger.error("There was something wrong with your request.")
return False
if response.status in [200, 204]:
logger.info("Host parameter updated successfully.")
@@ -107,8 +108,9 @@ def __exit__(self):
verify_ssl=False,
) as response:
await response.json(content_type="application/json")
except Exception:
logger.exception("There was something wrong with your request.")
except Exception as ex:
logger.debug(ex)
logger.error("There was something wrong with your request.")
return False
if response.status in [200, 201, 204]:
logger.info("Host parameter updated successfully.")
@@ -132,8 +134,9 @@ def __exit__(self):
verify_ssl=False,
) as response:
await response.json(content_type="application/json")
except Exception:
logger.exception("There was something wrong with your request.")
except Exception as ex:
logger.debug(ex)
logger.error("There was something wrong with your request.")
return False
if response.status in [200, 204]:
logger.info("User password updated successfully.")
@@ -165,8 +168,9 @@ def __exit__(self):
verify_ssl=False,
) as response:
await response.json(content_type="application/json")
except Exception:
logger.exception("There was something wrong with your request.")
except Exception as ex:
logger.debug(ex)
logger.error("There was something wrong with your request.")
return False
if response.status in [200, 204]:
logger.info("Foreman element updated successfully.")
@@ -236,18 +240,18 @@ def __exit__(self):
endpoint = "/status"
logger.debug("GET: %s" % endpoint)
try:
async with self.semaphore:
async with aiohttp.ClientSession(
loop=self.loop
) as session:
async with session.get(
self.url + endpoint,
auth=BasicAuth(self.username, self.password),
verify_ssl=False,
) as response:
await response.json(content_type="application/json")
except Exception:
logger.exception("There was something wrong with your request.")
async with aiohttp.ClientSession(
loop=self.loop
) as session:
async with session.get(
self.url + endpoint,
auth=BasicAuth(self.username, self.password),
verify_ssl=False,
) as response:
await response.json(content_type="application/json")
except Exception as ex:
logger.debug(ex)
logger.error("There was something wrong with your request.")
return False
if response.status == 200:
return True
@@ -359,8 +363,9 @@ def __exit__(self):
verify_ssl=False,
) as response:
await response.json(content_type="application/json")
except Exception:
logger.exception("There was something wrong with your request.")
except Exception as ex:
logger.debug(ex)
logger.error("There was something wrong with your request.")
success = False
continue
if response.status != 200:
@@ -15,7 +15,7 @@
from quads.tools.ssh_helper import SSHHelper

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")


def switch_config(host, old_cloud, new_cloud):
@@ -157,12 +157,13 @@ def switch_config(host, old_cloud, new_cloud):
str(conf["ipmi_cloud_username_id"]), ipmi_new_pass
]

await execute_ipmi(host, arguments=ipmi_set_pass, semaphore=semaphore)
new_semaphore = asyncio.Semaphore(20)
await execute_ipmi(host, arguments=ipmi_set_pass, semaphore=new_semaphore)

ipmi_set_operator = [
"user", "priv", str(conf["ipmi_cloud_username_id"]), "0x4"
]
await execute_ipmi(host, arguments=ipmi_set_operator, semaphore=semaphore)
await execute_ipmi(host, arguments=ipmi_set_operator, semaphore=new_semaphore)

if rebuild and _new_cloud_obj.name != _host_obj.default_cloud.name:
if "pdu_management" in conf and conf["pdu_management"]:
@@ -174,7 +175,7 @@ def switch_config(host, old_cloud, new_cloud):
"chassis", "bootdev", "pxe",
"options", "=", "persistent"
]
await execute_ipmi(host, arguments=ipmi_pxe_persistent, semaphore=semaphore)
await execute_ipmi(host, arguments=ipmi_pxe_persistent, semaphore=new_semaphore)

if is_supported(host):
try:
@@ -231,7 +232,7 @@ def switch_config(host, old_cloud, new_cloud):
else:
if is_supermicro(host):
try:
await ipmi_reset(host, semaphore)
await ipmi_reset(host, new_semaphore)
except Exception as ex:
logger.debug(ex)
logger.error(f"There was something wrong resetting IPMI on {host}.")

0 comments on commit 1696676

Please sign in to comment.
You can’t perform that action at this time.