Skip to content
This repository has been archived by the owner on Jun 29, 2022. It is now read-only.

Commit

Permalink
Process facts per group to minimize heap memory needed. Closes #561.
Browse files Browse the repository at this point in the history
  • Loading branch information
@chambridge
chambridge committed Dec 19, 2017
1 parent d75dd02 commit c47516b
Showing 1 changed file with 115 additions and 94 deletions.
209 changes: 115 additions & 94 deletions rho/inventory_scan.py
View file
Expand Up @@ -116,6 +116,117 @@ def hosts_by_group(yml_dict):
for group in yml_dict.keys()}


def process_host_vars(facts_to_collect, vars_by_host):
"""Process Ansible output into output facts.
:param facts_to_collect: list of facts to collect.
:param vars_by_host: dictionary (host: facts dictionary)
:returns: list of per host fact dicitionaries
"""
facts_out = []
for _, host_vars in utilities.iteritems(vars_by_host):
this_host = {}

this_host.update(host_vars.get('connection', {}))
this_host.update(host_vars.get('cpu', {}))
this_host.update(host_vars.get('date', {}))
this_host.update(host_vars.get('dmi', {}))
this_host.update(host_vars.get('etc_release', {}))
this_host.update(host_vars.get('file_contents', {}))
this_host.update(host_vars.get('redhat_packages', {}))
this_host.update(host_vars.get('redhat_release', {}))
this_host.update(host_vars.get('subman', {}))
this_host.update(host_vars.get('uname', {}))
this_host.update(host_vars.get('virt', {}))
this_host.update(host_vars.get('virt_what', {}))

this_host.update(
postprocessing.process_jboss_versions(facts_to_collect, host_vars))
this_host.update(
postprocessing.process_addon_versions(facts_to_collect, host_vars))
this_host.update(
postprocessing.process_id_u_jboss(facts_to_collect, host_vars))
this_host.update(
postprocessing.process_jboss_eap_common_files(
facts_to_collect, host_vars))
this_host.update(
postprocessing.process_jboss_eap_processes(
facts_to_collect, host_vars))
this_host.update(
postprocessing.process_jboss_eap_packages(
facts_to_collect, host_vars))
this_host.update(
postprocessing.process_jboss_eap_locate(
facts_to_collect, host_vars))
this_host.update(
postprocessing.process_jboss_eap_init_files(
facts_to_collect, host_vars))
this_host.update(
postprocessing.process_jboss_eap_home(facts_to_collect, host_vars))
this_host.update(
postprocessing.process_fuse_on_eap(facts_to_collect, host_vars))
this_host.update(
postprocessing.process_karaf_home(facts_to_collect, host_vars))
this_host.update(
postprocessing.process_fuse_init_files(
facts_to_collect, host_vars))
this_host.update(
postprocessing.process_brms_output(facts_to_collect, host_vars))
this_host.update(postprocessing.process_find_jboss_modules_jar(
facts_to_collect, host_vars))
this_host.update(postprocessing.process_find_karaf_jar(
facts_to_collect, host_vars))

postprocessing.handle_systemid(facts_to_collect, this_host)
postprocessing.handle_redhat_packages(facts_to_collect, this_host)
postprocessing.escape_characters(this_host)
this_host.update(postprocessing.generate_eap_summary(
facts_to_collect, this_host))
this_host.update(postprocessing.generate_fuse_summary(
facts_to_collect, this_host))

# After all of the facts have been generated, remove -mr facts
# which were for machine use only.
keys = list(this_host.keys())
for key in keys:
if key[-3:] == '-mr':
del this_host[key]

facts_out.append(this_host)
return facts_out


def write_fact_report(facts_to_collect, facts_out, report_path):
"""Write fact report.
:param facts_to_collect: The set of facts to output in the report
:param facts_out: list of per host fact dicitionaries
:param report_path: The path to output the report
"""
normalized_path = os.path.normpath(report_path)
with open(normalized_path, 'w') as write_file:
# Construct the CSV writer
writer = csv.DictWriter(
write_file, sorted(facts_to_collect), delimiter=',',
extrasaction='ignore')

# Write a CSV header if necessary
file_size = os.path.getsize(normalized_path)
if file_size == 0:
# handle Python 2.6 not having writeheader method
if sys.version_info[0] == 2 and sys.version_info[1] <= 6:
headers = {}
for fields in writer.fieldnames:
headers[fields] = fields
writer.writerow(headers)
else:
writer.writeheader()

# Write the data
for data in facts_out:
writer.writerow(data)


# pylint: disable=too-many-arguments, too-many-statements, too-many-branches
def inventory_scan(hosts_yml_path, facts_to_collect, report_path,
vault_pass, base_name, forks=None,
Expand Down Expand Up @@ -162,6 +273,7 @@ def inventory_scan(hosts_yml_path, facts_to_collect, report_path,
my_env["ANSIBLE_NOCOLOR"] = "True"

vars_by_host = {}
facts_out = []

for group in host_groups.keys():
variables_path = variables_prefix + group
Expand Down Expand Up @@ -218,6 +330,8 @@ def inventory_scan(hosts_yml_path, facts_to_collect, report_path,
print('Completed scanning %d systems.\n' %
(len(vars_by_host.keys())))
os.remove(variables_path)
group_facts = process_host_vars(facts_to_collect, vars_by_host)
facts_out += group_facts
else:
utilities.log.error('Error collecting data for group %s.'
'output file %s not found.',
Expand All @@ -229,97 +343,4 @@ def inventory_scan(hosts_yml_path, facts_to_collect, report_path,
"Please review the output to resolve the given issues"))
sys.exit(1)

# Postprocess Ansible output
facts_out = []
for _, host_vars in utilities.iteritems(vars_by_host):
this_host = {}

this_host.update(host_vars.get('connection', {}))
this_host.update(host_vars.get('cpu', {}))
this_host.update(host_vars.get('date', {}))
this_host.update(host_vars.get('dmi', {}))
this_host.update(host_vars.get('etc_release', {}))
this_host.update(host_vars.get('file_contents', {}))
this_host.update(host_vars.get('redhat_packages', {}))
this_host.update(host_vars.get('redhat_release', {}))
this_host.update(host_vars.get('subman', {}))
this_host.update(host_vars.get('uname', {}))
this_host.update(host_vars.get('virt', {}))
this_host.update(host_vars.get('virt_what', {}))

this_host.update(
postprocessing.process_jboss_versions(facts_to_collect, host_vars))
this_host.update(
postprocessing.process_addon_versions(facts_to_collect, host_vars))
this_host.update(
postprocessing.process_id_u_jboss(facts_to_collect, host_vars))
this_host.update(
postprocessing.process_jboss_eap_common_files(
facts_to_collect, host_vars))
this_host.update(
postprocessing.process_jboss_eap_processes(
facts_to_collect, host_vars))
this_host.update(
postprocessing.process_jboss_eap_packages(
facts_to_collect, host_vars))
this_host.update(
postprocessing.process_jboss_eap_locate(
facts_to_collect, host_vars))
this_host.update(
postprocessing.process_jboss_eap_init_files(
facts_to_collect, host_vars))
this_host.update(
postprocessing.process_jboss_eap_home(facts_to_collect, host_vars))
this_host.update(
postprocessing.process_fuse_on_eap(facts_to_collect, host_vars))
this_host.update(
postprocessing.process_karaf_home(facts_to_collect, host_vars))
this_host.update(
postprocessing.process_fuse_init_files(
facts_to_collect, host_vars))
this_host.update(
postprocessing.process_brms_output(facts_to_collect, host_vars))
this_host.update(postprocessing.process_find_jboss_modules_jar(
facts_to_collect, host_vars))
this_host.update(postprocessing.process_find_karaf_jar(
facts_to_collect, host_vars))

postprocessing.handle_systemid(facts_to_collect, this_host)
postprocessing.handle_redhat_packages(facts_to_collect, this_host)
postprocessing.escape_characters(this_host)
this_host.update(postprocessing.generate_eap_summary(
facts_to_collect, this_host))
this_host.update(postprocessing.generate_fuse_summary(
facts_to_collect, this_host))

# After all of the facts have been generated, remove -mr facts
# which were for machine use only.
keys = list(this_host.keys())
for key in keys:
if key[-3:] == '-mr':
del this_host[key]

facts_out.append(this_host)

normalized_path = os.path.normpath(report_path)
with open(normalized_path, 'w') as write_file:
# Construct the CSV writer
writer = csv.DictWriter(
write_file, sorted(facts_to_collect), delimiter=',',
extrasaction='ignore')

# Write a CSV header if necessary
file_size = os.path.getsize(normalized_path)
if file_size == 0:
# handle Python 2.6 not having writeheader method
if sys.version_info[0] == 2 and sys.version_info[1] <= 6:
headers = {}
for fields in writer.fieldnames:
headers[fields] = fields
writer.writerow(headers)
else:
writer.writeheader()

# Write the data
for data in facts_out:
writer.writerow(data)
write_fact_report(facts_to_collect, facts_out, report_path)

0 comments on commit c47516b

Please sign in to comment.