Skip to content

Commit

Permalink
New branch Ansible. Added psi_ansible which can be run at the command…
Browse files Browse the repository at this point in the history
… line specifying a playbook.

Email stats can be configured and sent

--HG--
branch : ansible
  • Loading branch information
mfallone committed Sep 16, 2014
1 parent 381ab44 commit 1cdfe2a
Show file tree
Hide file tree
Showing 3 changed files with 370 additions and 0 deletions.
Empty file added Automation/ansible/.empty
Empty file.
271 changes: 271 additions & 0 deletions Automation/psi_ansible.py
@@ -0,0 +1,271 @@
#!/usr/bin/python

import optparse
import ansible.runner
import ansible.playbook
import os
import sys
import datetime
import psi_ops
import pynliner

import psi_ops_config

PSI_OPS_DB_FILENAME = os.path.join(os.path.abspath('.'), 'psi_ops.dat')

from mako.template import Template
from mako.lookup import TemplateLookup
from mako import exceptions

# Using the FeedbackDecryptor's mail capabilities
sys.path.append(os.path.abspath(os.path.join('..', 'EmailResponder')))
sys.path.append(os.path.abspath(os.path.join('..', 'EmailResponder', 'FeedbackDecryptor')))
import sender
from config import config

def prepare_linode_base_host(host):
ansible_base_linode = create_host(host_name='linode_base_image',
host_vars={'ansible_ssh_host': host.base_ip_address,
'ansible_ssh_user': 'root',
'ansible_ssh_pass': host.base_root_password,
'ansible_ssh_port': host.base_ssh_port,
})
return ansible_base_linode

def create_host(host_name=None, host_vars=dict()):
"""
Create a new host object and return it.
host_name: String containing IP/name of server
host_vars: Variables that are set against the host.
"""
try:
# Create a new host entry and set variables
if isinstance(host_name, basestring):
host = ansible.inventory.host.Host(host_name)

for k,v in host_vars.iteritems():
host.set_variable(k,v)

except Exception as e:
print type(e), str(e)
raise e

return host

def add_hosts_to_group(hosts, group):
"""
Add a single or list of Ansible host objects to an Ansible group
hosts = ansible.inventory.Host
group = ansible.inventory.group.Group
"""
try:
if type(hosts) is ansible.inventory.Host:
# probably means we only have one host
group.add_host(hosts)
elif isinstance(hosts, list):
for host in hosts:
group.add_host(host)

except Exception as e:
print type(e), str(e)
raise e

def run_against_inventory(inv=ansible.inventory.Inventory([]), mod_name='ping', mod_args='', pattern='*', forks=10):
"""
Run a single task against an Inventory.
inv : Ansible Inventory object
mod_name : module name
mod_args : extra arguments for the module
pattern : hosts or groups to match against
forks : number of forks for the runner to create (default = 10)
"""
try:
# create a runnable task and execute
runner = ansible.runner.Runner(
module_name=mod_name,
module_args=mod_args,
pattern=pattern,
forks=forks,
inventory=inv,
)

return runner.run()

except Exception as e:
raise e

def organize_hosts_by_provider(hosts_list):
"""
Takes a list of psinet hosts and organizes into provider dictionary objects.
hosts_list : list of psinet hosts
"""
hosts_dict = dict()
try:
all_hosts = hosts_list
for host in all_hosts:
if host.provider not in hosts_dict.keys():
hosts_dict[host.provider] = list()

hosts_dict[host.provider].append(host)

except Exception as e:
raise e

return hosts_dict

def populate_ansible_hosts(hosts=list()):
"""
Maps a list of psinet hosts into Ansible Hosts
hosts : list of psinet hosts
"""
ansible_hosts = list()
try:
for host in hosts:
ansible_hosts.append(create_host(host_name=host.id,
host_vars={'ansible_ssh_host': host.ip_address,
'ansible_ssh_user': host.ssh_username,
'ansible_ssh_pass': host.ssh_password,
'ansible_ssh_port': host.ssh_port,
}))

except Exception as e:
raise e

return ansible_hosts

def run_playbook(playbook_file, inventory, verbose=psi_ops_config.ANSIBLE_VERBOSE_LEVEL, email_stats=True):
"""
Runs a playbook file and returns the result
playbook_file : Playbook file to open and run (String)
inventory : Ansible inventory to run playbook against
verbose : Output verbosity
"""
try:
start_time = datetime.datetime.now()
playbook_callbacks = ansible.callbacks.PlaybookCallbacks(verbose=verbose)
stats = ansible.callbacks.AggregateStats()
runner_callbacks = ansible.callbacks.PlaybookRunnerCallbacks(stats, verbose=verbose)

playbook = ansible.playbook.PlayBook(playbook=playbook_file,
callbacks=playbook_callbacks, runner_callbacks=runner_callbacks,
stats=stats, inventory=inventory)

res = playbook.run()
end_time = datetime.datetime.now()
print "Run completed at: %s\nTotal run time: %s" % (str(end_time), str(end_time-start_time))

if email_stats == True:
# stats.dark : (dict) number of hosts that could not be contacted
# stats.failures : (dict) number of hosts that failed to complete the tasks
record = (str(start_time), str(end_time), playbook_file, stats.processed, stats.dark, stats.failures, stats.changed, stats.skipped, res)
send_mail(record)

except Exception as e:
raise e

def send_mail(record, subject='PSI ANSIBLE'):
template_filename = 'psi_mail_ansible_stats.mako'
template_lookup = TemplateLookup(directories=[os.path.dirname(os.path.abspath('__file__'))])
template = Template(filename=template_filename, default_filters=['unicode', 'h'], lookup=template_lookup)

try:
rendered = template.render(data=record)
except:
raise Exception(exceptions.text_error_template().render())

# CSS in email HTML must be inline
rendered = pynliner.fromString(rendered)

sender.send(config['emailRecipients'], config['emailUsername'], subject, repr(record), rendered)

def refresh_base_images():
try:
psinet = psi_ops.PsiphonNetwork.load_from_file(PSI_OPS_DB_FILENAME)

except Exception as e:
raise

def main(infile=None, send_mail_stats=False):
try:
psinet = psi_ops.PsiphonNetwork.load_from_file(PSI_OPS_DB_FILENAME)
psinet_hosts_list = psinet.get_hosts()

inv = ansible.inventory.Inventory([])

#Run against subset
if psi_ops_config.RUN_AGAINST_SUBSET == True:
print 'Running Playbook against subset'
subset_hosts_list = psi_ops_config.ANSIBLE_TEST_DIGITALOCEAN + psi_ops_config.ANSIBLE_TEST_LINODES + psi_ops_config.ANSIBLE_TEST_FASTHOSTS
psinet_hosts_list = [h for h in psinet_hosts_list if h.id in subset_hosts_list]

psinet_hosts_dict = organize_hosts_by_provider(psinet_hosts_list)

for provider in psinet_hosts_dict:
group = ansible.inventory.Group(provider)
ansible_hosts_list = populate_ansible_hosts(psinet_hosts_dict[provider])
add_hosts_to_group(ansible_hosts_list, group)
inv.add_group(group)

# Add test group if set
if psi_ops_config.ANSIBLE_INCLUDE_TEST_GROUP == True:
print "Creating Test Group"
test_hosts_list = list()
for h in psinet_hosts_list:
if h.id in psi_ops_config.ANSIBLE_TEST_HOSTS:
test_hosts_list.append(h)

ansible_hosts_list = populate_ansible_hosts(test_hosts_list)
group = ansible.inventory.Group(psi_ops_config.ANSIBLE_TEST_GROUP)
add_hosts_to_group(ansible_hosts_list, group)
inv.add_group(group)

# Add linode base image group
if psi_ops_config.ANSIBLE_INCLUDE_BASE_IMAGE == True:
print "Creating Linode Base Image Group"
linode_base_host = prepare_linode_base_host(psinet._PsiphonNetwork__linode_account)
group = ansible.inventory.Group('linode_base_image')
add_hosts_to_group(linode_base_host, group)
inv.add_group(group)

if not infile:
raise "Must specify input file"

playbook_file = infile
res = run_playbook(playbook_file, inv, send_mail_stats)
print res

except Exception as e:
raise type(e), str(e)


if __name__ == "__main__":
parser = optparse.OptionParser('usage: %prog [options]')
parser.add_option("-i", "--infile", help="Specify ansible playbook file")
parser.add_option("-t", "--test_servers", action="store_true", help="Runs playbook against test systems")
parser.add_option("-s", "--subset", action="store_true", help="Run against a subset of servers")
parser.add_option("-b", "--base_image", action="store_true", help="Forces base image to be included")
parser.add_option("-r", "--refresh_base_images", action="store_true", help="Updates base images for linode and digitalocean")
parser.add_option("-m", "--send_mail", action="store_true", help="Send email after playbook is run")


infile=None
send_mail_stats = False

(options, _) = parser.parse_args()
if options.infile:
infile = options.infile
print infile
if options.send_mail:
send_mail_stats=True
if options.test_servers:
psi_ops_config.ANSIBLE_INCLUDE_TEST_GROUP = True
if options.subset:
psi_ops_config.RUN_AGAINST_SUBSET = True
if options.base_image:
psi_ops_config.ANSIBLE_INCLUDE_BASE_IMAGE = True
if options.refresh_base_images:
refresh_base_images()

main(infile=infile, send_mail_stats=send_mail_stats)


99 changes: 99 additions & 0 deletions Automation/psi_mail_ansible_stats.mako
@@ -0,0 +1,99 @@
## Copyright (c) 2014, Psiphon Inc.
## All rights reserved.
##
## This program is free software: you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation, either version 3 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program. If not, see <http://www.gnu.org/licenses/>.

<h1>Psiphon 3 Ansible Stats</h1>
<%
start_time, end_time, playbook_file, hosts_processed, hosts_dark, hosts_failed, hosts_changed, hosts_skipped, hosts_summary = data
import datetime
import operator
elapsed_time = datetime.datetime.strptime(end_time, "%Y-%m-%d %H:%M:%S.%f") - datetime.datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S.%f")
count_processed = len(hosts_processed)
count_unreachable = len(hosts_dark)
count_failed = len(hosts_failed)
count_changed = len(hosts_changed)
count_skipped = len(hosts_skipped)
%>

<h2>Playbook: ${playbook_file}</h2>

<h3>Host Stats</h3>
<ul>
<li>Unreachable: ${count_unreachable}</li>
<li>Processed: ${count_processed} </li>
<li>Failed: ${count_failed}</li>
<li>Changed: ${count_changed}</li>
<li>Skipped: ${count_skipped}</li>
</ul>

<hr>

% if count_unreachable > 0:
<h3>Unreachable Hosts</h3>
<tbody>
% for c in hosts_dark:
<tr>${c}</tr>
% endfor
</tbody>
% endif

% if count_failed > 0:
<h3>Failed Hosts</h3>
<tbody>
% for c in hosts_failed:
<tr>${c}</tr>
% endfor
</tbody>
% endif

% if count_processed > 0:
<h3>Procssed Hosts</h3>
<tbody>
% for c in hosts_processed:
<tr>${c}</tr>
% endfor
</tbody>
% endif

% if count_skipped > 0:
<h3>Skipped Hosts</h3>
<tbody>
% for c in hosts_skipped:
<tr>${c}</tr>
% endfor
</tbody>
% endif

% if count_changed > 0:
<h3>Changed Hosts</h3>
<tbody>
% for c in hosts_changed:
<tr>${c}</tr>
% endfor
</tbody>
% endif

<hr>
<p>
<sub>
Start Time: ${start_time}
End Time: ${end_time}
Elapsed: ${elapsed_time}
</sub>
</p>

0 comments on commit 1cdfe2a

Please sign in to comment.