Merge pull request #121 from grycap/vault

Merge Vault

IM/ConfManager.py

@@ -26,7 +26,9 @@
 from StringIO import StringIO
 from multiprocessing import Queue
 
-from IM.ansible.ansible_launcher import AnsibleThread
+from ansible.parsing.vault import VaultEditor
+
+from IM.ansible_utils.ansible_launcher import AnsibleThread
 
 import InfrastructureManager
 from VirtualMachine import VirtualMachine
@@ -325,7 +327,11 @@ def launch_ctxt_agent(self, vm, tasks):
                              os.path.basename(conf_file))
 
                 if vm.configured is None:
-                    (pid, _, _) = ssh.execute("nohup python_ansible " + Config.REMOTE_CONF_DIR + "/" +
+                    vault_export = ""
+                    vault_password = vm.info.systems[0].getValue("vault.password")
+                    if vault_password:
+                        vault_export = "export VAULT_PASS='%s' && " % vault_password
+                    (pid, _, _) = ssh.execute(vault_export + "nohup python_ansible " + Config.REMOTE_CONF_DIR + "/" +
                                               str(self.inf.id) + "/" + "/ctxt_agent.py " +
                                               Config.REMOTE_CONF_DIR + "/" + str(self.inf.id) + "/" +
                                               "/general_info.cfg " + remote_dir + "/" + os.path.basename(conf_file) +
@@ -685,17 +691,24 @@ def generate_playbook(self, vm, ctxt_elem, tmp_dir):
         conf_filename = tmp_dir + "/" + ctxt_elem.configure + \
             "_" + ctxt_elem.system + "_task.yml"
         if not os.path.isfile(conf_filename):
-            configure = self.inf.radl.get_configure_by_name(
-                ctxt_elem.configure)
-            conf_content = self.add_ansible_header(
-                ctxt_elem.system, vm.getOS().lower())
-            conf_content = self.mergeYAML(conf_content, configure.recipes)
+            configure = self.inf.radl.get_configure_by_name(ctxt_elem.configure)
+            conf_content = self.add_ansible_header(ctxt_elem.system, vm.getOS().lower())
+            vault_password = vm.info.systems[0].getValue("vault.password")
+            if vault_password:
+                vault_edit = VaultEditor(vault_password)
+                if configure.recipes.strip().startswith("$ANSIBLE_VAULT"):
+                    recipes = vault_edit.vault.decrypt(configure.recipes.strip())
+                else:
+                    recipes = configure.recipes
+                conf_content = self.mergeYAML(conf_content, recipes)
+                conf_content = vault_edit.vault.encrypt(conf_content)
+            else:
+                conf_content = self.mergeYAML(conf_content, configure.recipes)
 
             conf_out = open(conf_filename, 'w')
-            conf_out.write(conf_content + "\n\n")
+            conf_out.write(conf_content)
             conf_out.close()
-            recipe_files.append(ctxt_elem.configure + "_" +
-                                ctxt_elem.system + "_task.yml")
+            recipe_files.append(ctxt_elem.configure + "_" + ctxt_elem.system + "_task.yml")
 
             # create the "all" to enable this playbook to see the facts of all
             # the nodes
@@ -759,11 +772,11 @@ def configure_master(self):
                     files.append(
                         (Config.IM_PATH + "/SSH.py", remote_dir + "/SSH.py"))
                     files.append(
-                        (Config.IM_PATH + "/ansible/ansible_callbacks.py", remote_dir + "/ansible_callbacks.py"))
-                    files.append((Config.IM_PATH + "/ansible/ansible_executor_v2.py",
+                        (Config.IM_PATH + "/ansible_utils/ansible_callbacks.py", remote_dir + "/ansible_callbacks.py"))
+                    files.append((Config.IM_PATH + "/ansible_utils/ansible_executor_v2.py",
                                   remote_dir + "/ansible_executor_v2.py"))
                     files.append(
-                        (Config.IM_PATH + "/ansible/ansible_launcher.py", remote_dir + "/ansible_launcher.py"))
+                        (Config.IM_PATH + "/ansible_utils/ansible_launcher.py", remote_dir + "/ansible_launcher.py"))
                     files.append((Config.CONTEXTUALIZATION_DIR +
                                   "/ctxt_agent.py", remote_dir + "/ctxt_agent.py"))
 
@@ -1356,31 +1369,25 @@ def configure_ansible(self, ssh, tmp_dir):
                         recipe_out.write(
                             '    - name: Create YAML file to install the %s role with ansible-galaxy\n' % rolename)
                         recipe_out.write('      copy:\n')
-                        recipe_out.write(
-                            '        dest: "/tmp/%s.yml"\n' % rolename)
-                        recipe_out.write(
-                            '        content: "- src: %s\\n  name: %s"\n' % (url, rolename))
+                        recipe_out.write('        dest: "/tmp/%s.yml"\n' % rolename)
+                        recipe_out.write('        content: "- src: %s\\n  name: %s"\n' % (url, rolename))
                         url = "-r /tmp/%s.yml" % rolename
                     else:
                         url = rolename = galaxy_name
 
                     if galaxy_name.startswith("git"):
                         recipe_out.write("    - yum: name=git\n")
-                        recipe_out.write(
-                            '      when: ansible_os_family == "RedHat"\n')
+                        recipe_out.write('      when: ansible_os_family == "RedHat"\n')
                         recipe_out.write("    - apt: name=git\n")
-                        recipe_out.write(
-                            '      when: ansible_os_family == "Debian"\n')
-
+                        recipe_out.write('      when: ansible_os_family == "Debian"\n')
                     recipe_out.write(
                         "    - name: Install the %s role with ansible-galaxy\n" % rolename)
                     recipe_out.write(
                         "      command: ansible-galaxy -f install %s\n" % url)
 
                     recipe_out.close()
 
-            self.inf.add_cont_msg(
-                "Performing preliminary steps to configure Ansible.")
+            self.inf.add_cont_msg("Performing preliminary steps to configure Ansible.")
             # TODO: check to do it with ansible
             ConfManager.logger.debug("Inf ID: " + str(self.inf.id) +
                                      ": Check if python-simplejson is installed in REL 5 systems")

IM/ansible/ansible_launcher.py → IM/ansible_utils/ansible_launcher.py

@@ -70,7 +70,7 @@ class AnsibleThread(Process):
     """
 
     def __init__(self, result, output, playbook_file, host=None, threads=1, pk_file=None, passwd=None, retries=1,
-                 inventory_file=None, user=None, extra_vars={}):
+                 inventory_file=None, user=None, vault_pass=None, extra_vars={}):
         super(AnsibleThread, self).__init__()
         self.playbook_file = playbook_file
         self.host = host
@@ -83,6 +83,7 @@ def __init__(self, result, output, playbook_file, host=None, threads=1, pk_file=
         self.extra_vars = extra_vars
         self.output = output
         self.result = result
+        self.vault_pass = vault_pass
 
     def teminate(self):
         try:
@@ -183,6 +184,9 @@ def launch_playbook_v2(self):
         # In version 2.0.1 it must be fixed
         ansible.inventory.HOSTS_PATTERNS_CACHE = {}
 
+        if self.vault_pass:
+            loader.set_vault_password(self.vault_pass)
+
         inventory = ansible.inventory.Inventory(
             loader=loader, variable_manager=variable_manager, host_list=options.inventory)
         variable_manager.set_inventory(inventory)

contextualization/ctxt_agent.py

@@ -185,7 +185,8 @@ def wait_thread(thread_data, output=None):
     return (return_code == 0, hosts_with_errors)
 
 
-def LaunchAnsiblePlaybook(output, playbook_file, vm, threads, inventory_file, pk_file, retries, change_pass_ok):
+def LaunchAnsiblePlaybook(output, playbook_file, vm, threads, inventory_file, pk_file,
+                          retries, change_pass_ok, vault_pass):
     logger.debug('Call Ansible')
 
     extra_vars = {'IM_HOST': vm['ip'] + "_" + str(vm['remote_port'])}
@@ -221,7 +222,7 @@ def LaunchAnsiblePlaybook(output, playbook_file, vm, threads, inventory_file, pk
 
     result = Queue()
     t = AnsibleThread(result, output, playbook_file, None, threads, gen_pk_file,
-                      passwd, retries, inventory_file, user, extra_vars)
+                      passwd, retries, inventory_file, user, vault_pass, extra_vars)
     t.start()
     return (t, result)
 
@@ -368,6 +369,10 @@ def replace_vm_ip(vm_data):
 
 
 def contextualize_vm(general_conf_data, vm_conf_data):
+    vault_pass = None
+    if 'VAULT_PASS' in os.environ:
+        vault_pass = os.environ['VAULT_PASS']
+
     res_data = {}
     logger.info('Generate and copy the ssh key')
 
@@ -458,8 +463,8 @@ def contextualize_vm(general_conf_data, vm_conf_data):
 
                 if ctxt_vm['os'] != "windows":
                     # this step is not needed in windows systems
-                    ansible_thread = LaunchAnsiblePlaybook(
-                        logger, playbook, ctxt_vm, 2, inventory_file, pk_file, INTERNAL_PLAYBOOK_RETRIES, change_creds)
+                    ansible_thread = LaunchAnsiblePlaybook(logger, playbook, ctxt_vm, 2, inventory_file, pk_file,
+                                                           INTERNAL_PLAYBOOK_RETRIES, change_creds, vault_pass)
             else:
                 # In some strange cases the pk_file disappears. So test it and
                 # remake basic recipe
@@ -478,13 +483,14 @@ def contextualize_vm(general_conf_data, vm_conf_data):
                         basic_playbook = general_conf_data[
                             'conf_dir'] + "/basic_task_all.yml"
                         output_basic = StringIO()
-                        ansible_thread = LaunchAnsiblePlaybook(output_basic, basic_playbook, ctxt_vm, 2,
-                                                               inventory_file, None, INTERNAL_PLAYBOOK_RETRIES, True)
+                        ansible_thread = LaunchAnsiblePlaybook(output_basic, basic_playbook, ctxt_vm, 2, inventory_file,
+                                                               None, INTERNAL_PLAYBOOK_RETRIES, True, vault_pass)
                         (task_ok, _) = wait_thread(ansible_thread)
 
                 # in the other tasks pk_file can be used
                 ansible_thread = LaunchAnsiblePlaybook(logger, playbook, ctxt_vm, 2, inventory_file, PK_FILE,
-                                                       INTERNAL_PLAYBOOK_RETRIES, vm_conf_data['changed_pass'])
+                                                       INTERNAL_PLAYBOOK_RETRIES, vm_conf_data['changed_pass'],
+                                                       vault_pass)
 
             if ansible_thread:
                 (task_ok, _) = wait_thread(ansible_thread)

examples/vault.radl

@@ -0,0 +1,31 @@
+network public (outbound = 'yes')
+network privada ()
+
+system node (
+#cpu.arch='x86_64' and
+cpu.count>=1 and
+memory.size>=1024m and
+net_interface.0.connection = 'public' and
+net_interface.1.connection = 'privada' and
+net_interface.0.dns_name = 'testnode' and
+disk.0.os.name='linux' and
+disk.0.os.flavour='ubuntu' and
+disk.0.os.credentials.new.password = 'N0tan+mala' and
+vault.password = 'grycap01'
+)
+
+configure node (
+@begin
+$ANSIBLE_VAULT;1.1;AES256
+66373337376135613262376333346536323764333138616134663939663935656435366633623839
+6264663032666363363766366365396130316334353231350a343332373433336130346632363161
+31333338626131626533316263643234316331303135633932646137386531383862633933376164
+3233306332326233340a653165626537356264653261393365316338373638353165316366663135
+33383666626634393763643239353639633461363736663161353433383165366438343537313362
+30356632356661386238336535633530353164616462636338353465346633613963626665383730
+336331656333633934636462303063626336
+@end
+)
+
+deploy node 1
+

test/files/play_vault.yaml

@@ -0,0 +1,30 @@
+$ANSIBLE_VAULT;1.1;AES256
+30323065653366626161353361363763346131656637333838383931643638336663666330373937
+6264303437316462363934306235346435306464643532350a306263633530303031313336643635
+62353833643939313233313833653861653934396162366339313731353635303666343562323239
+3762373635363839610a333161396230396363303765626334666239653064643231646431363165
+35636162313264393437346539303731616635366561656333343362653338346130646436623133
+64653437313533663261653332326164323839353334393137343237333639646332653335326232
+31616538653965353631383763613839623931633135376161643233373431323466393137306661
+38396666613531383039323330613333613137626466383630623165393262653562643663393962
+30313030333137306564646565333731623537383666333639383534353665613864353130646231
+36663264353462613732376134386231633338623938666465373137643761333337313838336566
+39383430326332356463396237616532633837383161373535316463346461643939353634626439
+35313238353665343333303164316532303136623562393637396266643837623931346165363431
+65303430666364626537653339383938646462633636333833326531326563373937656637353036
+38373264303664373663623766346233373932343838636534343737303234373438303638633864
+38343363303564313439316366623762626664323037373533376337316431313931663534376430
+30306133323562383063656438333166373861326162646332333830633536626135326661346364
+65616566356639666438333239386165363566323531616138663437336161363235383364323639
+37373762383061633632383262373132656335656364346536643131396431316234646563626264
+39313230323732663834653934663963653034633664643131313132313864383432636132616530
+62623030653530366462363162643965613532356333343637306664666234623839633835393732
+33333337323034666231333131346362356534333330363236303933656531356161366438386334
+39353435666334363639616231643032326462643430323031323462343837663563396465646166
+31363763646561303763326134633637353234393835383966396530373132633839316666666166
+66373435653434323937663764616362316366373633363032353337373238643138383436653737
+31343732666236306364383032373633656633666336376434663363356432663534346636346162
+37383263616534643935633666353935343733626162326630373233333234346461323231646539
+31613133633361656338643930336336623633333038363638663731633366643237306539393233
+39333430613235616636386534653266393532653935663565633230336234323862626361376664
+34396139316531663139393530366266323136616139366334326633613062303364

test/unit/test_ansible.py

@@ -22,7 +22,7 @@
 from StringIO import StringIO
 import time
 
-from IM.ansible.ansible_launcher import AnsibleThread
+from IM.ansible_utils.ansible_launcher import AnsibleThread
 from mock import patch, MagicMock
 
 
@@ -46,5 +46,20 @@ def test_ansible_thread(self):
         self.assertIn("changed=2", output.getvalue())
         print output.getvalue()
 
+    def test_ansible_thread_with_vault(self):
+        result = Queue()
+        tests_path = os.path.dirname(os.path.abspath(__file__))
+        play_file_path = os.path.join(tests_path, "../files/play_vault.yaml")
+        inventory = os.path.join(tests_path, "../files/inventory")
+        ansible_process = AnsibleThread(result, StringIO(), play_file_path, None, 1, None,
+                                        "password", 1, inventory, "username", "ansible")
+        ansible_process.run()
+
+        _, (return_code, _), output = result.get()
+        self.assertEqual(return_code, 0)
+        self.assertIn("failed=0", output.getvalue())
+        self.assertIn("changed=2", output.getvalue())
+        print output.getvalue()
+
 if __name__ == '__main__':
     unittest.main()