Refactor manifest execution

Previously, Packstack created many individual manifest files from
smaller snippets (templates), and executed them following a certain
order. This is sub-optimal, since it forces code duplication and goes
against the Puppet design of running a single manifest.

This patch refactors the manifest execution, following these principles:

- Only 4 templates used, so max 4 Puppet executions on a host: controller
  manifest, network node manifest, compute node manifest, and a firewall
  manifest, which is still generated on the fly for simplicity.

- The previous snippets are now part of the Packstack Puppet module, and
  included as needed by the controller/network/compute manifests. This
  concept is similar to the one used by the puppet-openstack-integration
  project.

- The remaining Python code is left untouched, so we can keep complete
  compatibility with previous answer files.

There are some missing features in the refactor (the most notable are
VMware and Redis HA support), which will be fixed by upcoming patches.

TODO.txt

@@ -0,0 +1,8 @@
+The following items are still pending:
+
+- VMware cluster logic for compute nodes
+- Some logic in compute nodes when host is not in network_hosts
+- Similar logic for compute nodes when using Nova Network
+- Redis HA logic (ceilometer_800.py)
+- Change message for plugins, when "creating manifests"
+- Additional cleanup: remove unused functions in Python code

packstack/plugins/amqp_002.py

@@ -25,7 +25,6 @@
 from packstack.modules.documentation import update_params_usage
 from packstack.modules.ospluginutils import appendManifestFile
 from packstack.modules.ospluginutils import createFirewallResources
-from packstack.modules.ospluginutils import getManifestTemplate
 from packstack.modules.ospluginutils import generate_ssl_cert
 
 # ------------- AMQP Packstack Plugin Initialization --------------
@@ -188,9 +187,6 @@ def create_manifest(config, messages):
         config['CONFIG_AMQP_AUTH_PASSWORD'] = 'guest'
         config['CONFIG_AMQP_AUTH_USER'] = 'guest'
 
-    manifestfile = "%s_amqp.pp" % config['CONFIG_AMQP_HOST']
-    manifestdata = getManifestTemplate('amqp')
-
     if config['CONFIG_IP_VERSION'] == 'ipv6':
         config['CONFIG_AMQP_HOST_URL'] = "[%s]" % config['CONFIG_AMQP_HOST']
     else:
@@ -208,5 +204,6 @@ def create_manifest(config, messages):
         fw_details[key]['proto'] = "tcp"
     config['FIREWALL_AMQP_RULES'] = fw_details
 
-    manifestdata += createFirewallResources('FIREWALL_AMQP_RULES')
+    manifestfile = "%s_firewall.pp" % config['CONFIG_AMQP_HOST']
+    manifestdata = createFirewallResources('FIREWALL_AMQP_RULES')
     appendManifestFile(manifestfile, manifestdata, 'pre')

packstack/plugins/aodh_810.py

@@ -22,10 +22,8 @@
 from packstack.installer import processors
 
 from packstack.modules.documentation import update_params_usage
-from packstack.modules.shortcuts import get_mq
 from packstack.modules.ospluginutils import appendManifestFile
 from packstack.modules.ospluginutils import createFirewallResources
-from packstack.modules.ospluginutils import getManifestTemplate
 from packstack.modules.ospluginutils import generate_ssl_cert
 
 # ------------- Aodh Packstack Plugin Initialization --------------
@@ -77,20 +75,15 @@ def initSequences(controller):
         return
 
     steps = [{'title': 'Adding Aodh manifest entries',
-              'functions': [create_manifest]},
-             {'title': 'Adding Aodh Keystone manifest entries',
-              'functions': [create_keystone_manifest]}]
+              'functions': [create_manifest]}]
     controller.addSequence("Installing OpenStack Aodh", [], [],
                            steps)
 
 
 # -------------------------- step functions --------------------------
 
 def create_manifest(config, messages):
-    manifestfile = "%s_aodh.pp" % config['CONFIG_CONTROLLER_HOST']
-    manifestdata = getManifestTemplate(get_mq(config, "aodh"))
-    manifestdata += getManifestTemplate("aodh")
-    manifestdata += getManifestTemplate("apache_ports")
+    manifestfile = "%s_firewall.pp" % config['CONFIG_CONTROLLER_HOST']
 
     if config['CONFIG_AMQP_ENABLE_SSL'] == 'y':
         ssl_cert_file = config['CONFIG_AODH_SSL_CERT'] = (
@@ -113,11 +106,5 @@ def create_manifest(config, messages):
     fw_details[key]['ports'] = ['8042']
     fw_details[key]['proto'] = "tcp"
     config['FIREWALL_AODH_RULES'] = fw_details
-    manifestdata += createFirewallResources('FIREWALL_AODH_RULES')
+    manifestdata = createFirewallResources('FIREWALL_AODH_RULES')
     appendManifestFile(manifestfile, manifestdata, 'aodh')
-
-
-def create_keystone_manifest(config, messages):
-    manifestfile = "%s_keystone.pp" % config['CONFIG_CONTROLLER_HOST']
-    manifestdata = getManifestTemplate("keystone_aodh")
-    appendManifestFile(manifestfile, manifestdata)

packstack/plugins/ceilometer_800.py

@@ -25,10 +25,8 @@
 from packstack.installer.utils import split_hosts
 
 from packstack.modules.documentation import update_params_usage
-from packstack.modules.shortcuts import get_mq
 from packstack.modules.ospluginutils import appendManifestFile
 from packstack.modules.ospluginutils import createFirewallResources
-from packstack.modules.ospluginutils import getManifestTemplate
 from packstack.modules.ospluginutils import generate_ssl_cert
 
 # ------------- Ceilometer Packstack Plugin Initialization --------------
@@ -258,21 +256,15 @@ def initSequences(controller):
              {'title': 'Adding Redis manifest entries',
               'functions': [create_redis_manifest]},
              {'title': 'Adding Ceilometer manifest entries',
-              'functions': [create_manifest]},
-             {'title': 'Adding Ceilometer Keystone manifest entries',
-              'functions': [create_keystone_manifest]}]
+              'functions': [create_manifest]}]
     controller.addSequence("Installing OpenStack Ceilometer", [], [],
                            steps)
 
 
 # -------------------------- step functions --------------------------
 
 def create_manifest(config, messages):
-    manifestfile = "%s_ceilometer.pp" % config['CONFIG_CONTROLLER_HOST']
-    manifestdata = getManifestTemplate(get_mq(config, "ceilometer"))
-    manifestdata += getManifestTemplate("ceilometer")
-    if config['CONFIG_CEILOMETER_SERVICE_NAME'] == 'httpd':
-        manifestdata += getManifestTemplate("apache_ports")
+    manifestfile = "%s_firewall.pp" % config['CONFIG_CONTROLLER_HOST']
 
     if config['CONFIG_CEILOMETER_COORDINATION_BACKEND'] == 'redis':
         # Determine if we need to configure multiple sentinel hosts as
@@ -321,12 +313,8 @@ def create_manifest(config, messages):
     fw_details[key]['ports'] = ['8777']
     fw_details[key]['proto'] = "tcp"
     config['FIREWALL_CEILOMETER_RULES'] = fw_details
-    manifestdata += createFirewallResources('FIREWALL_CEILOMETER_RULES')
+    manifestdata = createFirewallResources('FIREWALL_CEILOMETER_RULES')
 
-    # Add a template that creates a group for nova because the ceilometer
-    # class needs it
-    if config['CONFIG_NOVA_INSTALL'] == 'n':
-        manifestdata += getManifestTemplate("ceilometer_nova_disabled")
     appendManifestFile(manifestfile, manifestdata, 'ceilometer')
 
 
@@ -336,8 +324,8 @@ def create_mongodb_manifest(config, messages):
         config['CONFIG_MONGODB_HOST_URL'] = "[%s]" % host
     else:
         config['CONFIG_MONGODB_HOST_URL'] = host
-    manifestfile = "%s_mongodb.pp" % config['CONFIG_MONGODB_HOST']
-    manifestdata = getManifestTemplate("mongodb")
+
+    manifestfile = "%s_firewall.pp" % config['CONFIG_MONGODB_HOST']
 
     fw_details = dict()
     key = "mongodb_server"
@@ -349,7 +337,7 @@ def create_mongodb_manifest(config, messages):
     fw_details[key]['proto'] = "tcp"
     config['FIREWALL_MONGODB_RULES'] = fw_details
 
-    manifestdata += createFirewallResources('FIREWALL_MONGODB_RULES')
+    manifestdata = createFirewallResources('FIREWALL_MONGODB_RULES')
     appendManifestFile(manifestfile, manifestdata, 'pre')
 
 
@@ -362,56 +350,49 @@ def create_redis_manifest(config, messages):
             config['CONFIG_REDIS_MASTER_HOST_URL'] = redis_master_host
 
         # master
-        manifestfile = "%s_redis.pp" % config['CONFIG_REDIS_MASTER_HOST']
-        manifestdata = getManifestTemplate("redis.pp")
-
+        manifestfile = "%s_firewall.pp" % config['CONFIG_REDIS_MASTER_HOST']
         master_clients = set([config['CONFIG_CONTROLLER_HOST']]).union(
             split_hosts(config['CONFIG_REDIS_SLAVE_HOSTS'])).union(
             split_hosts(config['CONFIG_REDIS_SENTINEL_HOSTS']))
         config['FIREWALL_REDIS_RULES'] = _create_redis_firewall_rules(
             master_clients, config['CONFIG_REDIS_PORT'])
 
-        manifestdata += createFirewallResources('FIREWALL_REDIS_RULES')
+        manifestdata = createFirewallResources('FIREWALL_REDIS_RULES')
         appendManifestFile(manifestfile, manifestdata, 'pre')
 
+        # TODO(jpena): we are disabling redis slave support for now
         # slaves
-        if config['CONFIG_REDIS_HA'] == 'y':
-            for slave in split_hosts(config['CONFIG_REDIS_SLAVE_HOSTS']):
-                config['CONFIG_REDIS_HOST'] = slave
-                manifestfile = "%s_redis_slave.pp" % slave
-                manifestdata = getManifestTemplate("redis_slave.pp")
-
-                slave_clients = set([config['CONFIG_CONTROLLER_HOST']]).union(
-                    split_hosts(config['CONFIG_REDIS_SLAVE_HOSTS'])).union(
-                        split_hosts(config['CONFIG_REDIS_SENTINEL_HOSTS']))
-                config['FIREWALL_REDIS_SLAVE_RULES'] = (
-                    _create_redis_firewall_rules(
-                        slave_clients, config['CONFIG_REDIS_PORT']))
-
-                manifestdata += createFirewallResources(
-                    'FIREWALL_REDIS_SLAVE_RULES')
-                appendManifestFile(manifestfile, manifestdata, 'pre')
+        # if config['CONFIG_REDIS_HA'] == 'y':
+        #    for slave in split_hosts(config['CONFIG_REDIS_SLAVE_HOSTS']):
+        #        config['CONFIG_REDIS_HOST'] = slave
+        #        manifestfile = "%s_redis_slave.pp" % slave
+        #        manifestdata = getManifestTemplate("redis_slave.pp")
+        #
+        #        slave_clients = set([config['CONFIG_CONTROLLER_HOST']]).union(
+        #            split_hosts(config['CONFIG_REDIS_SLAVE_HOSTS'])).union(
+        #                split_hosts(config['CONFIG_REDIS_SENTINEL_HOSTS']))
+        #        config['FIREWALL_REDIS_SLAVE_RULES'] = (
+        #            _create_redis_firewall_rules(
+        #                slave_clients, config['CONFIG_REDIS_PORT']))
+        #
+        #        manifestdata += createFirewallResources(
+        #            'FIREWALL_REDIS_SLAVE_RULES')
+        #        appendManifestFile(manifestfile, manifestdata, 'pre')
 
         # sentinels
-        if config['CONFIG_REDIS_HA'] == 'y':
-            for sentinel in split_hosts(config['CONFIG_REDIS_SENTINEL_HOSTS']):
-                manifestfile = "%s_redis_sentinel.pp" % sentinel
-                manifestdata = getManifestTemplate("redis_sentinel.pp")
-
-                config['FIREWALL_SENTINEL_RULES'] = (
-                    _create_redis_firewall_rules(
-                        split_hosts(config['CONFIG_REDIS_SENTINEL_HOSTS']),
-                        config['CONFIG_REDIS_SENTINEL_PORT']))
-
-                manifestdata += createFirewallResources(
-                    'FIREWALL_SENTINEL_RULES')
-                appendManifestFile(manifestfile, manifestdata, 'pre')
-
-
-def create_keystone_manifest(config, messages):
-    manifestfile = "%s_keystone.pp" % config['CONFIG_CONTROLLER_HOST']
-    manifestdata = getManifestTemplate("keystone_ceilometer")
-    appendManifestFile(manifestfile, manifestdata)
+        # if config['CONFIG_REDIS_HA'] == 'y':
+        #    for sentinel in split_hosts(config['CONFIG_REDIS_SENTINEL_HOSTS']):
+        #        manifestfile = "%s_redis_sentinel.pp" % sentinel
+        #        manifestdata = getManifestTemplate("redis_sentinel.pp")
+        #
+        #        config['FIREWALL_SENTINEL_RULES'] = (
+        #            _create_redis_firewall_rules(
+        #                split_hosts(config['CONFIG_REDIS_SENTINEL_HOSTS']),
+        #                config['CONFIG_REDIS_SENTINEL_PORT']))
+        #
+        #        manifestdata += createFirewallResources(
+        #            'FIREWALL_SENTINEL_RULES')
+        #        appendManifestFile(manifestfile, manifestdata, 'pre')
 
 
 # ------------------------- helper functions -------------------------

packstack/plugins/cinder_250.py

@@ -27,10 +27,8 @@
 from packstack.installer import utils
 
 from packstack.modules.documentation import update_params_usage
-from packstack.modules.shortcuts import get_mq
 from packstack.modules.ospluginutils import appendManifestFile
 from packstack.modules.ospluginutils import createFirewallResources
-from packstack.modules.ospluginutils import getManifestTemplate
 from packstack.modules.ospluginutils import generate_ssl_cert
 
 # ------------------ Cinder Packstack Plugin initialization ------------------
@@ -601,10 +599,7 @@ def initSequences(controller):
         if key in config:
             config[key] = [i.strip() for i in config[key].split(',') if i]
 
-    cinder_steps = [
-        {'title': 'Adding Cinder Keystone manifest entries',
-         'functions': [create_keystone_manifest]}
-    ]
+    cinder_steps = []
 
     if 'lvm' in config['CONFIG_CINDER_BACKEND']:
         cinder_steps.append(
@@ -719,12 +714,6 @@ def check_cinder_vg(config, messages):
     config['CONFIG_CINDER_VOLUMES_SIZE'] = '%sM' % cinders_volume_size
 
 
-def create_keystone_manifest(config, messages):
-    manifestfile = "%s_keystone.pp" % config['CONFIG_CONTROLLER_HOST']
-    manifestdata = getManifestTemplate("keystone_cinder")
-    appendManifestFile(manifestfile, manifestdata)
-
-
 def create_manifest(config, messages):
     if config['CONFIG_AMQP_ENABLE_SSL'] == 'y':
         ssl_host = config['CONFIG_STORAGE_HOST']
@@ -738,17 +727,7 @@ def create_manifest(config, messages):
         generate_ssl_cert(config, ssl_host, service, ssl_key_file,
                           ssl_cert_file)
 
-    manifestdata = getManifestTemplate(get_mq(config, "cinder"))
-    manifestfile = "%s_cinder.pp" % config['CONFIG_STORAGE_HOST']
-    manifestdata += getManifestTemplate("cinder")
-
-    for backend in config['CONFIG_CINDER_BACKEND']:
-        manifestdata += getManifestTemplate('cinder_%s' % backend)
-
-    if config['CONFIG_CEILOMETER_INSTALL'] == 'y':
-        manifestdata += getManifestTemplate('cinder_ceilometer')
-    if config['CONFIG_SWIFT_INSTALL'] == 'y':
-        manifestdata += getManifestTemplate('cinder_backup')
+    manifestfile = "%s_firewall.pp" % config['CONFIG_STORAGE_HOST']
 
     fw_details = dict()
     for host in split_hosts(config['CONFIG_COMPUTE_HOSTS']):
@@ -768,7 +747,7 @@ def create_manifest(config, messages):
         fw_details[key]['proto'] = "tcp"
 
     config['FIREWALL_CINDER_RULES'] = fw_details
-    manifestdata += createFirewallResources('FIREWALL_CINDER_RULES')
+    manifestdata = createFirewallResources('FIREWALL_CINDER_RULES')
 
     # cinder API should be open for everyone
     fw_details = dict()

packstack/plugins/dashboard_500.py

@@ -25,8 +25,6 @@
 from packstack.installer import validators
 
 from packstack.modules.documentation import update_params_usage
-from packstack.modules.ospluginutils import appendManifestFile
-from packstack.modules.ospluginutils import getManifestTemplate
 from packstack.modules.ospluginutils import generate_ssl_cert
 from packstack.modules.ospluginutils import deliver_ssl_file
 
@@ -141,7 +139,6 @@ def initSequences(controller):
 
 def create_manifest(config, messages):
     horizon_host = config['CONFIG_CONTROLLER_HOST']
-    manifestfile = "%s_horizon.pp" % horizon_host
 
     proto = "http"
     config["CONFIG_HORIZON_PORT"] = 80
@@ -207,10 +204,6 @@ def create_manifest(config, messages):
         if config["CONFIG_NEUTRON_VPNAAS"] == 'y':
             config["CONFIG_HORIZON_NEUTRON_VPN"] = True
 
-    manifestdata = getManifestTemplate("horizon")
-    manifestdata += getManifestTemplate("apache_ports")
-    appendManifestFile(manifestfile, manifestdata)
-
     msg = ("To access the OpenStack Dashboard browse to %s://%s/dashboard .\n"
            "Please, find your login credentials stored in the keystonerc_admin"
            " in your home directory."