fix ceilometer.conf to point to https auth uri in a ssl enabled keyst…

…one setup.

fix keystone haproxy backend syntax to support ssl.

Closes-Bug: 1645570
(cherry picked from commit bb6bd44)

Conflicts:
	fabfile/tasks/ssl.py
Change-Id: Ifd49cdff38ec63c5f7a8d9aa25d497f026f19e7b

fabfile/tasks/ha.py

@@ -4,14 +4,21 @@
 from fabfile.templates import openstack_haproxy, collector_haproxy
 from fabfile.tasks.helpers import enable_haproxy
 from fabfile.tasks.rabbitmq import purge_node_from_rabbitmq_cluster
-from fabfile.utils.fabos import detect_ostype, get_as_sudo, is_package_installed, get_openstack_services
-from fabfile.utils.host import get_authserver_ip, get_control_host_string,\
-    hstr_to_ip, get_from_testbed_dict, get_service_token, get_env_passwords,\
-    get_openstack_internal_vip, get_openstack_external_vip,\
-    get_contrail_internal_vip, get_contrail_external_vip, \
-    get_openstack_internal_virtual_router_id, get_contrail_internal_virtual_router_id, \
-    get_openstack_external_virtual_router_id, get_contrail_external_virtual_router_id,\
-    get_haproxy_token
+from fabfile.utils.fabos import (
+        detect_ostype, get_as_sudo, is_package_installed,
+        get_openstack_services
+        )
+from fabfile.utils.host import (
+        get_authserver_ip, get_control_host_string, hstr_to_ip,
+        get_from_testbed_dict, get_service_token, get_env_passwords,
+        get_openstack_internal_vip, get_openstack_external_vip,
+        get_contrail_internal_vip, get_contrail_external_vip,
+        get_openstack_internal_virtual_router_id,
+        get_contrail_internal_virtual_router_id,
+        get_openstack_external_virtual_router_id,
+        get_contrail_external_virtual_router_id,
+        get_haproxy_token, keystone_ssl_enabled,
+        )
 from fabfile.utils.cluster import get_orchestrator
 from fabfile.tasks.provision import fixup_restart_haproxy_in_all_cfgm
 from fabfile.utils.commandline import frame_vnc_database_cmd, frame_vnc_config_cmd
@@ -417,6 +424,8 @@ def fixup_restart_haproxy_in_openstack():
 
 @task
 def fixup_restart_haproxy_in_openstack_node(*args):
+    keystnone_frontend = 'frontend openstack-keystone *:5000'
+    keystnone_admin_frontend = 'frontend openstack-keystone-admin *:35357'
     keystone_server_lines = ''
     keystone_admin_server_lines = ''
     glance_server_lines = ''
@@ -432,16 +441,44 @@ def fixup_restart_haproxy_in_openstack_node(*args):
     barbican_server_lines = ''
     space = ' ' * 3 
 
+    if keystone_ssl_enabled():
+        keystone_frontend_lines = [
+                'frontend openstack-keystone',
+                '%s bind  *:5000 ssl crt /etc/keystone/ssl/certs/keystonecertbundle.pem' % space,
+                '%s option http-server-close' % space,
+                '%s option forwardfor' % space,
+                '%s reqadd X-Forwarded-Proto:\ https' % space,
+                '%s reqadd X-Forwarded-Port:\ 5000' % space,
+                ]
+        keystone_frontend = '\n'.join(keystone_frontend_lines)
+        keystone_admin_frontend_lines = [
+                'frontend openstack-keystone-admin',
+                '%s bind  *:35357 ssl crt /etc/keystone/ssl/certs/keystonecertbundle.pem' % space,
+                '%s option http-server-close' % space,
+                '%s option forwardfor' % space,
+                '%s reqadd X-Forwarded-Proto:\ https' % space,
+                '%s reqadd X-Forwarded-Port:\ 35357' % space,
+                ]
+        keystone_admin_frontend = '\n'.join(keystone_admin_frontend_lines)
+
     for host_string in env.roledefs['openstack']:
         server_index = env.roledefs['openstack'].index(host_string) + 1
         mgmt_host_ip = hstr_to_ip(host_string)
         host_ip = hstr_to_ip(get_control_host_string(host_string))
         keystone_server_lines +=\
-            '%s server %s %s:6000 check inter 2000 rise 2 fall 1\n'\
+            '%s server %s %s:6000 check inter 2000 rise 2 fall 1'\
              % (space, host_ip, host_ip)
+        if keystone_ssl_enabled():
+            keystone_server_lines += " ssl verify none\n"
+        else:
+            keystone_server_lines += "\n"
         keystone_admin_server_lines +=\
-            '%s server %s %s:35358 check inter 2000 rise 2 fall 1\n'\
+            '%s server %s %s:35358 check inter 2000 rise 2 fall 1'\
              % (space, host_ip, host_ip)
+        if keystone_ssl_enabled():
+            keystone_admin_server_lines += " ssl verify none\n"
+        else:
+            keystone_admin_server_lines += "\n"
         glance_server_lines +=\
             '%s server %s %s:9393 check inter 2000 rise 2 fall 1\n'\
              % (space, host_ip, host_ip)
@@ -489,7 +526,9 @@ def fixup_restart_haproxy_in_openstack_node(*args):
 
     for host_string in env.roledefs['openstack']:
         haproxy_config = openstack_haproxy.template.safe_substitute({
+            '__keystone_frontend__' : keystone_frontend,
             '__keystone_backend_servers__' : keystone_server_lines,
+            '__keystone_admin_frontend__' : keystone_admin_frontend,
             '__keystone_admin_backend_servers__' : keystone_admin_server_lines,
             '__glance_backend_servers__' : glance_server_lines,
             '__heat_backend_servers__' : heat_server_lines,

fabfile/tasks/provision.py

@@ -705,6 +705,9 @@ def fixup_ceilometer_conf_common():
 #end fixup_ceilometer_conf_common
 
 def fixup_ceilometer_conf_keystone(openstack_ip):
+    auth_protocol = 'http'
+    if keystone_ssl_enabled():
+        auth_protocol = 'https'
     conf_file = '/etc/ceilometer/ceilometer.conf'
     with settings(warn_only=True):
         authtoken_config = sudo("grep '^auth_host =' /etc/ceilometer/ceilometer.conf").succeeded
@@ -713,15 +716,15 @@ def fixup_ceilometer_conf_keystone(openstack_ip):
         sudo("%s admin_password CEILOMETER_PASS" % config_cmd)
         sudo("%s admin_user ceilometer" % config_cmd)
         sudo("%s admin_tenant_name service" % config_cmd)
-        sudo("%s auth_uri http://%s:5000" % (config_cmd, openstack_ip))
-        sudo("%s auth_protocol http" % config_cmd)
+        sudo("%s auth_uri %s://%s:5000" % (config_cmd, auth_protocol, openstack_ip))
+        sudo("%s auth_protocol %s" % (config_cmd, auth_protocol))
         sudo("%s auth_port 35357" % config_cmd)
         sudo("%s auth_host %s" % (config_cmd, openstack_ip))
         config_cmd = "openstack-config --set %s service_credentials" % conf_file
         sudo("%s os_password CEILOMETER_PASS" % config_cmd)
         sudo("%s os_tenant_name service" % config_cmd)
         sudo("%s os_username ceilometer" % config_cmd)
-        sudo("%s os_auth_url http://%s:5000/v2.0" % (config_cmd, openstack_ip))
+        sudo("%s os_auth_url %s://%s:5000/v2.0" % (config_cmd, auth_protocol, openstack_ip))
 #end fixup_ceilometer_conf_keystone
 
 def fixup_ceilometer_pipeline_conf(analytics_ip):
@@ -1005,7 +1008,7 @@ def setup_ceilometer_node(*args):
                 ceilometer_service_exists = sudo("source /etc/contrail/openstackrc;keystone --insecure service-list | grep ceilometer").succeeded
             if not ceilometer_service_exists:
                 sudo("source /etc/contrail/openstackrc;keystone --insecure service-create --name=ceilometer --type=metering --description=\"Telemetry\"")
-                sudo("source /etc/contrail/openstackrc;keystone --insecure endpoint-create --service-id=$(keystone service-list | awk '/ metering / {print $2}') --publicurl=http://%s:8777 --internalurl=http://%s:8777 --adminurl=http://%s:8777 --region=RegionOne" %(self_ip, self_ip, self_ip))
+                sudo("source /etc/contrail/openstackrc;keystone --insecure endpoint-create --service-id=$(keystone --insecure service-list | awk '/ metering / {print $2}') --publicurl=http://%s:8777 --internalurl=http://%s:8777 --adminurl=http://%s:8777 --region=RegionOne" %(self_ip, self_ip, self_ip))
             # Fixup ceilometer pipeline cfg
             fixup_ceilometer_pipeline_conf(analytics_ip)
             for svc in ceilometer_services:
@@ -1033,6 +1036,13 @@ def setup_network_service_node(*args):
         sudo("service neutron-server restart")
 #end setup_network_service_node
 
+@task
+@roles('openstack')
+def setup_identity_service():
+    """Provisions identity services in openstack nodes"""
+    if env.roledefs['openstack']:
+        execute("setup_identity_service_node", env.host_string)
+
 @task
 def setup_identity_service_node(*args):
     """Provisions identity services in one or list of nodes.
@@ -1051,6 +1061,13 @@ def setup_identity_service_node(*args):
         sudo("service keystone restart")
 #end setup_identity_service_node
 
+@task
+@roles('openstack')
+def setup_image_service():
+    """Provisions image services in openstack nodes"""
+    if env.roledefs['openstack']:
+        execute("setup_image_service_node", env.host_string)
+
 @task
 def setup_image_service_node(*args):
     """Provisions image services in one or list of nodes. USAGE: fab setup_image_service_node:user@1.1.1.1,user@2.2.2.2"""
@@ -1088,11 +1105,6 @@ def setup_openstack():
             execute("setup_openstack_node", env.host_string)
         if is_package_installed('contrail-openstack-dashboard'):
             execute('setup_contrail_horizon_node', env.host_string)
-        if is_ceilometer_provision_supported():
-            execute("setup_ceilometer_node", env.host_string)
-            execute("setup_network_service") #Provisions in cfgm node
-            execute("setup_image_service_node", env.host_string)
-            execute("setup_identity_service_node", env.host_string)
 
 @task
 @roles('openstack')
@@ -1741,7 +1753,7 @@ def setup_only_vrouter_node(manage_nova_compute='yes', configure_nova='yes', *ar
 @EXECUTE_TASK
 def prov_alarm():
     cfgm_host = env.roledefs['cfgm'][0]
-    cfgm_ip = hstr_to_ip(get_control_host_string(cfgm_host))
+    cfgm_ip = get_contrail_internal_vip() or hstr_to_ip(get_control_host_string(cfgm_host))
     cfgm_host_password = get_env_passwords(cfgm_host)
     with settings(cd(UTILS_DIR), host_string=cfgm_host,
             password=cfgm_host_password):
@@ -1764,7 +1776,7 @@ def prov_config_node(*args, **kwargs):
     oper = kwargs.get('oper', 'add')
     tgt_node = kwargs.get('tgt_node', None)
     cfgm_host = env.roledefs['cfgm'][0]
-    cfgm_ip = hstr_to_ip(get_control_host_string(cfgm_host))
+    cfgm_ip = get_contrail_internal_vip() or hstr_to_ip(get_control_host_string(cfgm_host))
     cfgm_host_password = get_env_passwords(cfgm_host)
     for host_string in args:
         with settings(host_string = host_string):
@@ -1798,7 +1810,7 @@ def prov_database_node(*args, **kwargs):
     oper = kwargs.get('oper', 'add')
     tgt_node = kwargs.get('tgt_node', None)
     cfgm_host = env.roledefs['cfgm'][0]
-    cfgm_ip = hstr_to_ip(get_control_host_string(cfgm_host))
+    cfgm_ip = get_contrail_internal_vip() or hstr_to_ip(get_control_host_string(cfgm_host))
     cfgm_host_password = get_env_passwords(cfgm_host)
     for host_string in args:
         with settings(host_string = host_string):
@@ -1833,7 +1845,7 @@ def prov_analytics_node(*args, **kwargs):
     oper = kwargs.get('oper', 'add')
     tgt_node = kwargs.get('tgt_node', None)
     cfgm_host = env.roledefs['cfgm'][0]
-    cfgm_ip = hstr_to_ip(get_control_host_string(cfgm_host))
+    cfgm_ip = get_contrail_internal_vip() or hstr_to_ip(get_control_host_string(cfgm_host))
     cfgm_host_password = get_env_passwords(cfgm_host)
     for host_string in args:
         with settings(host_string = host_string):
@@ -1867,7 +1879,7 @@ def prov_control_bgp_node(*args, **kwargs):
     oper = kwargs.get('oper', 'add')
     tgt_node = kwargs.get('tgt_node', None)
     cfgm_host = kwargs.get('cfgm_host', env.roledefs['cfgm'][0])
-    cfgm_ip = hstr_to_ip(get_control_host_string(cfgm_host))
+    cfgm_ip = get_contrail_internal_vip() or hstr_to_ip(get_control_host_string(cfgm_host))
     cfgm_host_password = get_env_passwords(cfgm_host)
     for host_string in args:
         with settings(host_string = host_string):
@@ -1909,7 +1921,7 @@ def prov_external_bgp():
 def prov_external_bgp_node(*args):
     for host_string in args:
         with settings(host_string = host_string):
-            cfgm_ip = hstr_to_ip(get_control_host_string(env.roledefs['cfgm'][0]))
+            cfgm_ip = get_contrail_internal_vip() or hstr_to_ip(get_control_host_string(env.roledefs['cfgm'][0]))
             for ext_bgp in testbed.ext_routers:
                 ext_bgp_name = ext_bgp[0]
                 ext_bgp_ip   = ext_bgp[1]
@@ -2515,8 +2527,11 @@ def setup_orchestrator():
     if orch == 'openstack':
         execute('increase_ulimits')
         execute('setup_openstack')
-        if get_openstack_internal_vip():
-            execute('sync_keystone_ssl_certs')
+        if is_ceilometer_provision_supported():
+            execute("setup_ceilometer")
+            execute("setup_network_service") #Provisions in cfgm node
+            execute("setup_image_service",)
+            execute("setup_identity_service")
         execute('verify_openstack')
     #setup_vcenter can be called outside of setup_all and need not be below. So commenting.
     #elif orch == 'vcenter':

fabfile/tasks/ssl.py

@@ -4,12 +4,15 @@
 from fabric.contrib.files import exists
 
 from fabfile.config import *
-from fabfile.utils.host import (get_keystone_certfile, get_keystone_keyfile,
-                                get_keystone_cafile, get_apiserver_certfile,
-                                get_apiserver_keyfile, get_apiserver_cafile,
-                                get_env_passwords, get_openstack_internal_vip,
-                                get_contrail_internal_vip, hstr_to_ip,
-                                get_apiserver_cert_bundle, get_control_host_string)
+from fabfile.utils.host import (
+        get_keystone_certfile, get_keystone_keyfile,
+        get_keystone_cafile, get_apiserver_certfile,
+        get_apiserver_keyfile, get_apiserver_cafile,
+        get_env_passwords, get_openstack_internal_vip,
+        get_contrail_internal_vip, hstr_to_ip,
+        get_apiserver_cert_bundle, get_control_host_string,
+        get_keystone_cert_bundle,
+        )
 from fabfile.utils.fabos import get_as_sudo
 
 
@@ -25,6 +28,7 @@ def setup_keystone_ssl_certs_node(*nodes):
     default_certfile = '/etc/keystone/ssl/certs/keystone.pem'
     default_keyfile = '/etc/keystone/ssl/private/keystone.key'
     default_cafile = '/etc/keystone/ssl/certs/keystone_ca.pem'
+    keystonecertbundle = get_keystone_cert_bundle()
     ssl_certs = ((get_keystone_certfile(), default_certfile),
                  (get_keystone_keyfile(), default_keyfile),
                  (get_keystone_cafile(), default_cafile))
@@ -35,6 +39,7 @@ def setup_keystone_ssl_certs_node(*nodes):
                 if ssl_cert == default:
                     # Clear old certificate
                     sudo('rm -f %s' % ssl_cert)
+                    sudo('rm -f %s' % keystonecertbundle)
             for ssl_cert, default in ssl_certs:
                 if ssl_cert == default:
                     openstack_host = env.roledefs['openstack'][0]
@@ -54,8 +59,8 @@ def setup_keystone_ssl_certs_node(*nodes):
                             tmp_fname = os.path.join('/tmp', os.path.basename(ssl_cert))
                             get_as_sudo(ssl_cert, tmp_fname)
                         print "Copy to this(%s) openstack node" % env.host_string 
-                        sudo("mkdir -p /etc/keystone/ssl/certs/")
-                        sudo("mkdir -p /etc/keystone/ssl/private/")
+                        sudo('mkdir -p /etc/keystone/ssl/certs/')
+                        sudo('mkdir -p /etc/keystone/ssl/private/')
                         put(tmp_fname, ssl_cert, use_sudo=True)
                         os.remove(tmp_fname)
                 elif os.path.isfile(ssl_cert): 
@@ -66,6 +71,9 @@ def setup_keystone_ssl_certs_node(*nodes):
                     pass
                 else:
                     raise RuntimeError("%s doesn't exists locally or in openstack node")
+            if not exists(keystonecertbundle, use_sudo=True):
+                ((certfile, _), (keyfile, _), (cafile, _)) = ssl_certs
+                sudo('cat %s %s > %s' % (certfile, cafile, keystonecertbundle))
             sudo("chown -R keystone:keystone /etc/keystone/ssl")
 
 
@@ -111,8 +119,8 @@ def setup_apiserver_ssl_certs_node(*nodes):
                             tmp_fname = os.path.join('/tmp', os.path.basename(ssl_cert))
                             get_as_sudo(ssl_cert, tmp_fname)
                         print "Copy to this(%s) cfgm node" % env.host_string 
-                        sudo("mkdir -p /etc/contrail/ssl/certs")
-                        sudo("mkdir -p /etc/contrail/ssl/private")
+                        sudo('mkdir -p /etc/contrail/ssl/certs/')
+                        sudo('mkdir -p /etc/contrail/ssl/private/')
                         put(tmp_fname, ssl_cert, use_sudo=True)
                         os.remove(tmp_fname)
                 elif os.path.isfile(ssl_cert): 

fabfile/templates/openstack_haproxy.py

@@ -7,7 +7,7 @@
    stats uri /
    stats auth $__contrail_hap_user__:$__contrail_hap_passwd__
 
-frontend openstack-keystone *:5000
+$__keystone_frontend__
     default_backend    keystone-backend
 
 backend keystone-backend
@@ -33,7 +33,7 @@
 
 $__keystone_backend_servers__
 
-frontend openstack-keystone-admin *:35357
+$__keystone_admin_frontend__
     default_backend    keystone-admin-backend
 
 backend keystone-admin-backend

fabfile/utils/host.py

@@ -437,6 +437,10 @@ def get_keystone_cafile():
     return get_from_testbed_dict('keystone','cafile', default)
 
 
+def get_keystone_cert_bundle():
+    return '/etc/keystone/ssl/certs/keystonecertbundle.pem'
+
+
 def get_apiserver_certfile():
     default = '/etc/contrail/ssl/certs/contrail.pem'
     return get_from_testbed_dict('cfgm','certfile', default)