Integrate OSProfiler and Nova

* Add osprofiler wsgi middleware. This middleware is used for 2 things:
  1) It checks that person who want to trace is trusted and knows
     secret HMAC key.
  2) It starts tracing in case of proper trace headers
     and adds the first wsgi trace point with info about the HTTP request

* Add initialization of osprofiler on start of a service
  Currently that includes oslo.messaging notifier instance creation
  to send Ceilometer backend notifications.

oslo-spec: https://review.openstack.org/#/c/103825/
python-novaclient change: https://review.openstack.org/#/c/254699/
based on: https://review.openstack.org/#/c/105096/

Co-Authored-By: Boris Pavlovic <boris@pavlovic.me>
Co-Authored-By: Munoz, Obed N <obed.n.munoz@intel.com>
Co-Authored-By: Roman Podoliaka <rpodolyaka@mirantis.com>
Co-Authored-By: Tovin Seven <vinhnt@vn.fujitsu.com>

Implements: blueprint osprofiler-support-in-nova
Change-Id: I82d2badc8c1fcec27c3fce7c3c20e0f3b76414f1

etc/nova/api-paste.ini

@@ -28,13 +28,13 @@ use = call:nova.api.openstack.urlmap:urlmap_factory
 
 [composite:openstack_compute_api_v21]
 use = call:nova.api.auth:pipeline_factory_v21
-noauth2 = cors http_proxy_to_wsgi compute_req_id faultwrap sizelimit noauth2 osapi_compute_app_v21
-keystone = cors http_proxy_to_wsgi compute_req_id faultwrap sizelimit authtoken keystonecontext osapi_compute_app_v21
+noauth2 = cors http_proxy_to_wsgi compute_req_id faultwrap sizelimit osprofiler noauth2 osapi_compute_app_v21
+keystone = cors http_proxy_to_wsgi compute_req_id faultwrap sizelimit osprofiler authtoken keystonecontext osapi_compute_app_v21
 
 [composite:openstack_compute_api_v21_legacy_v2_compatible]
 use = call:nova.api.auth:pipeline_factory_v21
-noauth2 = cors http_proxy_to_wsgi compute_req_id faultwrap sizelimit noauth2 legacy_v2_compatible osapi_compute_app_v21
-keystone = cors http_proxy_to_wsgi compute_req_id faultwrap sizelimit authtoken keystonecontext legacy_v2_compatible osapi_compute_app_v21
+noauth2 = cors http_proxy_to_wsgi compute_req_id faultwrap sizelimit osprofiler noauth2 legacy_v2_compatible osapi_compute_app_v21
+keystone = cors http_proxy_to_wsgi compute_req_id faultwrap sizelimit osprofiler authtoken keystonecontext legacy_v2_compatible osapi_compute_app_v21
 
 [filter:request_id]
 paste.filter_factory = oslo_middleware:RequestId.factory
@@ -48,6 +48,9 @@ paste.filter_factory = nova.api.openstack:FaultWrapper.factory
 [filter:noauth2]
 paste.filter_factory = nova.api.openstack.auth:NoAuthMiddleware.factory
 
+[filter:osprofiler]
+paste.filter_factory = nova.profiler:WsgiMiddleware.factory
+
 [filter:sizelimit]
 paste.filter_factory = oslo_middleware:RequestBodySizeLimiter.factory
 

nova/cells/rpcapi.py

@@ -32,13 +32,15 @@
 from nova.i18n import _LE
 from nova import objects
 from nova.objects import base as objects_base
+from nova import profiler
 from nova import rpc
 
 LOG = logging.getLogger(__name__)
 
 CONF = nova.conf.CONF
 
 
+@profiler.trace_cls("rpc")
 class CellsAPI(object):
     '''Cells client-side RPC API
 

nova/cert/rpcapi.py

@@ -19,11 +19,13 @@
 import oslo_messaging as messaging
 
 import nova.conf
+from nova import profiler
 from nova import rpc
 
 CONF = nova.conf.CONF
 
 
+@profiler.trace_cls("rpc")
 class CertAPI(object):
     '''Client side of the cert rpc API.
 

nova/compute/api.py

@@ -76,6 +76,7 @@
 from nova.objects import quotas as quotas_obj
 from nova.pci import request as pci_request
 import nova.policy
+from nova import profiler
 from nova import rpc
 from nova.scheduler import client as scheduler_client
 from nova.scheduler import utils as scheduler_utils
@@ -191,6 +192,7 @@ def _diff_dict(orig, new):
     return result
 
 
+@profiler.trace_cls("compute_api")
 class API(base.Base):
     """API for interacting with the compute manager."""
 

nova/compute/rpcapi.py

@@ -27,6 +27,7 @@
 from nova.objects import base as objects_base
 from nova.objects import migrate_data as migrate_data_obj
 from nova.objects import service as service_obj
+from nova import profiler
 from nova import rpc
 
 CONF = nova.conf.CONF
@@ -54,6 +55,7 @@ def _compute_host(host, instance):
     return instance.host
 
 
+@profiler.trace_cls("rpc")
 class ComputeAPI(object):
     '''Client side of the compute rpc API.
 

nova/conductor/manager.py

@@ -42,6 +42,7 @@
 from nova import notifications
 from nova import objects
 from nova.objects import base as nova_object
+from nova import profiler
 from nova import rpc
 from nova.scheduler import client as scheduler_client
 from nova.scheduler import utils as scheduler_utils
@@ -176,6 +177,7 @@ def obj_target_cell(obj, cell):
             yield
 
 
+@profiler.trace_cls("rpc")
 class ComputeTaskManager(base.Base):
     """Namespace for compute methods.
 

nova/conductor/rpcapi.py

@@ -21,11 +21,13 @@
 
 import nova.conf
 from nova.objects import base as objects_base
+from nova import profiler
 from nova import rpc
 
 CONF = nova.conf.CONF
 
 
+@profiler.trace_cls("rpc")
 class ConductorAPI(object):
     """Client side of the conductor RPC API
 
@@ -247,6 +249,7 @@ def object_backport_versions(self, context, objinst, object_versions):
                           object_versions=object_versions)
 
 
+@profiler.trace_cls("rpc")
 class ComputeTaskAPI(object):
     """Client side of the conductor 'compute' namespaced RPC API
 

nova/config.py

@@ -16,13 +16,16 @@
 #    under the License.
 
 from oslo_log import log
+from oslo_utils import importutils
 
 from nova.common import config
 import nova.conf
 from nova.db.sqlalchemy import api as sqlalchemy_api
 from nova import rpc
 from nova import version
 
+profiler = importutils.try_import('osprofiler.opts')
+
 
 CONF = nova.conf.CONF
 
@@ -39,6 +42,8 @@ def parse_args(argv, default_config_files=None, configure_db=True,
     log.set_defaults(default_log_levels=log.get_default_log_levels() +
                      extra_default_log_levels)
     rpc.set_defaults(control_exchange='nova')
+    if profiler:
+        profiler.set_defaults(CONF)
     config.set_middleware_defaults()
 
     CONF(argv[1:],

nova/console/rpcapi.py

@@ -19,11 +19,13 @@
 import oslo_messaging as messaging
 
 import nova.conf
+from nova import profiler
 from nova import rpc
 
 CONF = nova.conf.CONF
 
 
+@profiler.trace_cls("rpc")
 class ConsoleAPI(object):
     '''Client side of the console rpc API.
 

nova/consoleauth/rpcapi.py

@@ -19,11 +19,13 @@
 import oslo_messaging as messaging
 
 import nova.conf
+from nova import profiler
 from nova import rpc
 
 CONF = nova.conf.CONF
 
 
+@profiler.trace_cls("rpc")
 class ConsoleAuthAPI(object):
     '''Client side of the consoleauth rpc API.
 

nova/db/sqlalchemy/api.py

@@ -31,6 +31,7 @@
 from oslo_db.sqlalchemy import utils as sqlalchemyutils
 from oslo_log import log as logging
 from oslo_utils import excutils
+from oslo_utils import importutils
 from oslo_utils import timeutils
 from oslo_utils import uuidutils
 import six
@@ -68,6 +69,7 @@
 from nova import quota
 from nova import safe_utils
 
+profiler_sqlalchemy = importutils.try_import('osprofiler.sqlalchemy')
 
 CONF = nova.conf.CONF
 
@@ -110,6 +112,14 @@ def configure(conf):
     main_context_manager.configure(**_get_db_conf(conf.database))
     api_context_manager.configure(**_get_db_conf(conf.api_database))
 
+    if profiler_sqlalchemy and CONF.profiler.enabled \
+            and CONF.profiler.trace_sqlalchemy:
+
+        main_context_manager.append_on_engine_create(
+            lambda eng: profiler_sqlalchemy.add_tracing(sa, eng, "db"))
+        api_context_manager.append_on_engine_create(
+            lambda eng: profiler_sqlalchemy.add_tracing(sa, eng, "db"))
+
 
 def create_context_manager(connection=None):
     """Create a database context manager object.

nova/image/api.py

@@ -16,8 +16,10 @@
 """
 
 from nova.image import glance
+from nova import profiler
 
 
+@profiler.trace_cls("nova_image")
 class API(object):
 
     """Responsible for exposing a relatively stable internal API for other

nova/manager.py

@@ -52,9 +52,11 @@
 """
 
 from oslo_service import periodic_task
+import six
 
 import nova.conf
 from nova.db import base
+from nova import profiler
 from nova import rpc
 
 
@@ -66,7 +68,28 @@ def __init__(self):
         super(PeriodicTasks, self).__init__(CONF)
 
 
+class ManagerMeta(profiler.get_traced_meta(), type(PeriodicTasks)):
+    """Metaclass to trace all children of a specific class.
+
+    This metaclass wraps every public method (not starting with _ or __)
+    of the class using it. All children classes of the class using ManagerMeta
+    will be profiled as well.
+
+    Adding this metaclass requires that the __trace_args__ attribute be added
+    to the class we want to modify. That attribute is a dictionary
+    with one mandatory key: "name". "name" defines the name
+    of the action to be traced (for example, wsgi, rpc, db).
+
+    The OSprofiler-based tracing, although, will only happen if profiler
+    instance was initiated somewhere before in the thread, that can only happen
+    if profiling is enabled in nova.conf and the API call to Nova API contained
+    specific headers.
+    """
+
+
+@six.add_metaclass(ManagerMeta)
 class Manager(base.Base, PeriodicTasks):
+    __trace_args__ = {"name": "rpc"}
 
     def __init__(self, host=None, db_driver=None, service_name='undefined'):
         if not host:

nova/network/api.py

@@ -28,13 +28,15 @@
 from nova.network import rpcapi as network_rpcapi
 from nova import objects
 from nova.objects import base as obj_base
+from nova import profiler
 from nova import utils
 
 CONF = cfg.CONF
 
 LOG = logging.getLogger(__name__)
 
 
+@profiler.trace_cls("network_api")
 class API(base_api.NetworkAPI):
     """API for doing networking via the nova-network network manager.
 

nova/network/neutronv2/api.py

@@ -39,9 +39,9 @@
 from nova.pci import utils as pci_utils
 from nova.pci import whitelist as pci_whitelist
 from nova.policies import base as base_policies
+from nova import profiler
 from nova import service_auth
 
-
 CONF = nova.conf.CONF
 
 LOG = logging.getLogger(__name__)
@@ -74,6 +74,7 @@ def _load_auth_plugin(conf):
     raise neutron_client_exc.Unauthorized(message=err_msg)
 
 
+@profiler.trace_cls("neutron_api")
 class ClientWrapper(clientv20.Client):
     """A Neutron client wrapper class.
 

nova/network/rpcapi.py

@@ -22,12 +22,14 @@
 import nova.conf
 from nova import exception
 from nova.objects import base as objects_base
+from nova import profiler
 from nova import rpc
 
 
 CONF = nova.conf.CONF
 
 
+@profiler.trace_cls("rpc")
 class NetworkAPI(object):
     '''Client side of the network rpc API.
 

nova/profiler.py

@@ -0,0 +1,73 @@
+# Copyright 2016 IBM Corporation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_utils import importutils
+import webob.dec
+
+import nova.conf
+
+profiler = importutils.try_import('osprofiler.profiler')
+profiler_web = importutils.try_import('osprofiler.web')
+
+CONF = nova.conf.CONF
+
+
+class WsgiMiddleware(object):
+
+    def __init__(self, application, **kwargs):
+        self.application = application
+
+    @classmethod
+    def factory(cls, global_conf, **local_conf):
+        if profiler_web:
+            return profiler_web.WsgiMiddleware.factory(global_conf,
+                                                       **local_conf)
+
+        def filter_(app):
+            return cls(app, **local_conf)
+
+        return filter_
+
+    @webob.dec.wsgify
+    def __call__(self, request):
+        return request.get_response(self.application)
+
+
+def get_traced_meta():
+    if profiler and 'profiler' in CONF and CONF.profiler.enabled:
+        return profiler.TracedMeta
+    else:
+        # NOTE(rpodolyaka): if we do not return a child of type, then Python
+        # fails to build a correct MRO when osprofiler is not installed
+        class NoopMeta(type):
+            pass
+        return NoopMeta
+
+
+def trace_cls(name, **kwargs):
+    """Wrap the OSProfiler trace_cls decorator so that it will not try to
+    patch the class unless OSProfiler is present and enabled in the config
+
+    :param name: The name of action. E.g. wsgi, rpc, db, etc..
+    :param kwargs: Any other keyword args used by profiler.trace_cls
+    """
+
+    def decorator(cls):
+        if profiler and 'profiler' in CONF and CONF.profiler.enabled:
+            trace_decorator = profiler.trace_cls(name, kwargs)
+            return trace_decorator(cls)
+        return cls
+
+    return decorator