Issue:

Password is displayed in the log files of the config daemon, during
uncaught exceptions.

Fix:
cgitb sets sys.excepthook to format uncaught exceptions. Deriving the
cgitb Hook and modifying the handle method to mask password along
with formatting.

Change-Id: I5b4251f2ebe0205465b15430a9ef38ef04b3a634
Closes-Bug: 1626317

src/config/api-server/db_manage.py

@@ -8,11 +8,11 @@
 from netaddr import IPAddress, IPNetwork
 import argparse
 from cStringIO import StringIO
-import cgitb
 
 import kazoo.client
 import kazoo.exceptions
 import cfgm_common
+from cfgm_common import vnc_cgitb
 from cfgm_common.utils import cgitb_hook
 from cfgm_common.ifmap.client import client
 from cfgm_common.ifmap.request import NewSessionRequest
@@ -1452,7 +1452,7 @@ def heal_security_groups_id(self):
 # end class DatabaseCleaner
 
 def db_check(args_str):
-    cgitb.enable(format='text')
+    vnc_cgitb.enable(format='text')
 
     db_checker = DatabaseChecker(args_str)
     # Mode and node count check across all nodes
@@ -1469,7 +1469,7 @@ def db_check(args_str):
 # end db_check
 
 def db_clean(args_str):
-    cgitb.enable(format='text')
+    vnc_cgitb.enable(format='text')
 
     db_cleaner = DatabaseCleaner(args_str)
     db_cleaner.clean_obj_missing_mandatory_fields()
@@ -1484,7 +1484,7 @@ def db_clean(args_str):
 # end db_clean
 
 def db_heal(args_str):
-    cgitb.enable(format='text')
+    vnc_cgitb.enable(format='text')
 
     db_healer = DatabaseHealer(args_str)
     db_healer.heal_fq_name_index()

src/config/api-server/tests/test_askip.py

@@ -10,9 +10,6 @@
 import logging
 import coverage
 
-import cgitb
-cgitb.enable(format='text')
-
 import testtools
 from testtools.matchers import Equals, MismatchError, Not, Contains
 from testtools import content, content_type, ExpectedException
@@ -29,6 +26,8 @@
 import vnc_api.gen.vnc_api_test_gen
 from vnc_api.gen.resource_test import *
 import cfgm_common
+from cfgm_common import vnc_cgitb
+vnc_cgitb.enable(format='text')
 
 sys.path.append('../common/tests')
 from test_utils import *

src/config/api-server/tests/test_crud_basic.py

@@ -13,9 +13,6 @@
 import netaddr
 import tempfile
 
-import cgitb
-cgitb.enable(format='text')
-
 import fixtures
 import testtools
 from testtools.matchers import Equals, MismatchError, Not, Contains, LessThan
@@ -41,6 +38,8 @@
 import cfgm_common
 from cfgm_common import vnc_plugin_base
 from cfgm_common import imid
+from cfgm_common import vnc_cgitb
+vnc_cgitb.enable(format='text')
 
 sys.path.append('../common/tests')
 from test_utils import *

src/config/api-server/tests/test_ip_alloc.py

@@ -10,9 +10,6 @@
 import logging
 import coverage
 
-import cgitb
-cgitb.enable(format='text')
-
 import testtools
 from testtools.matchers import Equals, MismatchError, Not, Contains
 from testtools import content, content_type, ExpectedException
@@ -29,6 +26,8 @@
 import vnc_api.gen.vnc_api_test_gen
 from vnc_api.gen.resource_test import *
 import cfgm_common
+from cfgm_common import vnc_cgitb
+vnc_cgitb.enable(format='text')
 
 sys.path.append('../common/tests')
 from test_utils import *

src/config/api-server/tests/test_logical_router.py

@@ -10,8 +10,6 @@
 import logging
 import coverage
 
-import cgitb
-cgitb.enable(format='text')
 
 import testtools
 from testtools.matchers import Equals, MismatchError, Not, Contains
@@ -32,6 +30,8 @@
 from netaddr import IPNetwork, IPAddress
 
 import cfgm_common
+from cfgm_common import vnc_cgitb
+vnc_cgitb.enable(format='text')
 
 sys.path.append('../common/tests')
 from test_utils import *

src/config/api-server/tests/test_perms.py

@@ -11,9 +11,6 @@
 import logging
 import coverage
 
-import cgitb
-cgitb.enable(format='text')
-
 import fixtures
 import testtools
 from testtools.matchers import Equals, MismatchError, Not, Contains
@@ -29,6 +26,8 @@
 
 from vnc_api.vnc_api import *
 import cfgm_common
+from cfgm_common import vnc_cgitb
+vnc_cgitb.enable(format='text')
 
 sys.path.append('../common/tests')
 import test_utils

src/config/api-server/tests/test_perms2.py

@@ -10,9 +10,6 @@
 import logging
 import coverage
 
-import cgitb
-cgitb.enable(format='text')
-
 import fixtures
 import testtools
 from testtools.matchers import Equals, MismatchError, Not, Contains
@@ -34,6 +31,8 @@
 from cfgm_common import rest, utils
 from cfgm_common.rbaclib import *
 import cfgm_common
+from cfgm_common import vnc_cgitb
+vnc_cgitb.enable(format='text')
 
 sys.path.append('../common/tests')
 import test_utils

src/config/api-server/tests/test_rbac.py

@@ -10,9 +10,6 @@
 import logging
 import coverage
 
-import cgitb
-cgitb.enable(format='text')
-
 import fixtures
 import testtools
 from testtools.matchers import Equals, MismatchError, Not, Contains
@@ -32,6 +29,8 @@
 from keystonemiddleware import auth_token
 from cfgm_common import rest, utils
 import cfgm_common
+from cfgm_common import vnc_cgitb
+vnc_cgitb.enable(format='text')
 
 sys.path.append('../common/tests')
 import test_utils

src/config/api-server/tests/test_subnet_ip_count.py

@@ -10,8 +10,6 @@
 import logging
 import coverage
 
-import cgitb
-cgitb.enable(format='text')
 
 import testtools
 from testtools.matchers import Equals, MismatchError, Not, Contains
@@ -29,6 +27,8 @@
 import vnc_api.gen.vnc_api_test_gen
 from vnc_api.gen.resource_test import *
 import cfgm_common
+from cfgm_common import vnc_cgitb
+vnc_cgitb.enable(format='text')
 
 sys.path.append('../common/tests')
 from test_utils import *

src/config/api-server/vnc_cfg_api_server.py

@@ -34,6 +34,8 @@
 from lxml import etree
 # import GreenletProfiler
 
+from cfgm_common import vnc_cgitb
+
 logger = logging.getLogger(__name__)
 
 """
@@ -3553,8 +3555,7 @@ def main(args_str=None):
 # end main
 
 def server_main(args_str=None):
-    import cgitb
-    cgitb.enable(format='text')
+    vnc_cgitb.enable(format='text')
 
     main()
 #server_main

src/config/common/SConscript

@@ -40,6 +40,7 @@ local_sources = [
                   'dependency_tracker.py',
                   'vnc_api_stats.py',
                   'ssl_adapter.py',
+                  'vnc_cgitb.py',
                  ]
 local_sources_rules = []
 for file in local_sources:

src/config/common/tests/test_common.py

@@ -24,6 +24,7 @@
 import cfgm_common.zkclient
 from cfgm_common.uve.vnc_api.ttypes import VncApiConfigLog
 from cfgm_common import imid
+from cfgm_common import vnc_cgitb
 from cfgm_common.utils import cgitb_hook
 
 from test_utils import *
@@ -148,8 +149,7 @@ def launch_disc_server(test_id, listen_ip, listen_port, http_server_port, conf_s
     args_str = args_str + "--log_local "
     args_str = args_str + "--log_file discovery_server_%s.log " % test_id
 
-    import cgitb
-    cgitb.enable(format='text')
+    vnc_cgitb.enable(format='text')
 
     with tempfile.NamedTemporaryFile() as conf, tempfile.NamedTemporaryFile() as logconf:
         cfg_parser = generate_conf_file_contents(conf_sections)
@@ -219,8 +219,7 @@ def launch_api_server(test_id, listen_ip, listen_port, http_server_port,
     args_str = args_str + "--log_local "
     args_str = args_str + "--log_file api_server_%s.log " %(test_id)
 
-    import cgitb
-    cgitb.enable(format='text')
+    vnc_cgitb.enable(format='text')
 
     with tempfile.NamedTemporaryFile() as conf, tempfile.NamedTemporaryFile() as logconf:
         cfg_parser = generate_conf_file_contents(conf_sections)
@@ -427,8 +426,7 @@ def __init__(self, *args, **kwargs):
         self.addOnException(self._add_detailed_traceback)
 
     def _add_detailed_traceback(self, exc_info):
-        import cgitb
-        cgitb.enable(format='text')
+        vnc_cgitb.enable(format='text')
         from cStringIO  import StringIO
 
         tmp_file = StringIO()

src/config/common/tests/test_discovery.py

@@ -27,6 +27,9 @@
 import gevent.wsgi
 import uuid
 
+from cfgm_common import vnc_cgitb
+
+
 def lineno():
     """Returns the current line number in our program."""
     return inspect.currentframe().f_back.f_lineno
@@ -103,8 +106,7 @@ def launch_disc_server(listen_ip, listen_port, http_server_port, conf_sections):
     args_str = args_str + "--log_local "
     args_str = args_str + "--log_file discovery_server_sandesh.log "
 
-    import cgitb
-    cgitb.enable(format='text')
+    vnc_cgitb.enable(format='text')
 
     with tempfile.NamedTemporaryFile() as conf, tempfile.NamedTemporaryFile() as logconf:
         cfg_parser = generate_conf_file_contents(conf_sections)
@@ -163,12 +165,11 @@ def __init__(self, *args, **kwargs):
         }
 
     def _add_detailed_traceback(self, exc_info):
-        import cgitb
-        cgitb.enable(format='text')
+        vnc_cgitb.enable(format='text')
         from cStringIO  import StringIO
 
         tmp_file = StringIO()
-        cgitb.Hook(format="text", file=tmp_file).handle(exc_info)
+        vnc_cgitb.Hook(format="text", file=tmp_file).handle(exc_info)
         tb_str = tmp_file.getvalue()
         tmp_file.close()
         self.addDetail('detailed-traceback', content.text_content(tb_str))

src/config/common/utils.py

@@ -22,76 +22,17 @@
 
 
 import os
-import re
 import urllib
 from collections import OrderedDict
 import sys
-import cgitb
 import cStringIO
 import logging
 
-
-# Masking of password from openstack/common/log.py
-_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
-
-# NOTE(ldbragst): Let's build a list of regex objects using the list of
-# _SANITIZE_KEYS we already have. This way, we only have to add the new key
-# to the list of _SANITIZE_KEYS and we can generate regular expressions
-# for XML and JSON automatically.
-_SANITIZE_PATTERNS = []
-_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
-                    r'(<%(key)s>).*?(</%(key)s>)',
-                    r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
-                    r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])']
-
-for key in _SANITIZE_KEYS:
-    for pattern in _FORMAT_PATTERNS:
-        reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
-        _SANITIZE_PATTERNS.append(reg_ex)
-
-
-def mask_password(message, secret="***"):
-    """Replace password with 'secret' in message.
-    :param message: The string which includes security information.
-    :param secret: value with which to replace passwords.
-    :returns: The unicode value of message with the password fields masked.
-
-    For example:
-
-    >>> mask_password("'adminPass' : 'aaaaa'")
-    "'adminPass' : '***'"
-    >>> mask_password("'admin_pass' : 'aaaaa'")
-    "'admin_pass' : '***'"
-    >>> mask_password('"password" : "aaaaa"')
-    '"password" : "***"'
-    >>> mask_password("'original_password' : 'aaaaa'")
-    "'original_password' : '***'"
-    >>> mask_password("u'original_password' :   u'aaaaa'")
-    "u'original_password' :   u'***'"
-    """
-    if not any(key in message for key in _SANITIZE_KEYS):
-        return message
-
-    secret = r'\g<1>' + secret + r'\g<2>'
-    for pattern in _SANITIZE_PATTERNS:
-        message = re.sub(pattern, secret, message)
-    return message
-# end mask_password
+from cfgm_common import vnc_cgitb
 
 
 def cgitb_hook(info=None, **kwargs):
-    buf = sys.stdout
-    if 'file' in kwargs:
-        buf = kwargs['file']
-
-    local_buf = cStringIO.StringIO()
-    kwargs['file'] = local_buf
-    cgitb.Hook(**kwargs).handle(info or sys.exc_info())
-
-    doc = local_buf.getvalue()
-    local_buf.close()
-    buf.write(mask_password(doc))
-    buf.flush()
+    vnc_cgitb.Hook(**kwargs).handle(info or sys.exc_info())
 # end cgitb_hook