From 58ffdf5de804ab26b2179069cc0603a2e3e3f4a4 Mon Sep 17 00:00:00 2001 From: Jens Vagelpohl Date: Thu, 8 Oct 2020 16:35:37 +0200 Subject: [PATCH 1/2] Restore the ZMI `Debug Information` control panel page (#899) * - Restore the ZMI `Debug Information` control panel page * - use bug icon * include `class_types` in refcount analysis Co-authored-by: dieter --- CHANGES.rst | 3 + src/App/ApplicationManager.py | 115 +++++++++++++++++++++- src/App/CacheManager.py | 86 +++++++++++++++++ src/App/DavLockManager.py | 10 +- src/App/dtml/davLockManager.dtml | 4 +- src/App/dtml/debug.dtml | 107 +++++++++++++++++++++ src/App/tests/test_ApplicationManager.py | 116 +++++++++++++++++++++++ src/App/tests/test_cachemanager.py | 60 ++++++++++++ src/ZPublisher/WSGIPublisher.py | 12 +++ 9 files changed, 509 insertions(+), 4 deletions(-) create mode 100644 src/App/CacheManager.py create mode 100644 src/App/dtml/debug.dtml create mode 100644 src/App/tests/test_cachemanager.py diff --git a/CHANGES.rst b/CHANGES.rst index 30f82e43cd..61b791d074 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,6 +10,9 @@ https://zope.readthedocs.io/en/2.13/CHANGES.html 4.5.2 (unreleased) ------------------ +- Restore the ZMI `Debug Information` control panel page + (`#898 `_) + - HTTP header encoding support (`#905 `_) diff --git a/src/App/ApplicationManager.py b/src/App/ApplicationManager.py index 488106b5bd..932ab215c8 100644 --- a/src/App/ApplicationManager.py +++ b/src/App/ApplicationManager.py @@ -14,18 +14,23 @@ import os import sys import time +import types +from six import class_types +from six.moves._thread import get_ident from six.moves.urllib import parse from AccessControl.class_init import InitializeClass from AccessControl.requestmethod import requestmethod from Acquisition import Implicit +from App.CacheManager import CacheManager from App.config import getConfiguration from App.DavLockManager import DavLockManager from App.Management import Tabs from App.special_dtml import DTMLFile from App.Undo import UndoSupport from App.version_txt import version_txt +from DateTime.DateTime import DateTime from OFS.Traversable import Traversable from Persistence import Persistent from Products.PageTemplates.PageTemplateFile import PageTemplateFile @@ -59,6 +64,8 @@ class DatabaseChooser(Tabs, Traversable, Implicit): {'label': 'Control Panel', 'action': '../manage_main'}, {'label': 'Databases', 'action': 'manage_main'}, {'label': 'Configuration', 'action': '../Configuration/manage_main'}, + {'label': 'DAV Locks', 'action': '../DAVLocks/manage_main'}, + {'label': 'Debug Information', 'action': '../DebugInfo/manage_main'}, ) MANAGE_TABS_NO_BANNER = True @@ -103,6 +110,7 @@ class ConfigurationViewer(Tabs, Traversable, Implicit): {'label': 'Databases', 'action': '../Database/manage_main'}, {'label': 'Configuration', 'action': 'manage_main'}, {'label': 'DAV Locks', 'action': '../DavLocks/manage_main'}, + {'label': 'Debug Information', 'action': '../DebugInfo/manage_main'}, ) MANAGE_TABS_NO_BANNER = True @@ -133,7 +141,107 @@ def manage_getConfiguration(self): InitializeClass(ConfigurationViewer) -class ApplicationManager(Persistent, Tabs, Traversable, Implicit): +# refcount snapshot info +_v_rcs = None +_v_rst = None + + +class DebugManager(Tabs, Traversable, Implicit): + """ Debug and profiling information + """ + manage = manage_main = manage_workspace = DTMLFile('dtml/debug', globals()) + manage_main._setName('manage_main') + id = 'DebugInfo' + name = title = 'Debug Information' + meta_type = name + zmi_icon = 'fas fa-bug' + + manage_options = ( + {'label': 'Control Panel', 'action': '../manage_main'}, + {'label': 'Databases', 'action': '../Database/manage_main'}, + {'label': 'Configuration', 'action': '../Configuration/manage_main'}, + {'label': 'DAV Locks', 'action': '../DAVLocks/manage_main'}, + {'label': 'Debug Information', 'action': 'manage_main'}, + ) + + def refcount(self, n=None, t=(type(Implicit),) + class_types): + # return class reference info + counts = {} + for m in list(sys.modules.values()): + if m is None: + continue + if not isinstance(m, types.ModuleType) or 'six.' in m.__name__: + continue + for sym in dir(m): + ob = getattr(m, sym) + if type(ob) in t: + counts[ob] = sys.getrefcount(ob) + pairs = [] + for ob, v in counts.items(): + ob_name = getattr(ob, "__name__", "unknown") + if hasattr(ob, '__module__'): + name = '%s.%s' % (ob.__module__, ob_name) + else: + name = '%s' % ob_name + pairs.append((v, name)) + pairs.sort() + pairs.reverse() + if n is not None: + pairs = pairs[:n] + return pairs + + def refdict(self): + counts = {} + for v, n in self.refcount(): + counts[n] = v + return counts + + def rcsnapshot(self): + global _v_rcs + global _v_rst + _v_rcs = self.refdict() + _v_rst = DateTime() + + def rcdate(self): + return _v_rst + + def rcdeltas(self): + if _v_rcs is None: + self.rcsnapshot() + nc = self.refdict() + rc = _v_rcs + rd = [] + for n, c in nc.items(): + try: + prev = rc.get(n, 0) + if c > prev: + rd.append((c - prev, (c, prev, n))) + except Exception: + pass + rd.sort() + rd.reverse() + return [{'name': n[1][2], + 'delta': n[0], + 'pc': n[1][1], + 'rc': n[1][0], + } for n in rd] + + def dbconnections(self): + import Zope2 # for data + return Zope2.DB.connectionDebugInfo() + + def manage_getSysPath(self): + return list(sys.path) + + +InitializeClass(DebugManager) + + +class ApplicationManager(CacheManager, + Persistent, + Tabs, + Traversable, + Implicit): """System management """ __allow_access_to_unprotected_subobjects__ = 1 @@ -148,6 +256,7 @@ class ApplicationManager(Persistent, Tabs, Traversable, Implicit): Database = DatabaseChooser() Configuration = ConfigurationViewer() DavLocks = DavLockManager() + DebugInfo = DebugManager() manage = manage_main = DTMLFile('dtml/cpContents', globals()) manage_main._setName('manage_main') @@ -156,6 +265,7 @@ class ApplicationManager(Persistent, Tabs, Traversable, Implicit): {'label': 'Databases', 'action': 'Database/manage_main'}, {'label': 'Configuration', 'action': 'Configuration/manage_main'}, {'label': 'DAV Locks', 'action': 'DavLocks/manage_main'}, + {'label': 'Debug Information', 'action': 'DebugInfo/manage_main'}, ) MANAGE_TABS_NO_BANNER = True @@ -190,6 +300,9 @@ def sys_version(self): def sys_platform(self): return sys.platform + def thread_get_ident(self): + return get_ident() + def debug_mode(self): return getConfiguration().debug_mode diff --git a/src/App/CacheManager.py b/src/App/CacheManager.py new file mode 100644 index 0000000000..ebbe1c379f --- /dev/null +++ b/src/App/CacheManager.py @@ -0,0 +1,86 @@ +############################################################################## +# +# Copyright (c) 2002 Zope Foundation and Contributors. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE +# +############################################################################## +"""Cache management support. + +This class is mixed into the application manager in App.ApplicationManager. +""" + +from AccessControl.class_init import InitializeClass +from Acquisition import aq_parent + + +class CacheManager: + """Cache management mix-in + """ + + def _getDB(self): + try: + return self._p_jar.db() + except AttributeError: + return aq_parent(self)._p_jar.db() + + def cache_size(self): + db = self._getDB() + return db.getCacheSize() + + def cache_detail(self, REQUEST=None): + """ + Returns the name of the classes of the objects in the cache + and the number of objects in the cache for each class. + """ + detail = self._getDB().cacheDetail() + if REQUEST is not None: + # format as text + REQUEST.RESPONSE.setHeader('Content-Type', 'text/plain') + return '\n'.join( + ['%6d %s' % (count, name) for name, count in detail]) + # raw + return detail + + def cache_extreme_detail(self, REQUEST=None): + """ + Returns information about each object in the cache. + """ + detail = self._getDB().cacheExtremeDetail() + if REQUEST is not None: + # sort the list. + lst = [((dict['conn_no'], dict['oid']), dict) for dict in detail] + # format as text. + res = [ + '# Table shows connection number, oid, refcount, state, ' + 'and class.', + '# States: L = loaded, G = ghost, C = changed'] + for sortkey, dict in lst: + id = dict.get('id', None) + if id: + idinfo = ' (%s)' % id + else: + idinfo = '' + s = dict['state'] + if s == 0: + state = 'L' # loaded + elif s == 1: + state = 'C' # changed + else: + state = 'G' # ghost + res.append('%d %-34s %6d %s %s%s' % ( + dict['conn_no'], repr(dict['oid']), dict['rc'], + state, dict['klass'], idinfo)) + REQUEST.RESPONSE.setHeader('Content-Type', 'text/plain') + return '\n'.join(res) + else: + # raw + return detail + + +InitializeClass(CacheManager) diff --git a/src/App/DavLockManager.py b/src/App/DavLockManager.py index 677d0e74d3..1eaad73cdf 100644 --- a/src/App/DavLockManager.py +++ b/src/App/DavLockManager.py @@ -31,10 +31,16 @@ class DavLockManager(Item, Implicit): security.declareProtected(webdav_manage_locks, # NOQA: D001 'manage_davlocks') - manage_davlocks = manage_main = manage = DTMLFile( + manage_davlocks = manage_main = manage = manage_workspace = DTMLFile( 'dtml/davLockManager', globals()) manage_davlocks._setName('manage_davlocks') - manage_options = ({'label': 'Write Locks', 'action': 'manage_main'}, ) + manage_options = ( + {'label': 'Control Panel', 'action': '../manage_main'}, + {'label': 'Databases', 'action': 'manage_main'}, + {'label': 'Configuration', 'action': '../Configuration/manage_main'}, + {'label': 'DAV Locks', 'action': 'manage_main'}, + {'label': 'Debug Information', 'action': '../DebugInfo/manage_main'}, + ) @security.protected(webdav_manage_locks) def findLockedObjects(self, frompath=''): diff --git a/src/App/dtml/davLockManager.dtml b/src/App/dtml/davLockManager.dtml index 43b33abfd1..56330e34dc 100644 --- a/src/App/dtml/davLockManager.dtml +++ b/src/App/dtml/davLockManager.dtml @@ -1,5 +1,7 @@ - + + +
diff --git a/src/App/dtml/debug.dtml b/src/App/dtml/debug.dtml new file mode 100644 index 0000000000..36f14d9c56 --- /dev/null +++ b/src/App/dtml/debug.dtml @@ -0,0 +1,107 @@ + + + + + + + + + + + + +
+ +

+ Reference count and database connection information +

+ +

Top 100 reference counts

+ + + + +


+

Changes since last refresh

+ + + + + + + + + + + + + + + + + + +
+ Class + + + + + + Delta +
+ &dtml-name; + + &dtml-pc; + + &dtml-rc; + + +&dtml-delta; +
+ + + +

+ Cache detail | + Cache extreme detail +

+ +

+

+ Update Snapshot | + + Stop auto refresh + + Refresh | + Auto refresh interval (seconds): + + + +
+ +

+ +

ZODB database connections

+ + + + + + + + + + + +
OpenedInfo
&dtml-opened;&dtml-info;
+ +


+ +
+ + diff --git a/src/App/tests/test_ApplicationManager.py b/src/App/tests/test_ApplicationManager.py index 22ebf181ee..6ceefc94d6 100644 --- a/src/App/tests/test_ApplicationManager.py +++ b/src/App/tests/test_ApplicationManager.py @@ -5,6 +5,7 @@ import time import unittest +import Testing.testbrowser import Testing.ZopeTestCase @@ -366,6 +367,121 @@ def test_manage_pack(self): self.assertIsNone(am._getDB()._packed) +class DebugManagerTests(unittest.TestCase): + + def setUp(self): + import sys + self._sys = sys + self._old_sys_modules = sys.modules.copy() + + def tearDown(self): + self._sys.modules.clear() + self._sys.modules.update(self._old_sys_modules) + + def _getTargetClass(self): + from App.ApplicationManager import DebugManager + return DebugManager + + def _makeOne(self, id): + return self._getTargetClass()(id) + + def _makeModuleClasses(self): + import sys + import types + from ExtensionClass import Base + + class Foo(Base): + pass + + class Bar(Base): + pass + + class Baz(Base): + pass + + foo = sys.modules['foo'] = types.ModuleType('foo') + foo.Foo = Foo + Foo.__module__ = 'foo' + foo.Bar = Bar + Bar.__module__ = 'foo' + qux = sys.modules['qux'] = types.ModuleType('qux') + qux.Baz = Baz + Baz.__module__ = 'qux' + return Foo, Bar, Baz + + def test_refcount_no_limit(self): + import sys + dm = self._makeOne('test') + Foo, Bar, Baz = self._makeModuleClasses() + pairs = dm.refcount() + # XXX : Ugly empiricism here: I don't know why the count is up 1. + foo_count = sys.getrefcount(Foo) + self.assertTrue((foo_count + 1, 'foo.Foo') in pairs) + bar_count = sys.getrefcount(Bar) + self.assertTrue((bar_count + 1, 'foo.Bar') in pairs) + baz_count = sys.getrefcount(Baz) + self.assertTrue((baz_count + 1, 'qux.Baz') in pairs) + + def test_refdict(self): + import sys + dm = self._makeOne('test') + Foo, Bar, Baz = self._makeModuleClasses() + mapping = dm.refdict() + # XXX : Ugly empiricism here: I don't know why the count is up 1. + foo_count = sys.getrefcount(Foo) + self.assertEqual(mapping['foo.Foo'], foo_count + 1) + bar_count = sys.getrefcount(Bar) + self.assertEqual(mapping['foo.Bar'], bar_count + 1) + baz_count = sys.getrefcount(Baz) + self.assertEqual(mapping['qux.Baz'], baz_count + 1) + + def test_rcsnapshot(self): + import sys + import App.ApplicationManager + from DateTime.DateTime import DateTime + dm = self._makeOne('test') + Foo, Bar, Baz = self._makeModuleClasses() + before = DateTime() + dm.rcsnapshot() + after = DateTime() + # XXX : Ugly empiricism here: I don't know why the count is up 1. + self.assertTrue(before <= App.ApplicationManager._v_rst <= after) + mapping = App.ApplicationManager._v_rcs + foo_count = sys.getrefcount(Foo) + self.assertEqual(mapping['foo.Foo'], foo_count + 1) + bar_count = sys.getrefcount(Bar) + self.assertEqual(mapping['foo.Bar'], bar_count + 1) + baz_count = sys.getrefcount(Baz) + self.assertEqual(mapping['qux.Baz'], baz_count + 1) + + def test_rcdate(self): + import App.ApplicationManager + dummy = object() + App.ApplicationManager._v_rst = dummy + dm = self._makeOne('test') + found = dm.rcdate() + App.ApplicationManager._v_rst = None + self.assertTrue(found is dummy) + + def test_rcdeltas(self): + dm = self._makeOne('test') + dm.rcsnapshot() + Foo, Bar, Baz = self._makeModuleClasses() + mappings = dm.rcdeltas() + self.assertTrue(len(mappings)) + mapping = mappings[0] + self.assertTrue('rc' in mapping) + self.assertTrue('pc' in mapping) + self.assertEqual(mapping['delta'], mapping['rc'] - mapping['pc']) + + # def test_dbconnections(self): XXX -- TOO UGLY TO TEST + + def test_manage_getSysPath(self): + import sys + dm = self._makeOne('test') + self.assertEqual(dm.manage_getSysPath(), list(sys.path)) + + class MenuDtmlTests(ConfigTestBase, Testing.ZopeTestCase.FunctionalTestCase): """Browser testing ..dtml.menu.dtml.""" diff --git a/src/App/tests/test_cachemanager.py b/src/App/tests/test_cachemanager.py new file mode 100644 index 0000000000..05ed7cab3e --- /dev/null +++ b/src/App/tests/test_cachemanager.py @@ -0,0 +1,60 @@ +############################################################################## +# +# Copyright (c) 2003 Zope Foundation and Contributors. +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# +############################################################################## +"""Tests for the CacheManager. +""" + +import unittest + + +class DummyConnection: + + def __init__(self, db): + self.__db = db + + def db(self): + return self.__db + + +class DummyDB: + + def __init__(self, cache_size): + self._set_sizes(cache_size) + + def _set_sizes(self, cache_size): + self.__cache_size = cache_size + + def getCacheSize(self): + return self.__cache_size + + +class CacheManagerTestCase(unittest.TestCase): + + def _getManagerClass(self): + from App.CacheManager import CacheManager + + class TestCacheManager(CacheManager): + # Derived CacheManager that fakes enough of the DatabaseManager to + # make it possible to test at least some parts of the CacheManager. + def __init__(self, connection): + self._p_jar = connection + + return TestCacheManager + + def test_cache_size(self): + db = DummyDB(42) + connection = DummyConnection(db) + manager = self._getManagerClass()(connection) + self.assertEqual(manager.cache_size(), 42) + db._set_sizes(12) + self.assertEqual(manager.cache_size(), 12) diff --git a/src/ZPublisher/WSGIPublisher.py b/src/ZPublisher/WSGIPublisher.py index 508cc63f2b..9aded018a5 100644 --- a/src/ZPublisher/WSGIPublisher.py +++ b/src/ZPublisher/WSGIPublisher.py @@ -248,6 +248,18 @@ def publish(request, module_info): request['PARENTS'] = [obj] obj = request.traverse(path, validated_hook=validate_user) + + # Set debug information from the active request on the open connection + # Used to be done in ZApplicationWrapper.__bobo_traverse__ for ZServer + try: + # Grab the connection from the last (root application) object, + # which usually has a connection available. + request['PARENTS'][-1]._p_jar.setDebugInfo(request.environ, + request.other) + except AttributeError: + # If there is no connection don't worry + pass + notify(pubevents.PubAfterTraversal(request)) recordMetaData(obj, request) From 6e96a9a2eb8a79d9d8039db60f47b1d36f2acf59 Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Fri, 9 Oct 2020 08:14:52 +0200 Subject: [PATCH 2/2] Update to ZCatalog 5.2 and set branches for packages those master is Zope 5 only. --- constraints.txt | 2 +- requirements-full.txt | 2 +- sources.cfg | 4 ++-- versions-prod.cfg | 3 +-- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/constraints.txt b/constraints.txt index c0e7d32338..cc82b0775b 100644 --- a/constraints.txt +++ b/constraints.txt @@ -12,7 +12,7 @@ Paste==3.4.3 PasteDeploy==2.1.0 Persistence==3.0 Products.BTreeFolder2==4.2 -Products.ZCatalog==5.1 +Products.ZCatalog==5.2 Record==3.5 RestrictedPython==5.0 WSGIProxy2==0.4.6 diff --git a/requirements-full.txt b/requirements-full.txt index c3bbda6e48..0bc48052f1 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -13,7 +13,7 @@ Paste==3.4.3 PasteDeploy==2.1.0 Persistence==3.0 Products.BTreeFolder2==4.2 -Products.ZCatalog==5.1 +Products.ZCatalog==5.2 Record==3.5 RestrictedPython==5.0 WSGIProxy2==0.4.6 diff --git a/sources.cfg b/sources.cfg index 2ad4d765e3..ecb258989d 100644 --- a/sources.cfg +++ b/sources.cfg @@ -4,7 +4,7 @@ github_push = git@github.com:zopefoundation [sources] # Zope-specific -AccessControl = git ${remotes:github}/AccessControl pushurl=${remotes:github_push}/AccessControl +AccessControl = git ${remotes:github}/AccessControl pushurl=${remotes:github_push}/AccessControl branch=4.x Acquisition = git ${remotes:github}/Acquisition pushurl=${remotes:github_push}/Acquisition AuthEncoding = git ${remotes:github}/AuthEncoding pushurl=${remotes:github_push}/AuthEncoding DateTime = git ${remotes:github}/DateTime pushurl=${remotes:github_push}/DateTime @@ -23,7 +23,7 @@ Missing = git ${remotes:github}/Missing pushurl=${remotes:github_push}/Missing Products.BTreeFolder2 = git ${remotes:github}/Products.BTreeFolder2 pushurl=${remotes:github_push}/Products.BTreeFolder2 Products.MailHost = git ${remotes:github}/Products.MailHost pushurl=${remotes:github_push}/Products.MailHost Products.PythonScripts = git ${remotes:github}/Products.PythonScripts pushurl=${remotes:github_push}/Products.PythonScripts -Products.ZCatalog = git ${remotes:github}/Products.ZCatalog pushurl=${remotes:github_push}/Products.ZCatalog +Products.ZCatalog = git ${remotes:github}/Products.ZCatalog pushurl=${remotes:github_push}/Products.ZCatalog branch=5.x Products.Sessions = git ${remotes:github}/Products.Sessions pushurl=${remotes:github_push}/Products.Sessions Products.TemporaryFolder = git ${remotes:github}/Products.TemporaryFolder pushurl=${remotes:github_push}/Products.TemporaryFolder Products.ZODBMountPoint = git ${remotes:github}/Products.ZODBMountPoint pushurl=${remotes:github_push}/Products.ZODBMountPoint diff --git a/versions-prod.cfg b/versions-prod.cfg index 716be18f01..a7d6a6e70e 100644 --- a/versions-prod.cfg +++ b/versions-prod.cfg @@ -7,7 +7,6 @@ Zope2 = 4.0 # AccessControl 5+ no longer supports Zope 4. AccessControl = 4.2 Acquisition = 4.6 -# AuthEncoding 5+ requires Python 3 AuthEncoding = 4.2 BTrees = 4.7.2 Chameleon = 3.8.1 @@ -21,7 +20,7 @@ PasteDeploy = 2.1.0 Persistence = 3.0 Products.BTreeFolder2 = 4.2 # ZCatalog 6+ no longer supports Zope 4. -Products.ZCatalog = 5.1 +Products.ZCatalog = 5.2 Record = 3.5 RestrictedPython = 5.0 WSGIProxy2 = 0.4.6