Skip to content

Commit

Permalink
Merge pull request #18 from IMIO/MOD-910_centralize_caching_mechanism
Browse files Browse the repository at this point in the history
Mod 910 centralize caching mechanism
  • Loading branch information
sgeulette authored Aug 19, 2022
2 parents ee4ceae + c3e452a commit 089eabe
Show file tree
Hide file tree
Showing 26 changed files with 713 additions and 59 deletions.
21 changes: 21 additions & 0 deletions CHANGES.rst
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,29 @@ Changelog
0.62 (unreleased)
-----------------

- Added `IMIORAMCache` using `IMIOStorage` to extend used cache duration and
improve displayed statistics
[gbastien]
- Added cache on various acl methods following `decorate_acl_methods` env variable
[gbastien, sgeulette]
- Added IIMIOLayer BrowserLayer (need to execute upgrade step to 2).
[gbastien]
- Override `caching-controlpanel-ramcache` to compute totals for `Hits`, `Misses`,
`Size` and `Entries`, display `Older entry`, do not break to display statistics
when a pickle error occurs but add a portal message.
[gbastien]
- Added parameter `ttl=0` to `cache.get_cachekey_volatile` this way a date older
than given `ttl` (in seconds) will be recomputed.
- Added 'none_if_no_user' param in `content.get_user_fullname`.
[sgeulette]
- Always return unicode in `content.get_user_fullname`.
[sgeulette]
- Added `test_helpers.ImioTestHelpers` class with useful methods from iA.delib
[sgeulette]
- Added `vocabularies.SimplySortedUsers` and modified `vocabularies.SortedUsers`
[sgeulette]
- Added `cache.get_users_in_plone_groups`
[sgeulette]
- Added `setup.load_type_from_package` to reload a single type.
Moved `workflow.load_workflow_from_package` to `setup.load_workflow_from_package`.
[gbastien]
Expand Down
7 changes: 7 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -22,3 +22,10 @@ Requirements
The barcode generation method uses zint tool (https://sourceforge.net/projects/zint/).

You have to install zint version >= 2.6.0.


Caching
-------

Use cache.get_plone_groups_for_user(the_objects=True) instead portal_groups.getGroupsForPrincipal
Avoid portal_groups.getGroupById but use source_groups.get
6 changes: 5 additions & 1 deletion buildout.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ allow-picked-versions = true
[versions]
setuptools = 42.0.2
zc.buildout = 2.13.2
;prompt-toolkit = 3.0.19
#prompt-toolkit = 3.0.19
toml = 0.10.2
traitlets = 4.3.2
# Added by buildout at 2020-06-22 13:24:06.304460
Expand All @@ -28,6 +28,10 @@ pycodestyle = 2.6.0
pyflakes = 2.2.0
reportlab = 3.5.42
zipp = 1.2.0
z3c.unconfigure = 1.1
# Required by:
# z3c.unconfigure = 1.1
zope.configuration = 3.8.1
# a future version sufficient to get html.escape
future = 0.18.2

Expand Down
8 changes: 7 additions & 1 deletion ci.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,10 @@ parts +=

[createcoverage]
recipe = zc.recipe.egg
eggs = createcoverage
eggs = createcoverage

[versions]
z3c.unconfigure = 1.1
# Required by:
# z3c.unconfigure = 1.1
zope.configuration = 3.8.1
1 change: 1 addition & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
'plone.dexterity',
'setuptools',
'Plone',
'z3c.unconfigure',
],
extras_require={
'test': [
Expand Down
4 changes: 3 additions & 1 deletion sources.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ collective = https://github.com/collective
collective_push = git@github.com:collective
plone = https://github.com/plone
plone_push = git@github.com:plone
imio = https://github.com/imio
imio_push = git@github.com:imio

# ==================
# Ecreall developers
Expand All @@ -15,5 +17,5 @@ cedricmessiant = https://github.com/cedricmessiant
cedricmessiant_push = git@github.com:cedricmessiant

[sources]
appy = svn https://svn.forge.pallavi.be/appy-dev/dev0
appy = git ${remotes:imio}/appy.git pushurl=${remotes:imio_push}/appy.git
collective.behavior.talcondition = git ${remotes:collective}/collective.behavior.talcondition.git pushurl=${remotes:collective_push}/collective.behavior.talcondition.git
84 changes: 84 additions & 0 deletions src/imio/helpers/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,90 @@
# -*- coding: utf-8 -*-

from imio.helpers.cache import get_cachekey_volatile
from plone import api
from plone.memoize import ram as pmram
from Products.PlonePAS.plugins.role import GroupAwareRoleManager
from Products.PlonePAS.tools.groups import GroupsTool
from Products.PluggableAuthService.PluggableAuthService import PluggableAuthService
from zope.globalrequest import getRequest
from zope.i18nmessageid import MessageFactory

import logging
import os


_ = MessageFactory('imio.helpers')
logger = logging.getLogger('imio.helpers')


def GroupsTool__getGroupsForPrincipal_cachekey(method, self, principal):
req = getRequest()
if req is None:
raise pmram.DontCache
date = get_cachekey_volatile('_users_groups_value')
return date, principal and principal.getId()


def GroupAwareRoleManager__getRolesForPrincipal_cachekey(method, self, principal, request=None):
req = request or getRequest()
# if req is None:
# raise pmram.DontCache
date = get_cachekey_volatile('_users_groups_value')
return (date, principal and principal.getId(), repr(req), req and (req.get('__ignore_direct_roles__', False),
req.get('__ignore_group_roles__', False)) or (None, None))


def PluggableAuthService__getGroupsForPrincipal_cachekey(method, self, principal, request=None, **kwargs):
req = request or getRequest()
if req is None:
raise pmram.DontCache
try:
date = get_cachekey_volatile('_users_groups_value')
except api.portal.CannotGetPortalError:
raise pmram.DontCache
return date, principal and principal.getId()


def PluggableAuthService__findUser_cachekey(method, self, plugins, user_id, name=None, request=None):
req = request or getRequest()
if req is None:
raise pmram.DontCache
try:
date = get_cachekey_volatile('_users_groups_value')
except api.portal.CannotGetPortalError:
raise pmram.DontCache
return date, repr(plugins), user_id, name, str(req and req._debug or '')


def PluggableAuthService__verifyUser_cachekey(method, self, plugins, user_id=None, login=None):
req = getRequest()
if req is None:
raise pmram.DontCache

date = get_cachekey_volatile('_users_groups_value')
return date, repr(plugins), user_id, login


def GroupsTool_getGroupById_cachekey(method, self, group_id):
req = getRequest()
if req is None:
raise pmram.DontCache

date = get_cachekey_volatile('_users_groups_value')
return date, group_id


if os.getenv('decorate_acl_methods', 'Nope') in ('True', 'true'):
logger.info('DECORATING various acl related methods with cache')
decorator = pmram.cache(GroupsTool__getGroupsForPrincipal_cachekey)
GroupsTool.getGroupsForPrincipal = decorator(GroupsTool.getGroupsForPrincipal)
decorator = pmram.cache(GroupAwareRoleManager__getRolesForPrincipal_cachekey)
GroupAwareRoleManager.getRolesForPrincipal = decorator(GroupAwareRoleManager.getRolesForPrincipal)
decorator = pmram.cache(PluggableAuthService__getGroupsForPrincipal_cachekey)
PluggableAuthService._getGroupsForPrincipal = decorator(PluggableAuthService._getGroupsForPrincipal)
# decorator = pmram.cache(PluggableAuthService__findUser_cachekey)
# PluggableAuthService._findUser = decorator(PluggableAuthService._findUser)
# decorator = pmram.cache(PluggableAuthService__verifyUser_cachekey)
# PluggableAuthService._verifyUser = decorator(PluggableAuthService._verifyUser)
decorator = pmram.cache(GroupsTool_getGroupById_cachekey)
GroupsTool.getGroupById = decorator(GroupsTool.getGroupById)
9 changes: 9 additions & 0 deletions src/imio/helpers/browser/configure.zcml
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,13 @@
attribute="update_table"
permission="zope2.View" />

<!-- Override @@caching-controlpanel-ramcache -->
<browser:page
name="caching-controlpanel-ramcache"
for="Products.CMFPlone.interfaces.IPloneSiteRoot"
class=".views.IMIORAMCache"
template="ramcache.pt"
layer="imio.helpers.interfaces.IIMIOHelpersLayer"
permission="cmf.ManagePortal" />

</configure>
131 changes: 131 additions & 0 deletions src/imio/helpers/browser/ramcache.pt
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"
xmlns:tal="http://xml.zope.org/namespaces/tal"
xmlns:metal="http://xml.zope.org/namespaces/metal"
xmlns:i18n="http://xml.zope.org/namespaces/i18n"
lang="en"
metal:use-macro="context/prefs_main_template/macros/master"
i18n:domain="plone.app.caching">

<body>

<div metal:fill-slot="prefs_configlet_content">

<div id="region-content" class="documentEditable">

<div id="edit-bar">
<ul class="contentViews" id="content-views">
<li>
<a href=""
tal:attributes="href string:${portal_url}/@@caching-controlpanel"
i18n:translate="label_settings">Change settings</a>
</li>
<li>
<a href=""
tal:attributes="href string:${portal_url}/@@caching-controlpanel-import"
i18n:translate="label_import">Import settings</a>
</li>
<li tal:condition="view/purgingEnabled">
<a href=""
tal:attributes="href string:${portal_url}/@@caching-controlpanel-purge"
i18n:translate="label_purging">Purge caching proxy</a>
</li>
<li class="selected">
<a href=""
tal:attributes="href string:${portal_url}/@@caching-controlpanel-ramcache"
i18n:translate="label_ramcache">RAM cache</a>
</li>
</ul>
</div>

<div class="contentActions">
&#160;
</div>

<div class="documentContent" id="content">
<a name="documentContent"></a>

<div metal:use-macro="context/global_statusmessage/macros/portal_message">
Portal status message
</div>

<div class="configlet">

<h1 class="documentFirstHeading"
i18n:translate="heading_ramcache_stats">RAM cache statistics</h1>

<a href=""
class="link-parent"
tal:attributes="href string:${portal_url}/plone_control_panel"
i18n:translate="label_up_to_plone_setup">
Up to Site Setup
</a>

<p i18n:translate="description_ramcache_stats">
The table below shows statistics for the default RAM
cache. You can use the <em>Purge</em> button to manually
clear the cache if you suspect there are stale items there.
</p>

<table tal:define="stats view/stats"
class="listing faceted-table-results nosort" summary="RAM cache statistics"
i18n:attributes="summary heading_ramcache_stats;">
<thead>
<th>
<span i18n:translate="label_cache_key">Key</span><br />
<span>Totals</span>
</th>
<th>
<span i18n:translate="label_cache_hits">Hits</span><br />
<span tal:content="python: sum([data['hits'] for data in stats])">&nbsp;</span>
</th>
<th>
<span i18n:translate="label_cache_misses">Misses</span><br />
<span tal:content="python: sum([data['misses'] for data in stats])">&nbsp;</span>
</th>
<th>
<span i18n:translate="label_cache_size_bytes">Size (bytes)</span><br />
<span tal:content="python: sum([data['size'] for data in stats])">&nbsp;</span>
</th>
<th>
<span i18n:translate="label_cache_entries">Entries</span><br />
<span tal:content="python: sum([data['entries'] for data in stats])">&nbsp;</span>
</th>
<th>
<span i18n:translate="label_cache_entries">Older entry</span>
</th>
</thead>

<tbody>
<tr tal:repeat="data stats">
<td><span tal:content="data/path">&nbsp;</span></td>
<td><span tal:content="data/hits">&nbsp;</span></td>
<td><span tal:content="data/misses">&nbsp;</span></td>
<td><span tal:content="data/size">&nbsp;</span></td>
<td><span tal:content="data/entries">&nbsp;</span></td>
<td><span tal:content="python: data['older_date'] and data['older_date'].strftime('%Y/%m/%d %H:%M:%S')">&nbsp;</span></td>
</tr>
</tbody>
</table>

<form name="purge" tal:attributes="action string:${request/URL}" method="post"
tal:define="errors view/errors">

<div class="formControls">
<input
type="submit"
name="form.button.Purge"
class="destructive"
value="Purge"
i18n:attributes="value" />
</div>

<input tal:replace="structure context/@@authenticator/authenticator" />

</form>
</div>
</div>
</div>

</div>
</body>
</html>
10 changes: 10 additions & 0 deletions src/imio/helpers/browser/views.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
# -*- coding: utf-8 -*-

from imio.helpers.interfaces import IListContainedDexterityObjectsForDisplay
from plone import api
from plone.app.caching.browser.controlpanel import RAMCache
from plone.batching import Batch
from plone.dexterity.browser.view import DefaultView
from Products.Five import BrowserView
Expand Down Expand Up @@ -53,3 +55,11 @@ def authorname(self):

def update_table(self):
return self.context.restrictedTraverse('@@imio-folder-listing-table').index()


class IMIORAMCache(RAMCache):
""" """

def update(self):
super(IMIORAMCache, self).update()
self.stats = self.ramCache.getStatistics()
Loading

0 comments on commit 089eabe

Please sign in to comment.