Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow to generate more informative profile file name #638

Merged
merged 2 commits into from
Jan 22, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 5 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,11 @@ SILKY_PYTHON_PROFILER_RESULT_PATH = '/path/to/profiles/'

A download button will become available with a binary `.prof` file for every request. This file can be used for further analysis using [snakeviz](https://github.com/jiffyclub/snakeviz) or other cProfile tools

To retrieve which endpoint generates a specific profile file it is possible to add a stub of the request path in the file name with the following:

```python
SILKY_PYTHON_PROFILER_EXTENDED_FILE_NAME = True
```

Silk can also be used to profile specific blocks of code/functions. It provides a decorator and a context
manager for this purpose.
Expand Down
37 changes: 37 additions & 0 deletions project/tests/test_collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from tests.util import DictStorage

from silk.collector import DataCollector
from silk.config import SilkyConfig

from .factories import RequestMinFactory

Expand Down Expand Up @@ -45,3 +46,39 @@ def test_finalise(self):
content = f.read()
self.assertTrue(content)
self.assertGreater(len(content), 0)

def test_profile_file_name_with_disabled_extended_file_name(self):
SilkyConfig().SILKY_PYTHON_PROFILER_EXTENDED_FILE_NAME = False
request_path = 'normal/uri/'
resulting_prefix = self._get_prof_file_name(request_path)
self.assertEqual(resulting_prefix, '')

def test_profile_file_name_with_enabled_extended_file_name(self):

SilkyConfig().SILKY_PYTHON_PROFILER_EXTENDED_FILE_NAME = True
request_path = 'normal/uri/'
resulting_prefix = self._get_prof_file_name(request_path)
self.assertEqual(resulting_prefix, 'normal_uri_')

def test_profile_file_name_with_path_traversal_and_special_char(self):
SilkyConfig().SILKY_PYTHON_PROFILER_EXTENDED_FILE_NAME = True
request_path = 'spÉciàl/.././大/uri/@É/'
resulting_prefix = self._get_prof_file_name(request_path)
self.assertEqual(resulting_prefix, 'special_uri_e_')

def test_profile_file_name_with_long_path(self):
SilkyConfig().SILKY_PYTHON_PROFILER_EXTENDED_FILE_NAME = True
request_path = 'long/path/' + 'a' * 100
resulting_prefix = self._get_prof_file_name(request_path)
# the path is limited to 50 char plus the last `_`
self.assertEqual(len(resulting_prefix), 51)

@classmethod
def _get_prof_file_name(cls, request_path: str) -> str:
request = RequestMinFactory()
request.path = request_path
DataCollector().configure(request)
DataCollector().finalise()
file_path = DataCollector().request.prof_file.name
filename = file_path.rsplit('/')[-1]
return filename.replace(f"{request.id}.prof", "")
31 changes: 30 additions & 1 deletion silk/collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
import logging
import marshal
import pstats
import re
import unicodedata
from io import StringIO
from threading import local

Expand Down Expand Up @@ -143,7 +145,8 @@ def finalise(self):
self.request.pyprofile = profile_text

if SilkyConfig().SILKY_PYTHON_PROFILER_BINARY:
file_name = self.request.prof_file.storage.get_available_name(f"{str(self.request.id)}.prof")
proposed_file_name = self._get_proposed_file_name()
file_name = self.request.prof_file.storage.get_available_name(proposed_file_name)
with self.request.prof_file.storage.open(file_name, 'w+b') as f:
marshal.dump(ps.stats, f)
self.request.prof_file = f.name
Expand Down Expand Up @@ -189,3 +192,29 @@ def finalise(self):

def register_silk_query(self, *args):
self.register_objects(TYP_SILK_QUERIES, *args)

def _get_proposed_file_name(self) -> str:
"""Retrieve the profile file name to be proposed to the storage"""

if SilkyConfig().SILKY_PYTHON_PROFILER_EXTENDED_FILE_NAME:
slugified_path = slugify_path(self.request.path)
return f"{slugified_path}_{str(self.request.id)}.prof"
return f"{str(self.request.id)}.prof"


def slugify_path(request_path: str) -> str:
"""
Convert any characters not included in [a-zA-Z0-9_]) with a single underscore.
Convert to lowercase. Also strip any leading and trailing char that are not in the
accepted list

Inspired from django slugify
"""
request_path = str(request_path)
request_path = (
unicodedata.normalize("NFKD", request_path)
.encode("ascii", "ignore")
.decode("ascii")
)
request_path = request_path.lower()[:50]
return re.sub(r'\W+', '_', request_path).strip('_')
1 change: 1 addition & 0 deletions silk/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ class SilkyConfig(metaclass=Singleton):
'SILKY_PYTHON_PROFILER': False,
'SILKY_PYTHON_PROFILER_FUNC': None,
'SILKY_STORAGE_CLASS': 'silk.storage.ProfilerResultStorage',
'SILKY_PYTHON_PROFILER_EXTENDED_FILE_NAME': False,
'SILKY_MIDDLEWARE_CLASS': 'silk.middleware.SilkyMiddleware',
'SILKY_JSON_ENSURE_ASCII': True,
'SILKY_ANALYZE_QUERIES': False,
Expand Down