/
guppy_plugin.py
42 lines (32 loc) · 1.14 KB
/
guppy_plugin.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
from base import Plugin
import csv
import logging
from guppy import hpy
LOG = logging.getLogger("crawler")
class Heap(Plugin):
"""
Calculate heap consumed before and after request
"""
def __init__(self, write_csv=False):
super(Heap, self).__init__()
self.heap_urls = self.data['heap_urls'] = {}
self.hp = hpy()
self.write_csv = write_csv
if self.write_csv:
self.csv_writer = csv.writer(open('heap.csv', 'w'))
def pre_request(self, sender, **kwargs):
url = kwargs['url']
self.hp.setrelheap()
def post_request(self, sender, **kwargs):
url = kwargs['url']
heap = self.hp.heap()
self.heap_urls[url]=heap.size
LOG.debug("%s: heap consumed: %s", url, self.heap_urls[url])
if self.write_csv:
self.csv_writer.writerow([url, heap.size])
def finish_run(self, sender, **kwargs):
"Print the most heap consumed by a view"
alist = sorted(self.heap_urls.iteritems(), key=lambda (k,v): (v,k), reverse=True)
for url, mem in alist[:10]:
LOG.info("%s: %f heap" % (url, mem))
PLUGIN = Heap