Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Wrap hook functions with GIL, add example.

Wraps the SetHook and calls to the hook with the GIL, to prevent races.
Adds an example of using the interface for callbacks into python code.
  • Loading branch information...
commit 5b5a0f4999dfac66c9c27160737352c727a3517b 1 parent 3237093
@thouis thouis authored
View
33 numpy/core/src/multiarray/multiarraymodule.c
@@ -3672,22 +3672,29 @@ NPY_NO_EXPORT void *_PyDataMem_eventhook_user_data;
* Returns a pointer to the previous hook or NULL. If old_data is
* non-NULL, the previous user_data pointer will be copied to it.
*
- *
* If not NULL, hook will be called at the end of each PyDataMem_NEW/FREE/RENEW:
* result = PyDataMem_NEW(size) -> (*hook)(NULL, result, size, user_data)
* PyDataMem_FREE(ptr) -> (*hook)(ptr, NULL, 0, user_data)
* result = PyDataMem_RENEW(ptr, size) -> (*hook)(ptr, result, size, user_data)
+ *
+ * When the hook is called, the GIL will be held by the calling
+ * thread. The hook should be written to be reentrant, if it performs
+ * operations that might cause new allocation events (such as the
+ * creation/descruction numpy objects, or creating/destroying Python
+ * objects which might cause a gc)
*/
NPY_NO_EXPORT PyDataMem_EventHookFunc *
PyDataMem_SetEventHook(PyDataMem_EventHookFunc *newhook,
void *user_data, void **old_data)
{
+ PyGILState_STATE gilstate = PyGILState_Ensure();
PyDataMem_EventHookFunc *temp = _PyDataMem_eventhook;
_PyDataMem_eventhook = newhook;
if (old_data != NULL) {
*old_data = _PyDataMem_eventhook_user_data;
}
_PyDataMem_eventhook_user_data = user_data;
+ PyGILState_Release(gilstate);
return temp;
}
@@ -3701,8 +3708,12 @@ PyDataMem_NEW(size_t size)
result = malloc(size);
if (_PyDataMem_eventhook != NULL) {
- (*_PyDataMem_eventhook)(NULL, result, size,
- _PyDataMem_eventhook_user_data);
+ PyGILState_STATE gilstate = PyGILState_Ensure();
+ if (_PyDataMem_eventhook != NULL) {
+ (*_PyDataMem_eventhook)(NULL, result, size,
+ _PyDataMem_eventhook_user_data);
+ }
+ PyGILState_Release(gilstate);
}
return (char *)result;
}
@@ -3715,8 +3726,12 @@ PyDataMem_FREE(void *ptr)
{
free(ptr);
if (_PyDataMem_eventhook != NULL) {
- (*_PyDataMem_eventhook)(ptr, NULL, 0,
- _PyDataMem_eventhook_user_data);
+ PyGILState_STATE gilstate = PyGILState_Ensure();
+ if (_PyDataMem_eventhook != NULL) {
+ (*_PyDataMem_eventhook)(ptr, NULL, 0,
+ _PyDataMem_eventhook_user_data);
+ }
+ PyGILState_Release(gilstate);
}
}
@@ -3730,8 +3745,12 @@ PyDataMem_RENEW(void *ptr, size_t size)
result = realloc(ptr, size);
if (_PyDataMem_eventhook != NULL) {
- (*_PyDataMem_eventhook)(ptr, result, size,
- _PyDataMem_eventhook_user_data);
+ PyGILState_STATE gilstate = PyGILState_Ensure();
+ if (_PyDataMem_eventhook != NULL) {
+ (*_PyDataMem_eventhook)(ptr, result, size,
+ _PyDataMem_eventhook_user_data);
+ }
+ PyGILState_Release(gilstate);
}
return (char *)result;
}
View
42 tools/allocation_tracking/alloc_hook.pyx
@@ -0,0 +1,42 @@
+# A cython wrapper for using python functions as callbacks for
+# PyDataMem_SetEventHook.
+
+cimport numpy as np
+
+cdef extern from "Python.h":
+ object PyLong_FromVoidPtr(void *)
+ void *PyLong_AsVoidPtr(object)
+
+ctypedef void PyDataMem_EventHookFunc(void *inp, void *outp, size_t size,
+ void *user_data)
+cdef extern from "numpy/arrayobject.h":
+ PyDataMem_EventHookFunc * \
+ PyDataMem_SetEventHook(PyDataMem_EventHookFunc *newhook,
+ void *user_data, void **old_data)
+
+np.import_array()
+
+cdef void pyhook(void *old, void *new, size_t size, void *user_data):
+ cdef object pyfunc = <object> user_data
+ pyfunc(PyLong_FromVoidPtr(old),
+ PyLong_FromVoidPtr(new),
+ size)
+
+class NumpyAllocHook(object):
+ def __init__(self, callback):
+ self.callback = callback
+
+ def __enter__(self):
+ cdef void *old_hook, *old_data
+ old_hook = <void *> \
+ PyDataMem_SetEventHook(<PyDataMem_EventHookFunc *> pyhook,
+ <void *> self.callback,
+ <void **> &old_data)
+ self.old_hook = PyLong_FromVoidPtr(old_hook)
+ self.old_data = PyLong_FromVoidPtr(old_data)
+
+ def __exit__(self):
+ PyDataMem_SetEventHook(<PyDataMem_EventHookFunc *> \
+ PyLong_AsVoidPtr(self.old_hook),
+ <void *> PyLong_AsVoidPtr(self.old_data),
+ <void **> 0)
View
9 tools/allocation_tracking/setup.py
@@ -0,0 +1,9 @@
+from distutils.core import setup
+from distutils.extension import Extension
+from Cython.Distutils import build_ext
+import numpy
+
+setup(
+ cmdclass = {'build_ext': build_ext},
+ ext_modules = [Extension("alloc_hook", ["alloc_hook.pyx"],
+ include_dirs=[numpy.get_include()])])
View
493 tools/allocation_tracking/sorttable.js
@@ -0,0 +1,493 @@
+/*
+ SortTable
+ version 2
+ 7th April 2007
+ Stuart Langridge, http://www.kryogenix.org/code/browser/sorttable/
+
+ Instructions:
+ Download this file
+ Add <script src="sorttable.js"></script> to your HTML
+ Add class="sortable" to any table you'd like to make sortable
+ Click on the headers to sort
+
+ Thanks to many, many people for contributions and suggestions.
+ Licenced as X11: http://www.kryogenix.org/code/browser/licence.html
+ This basically means: do what you want with it.
+*/
+
+
+var stIsIE = /*@cc_on!@*/false;
+
+sorttable = {
+ init: function() {
+ // quit if this function has already been called
+ if (arguments.callee.done) return;
+ // flag this function so we don't do the same thing twice
+ arguments.callee.done = true;
+ // kill the timer
+ if (_timer) clearInterval(_timer);
+
+ if (!document.createElement || !document.getElementsByTagName) return;
+
+ sorttable.DATE_RE = /^(\d\d?)[\/\.-](\d\d?)[\/\.-]((\d\d)?\d\d)$/;
+
+ forEach(document.getElementsByTagName('table'), function(table) {
+ if (table.className.search(/\bsortable\b/) != -1) {
+ sorttable.makeSortable(table);
+ }
+ });
+
+ },
+
+ makeSortable: function(table) {
+ if (table.getElementsByTagName('thead').length == 0) {
+ // table doesn't have a tHead. Since it should have, create one and
+ // put the first table row in it.
+ the = document.createElement('thead');
+ the.appendChild(table.rows[0]);
+ table.insertBefore(the,table.firstChild);
+ }
+ // Safari doesn't support table.tHead, sigh
+ if (table.tHead == null) table.tHead = table.getElementsByTagName('thead')[0];
+
+ if (table.tHead.rows.length != 1) return; // can't cope with two header rows
+
+ // Sorttable v1 put rows with a class of "sortbottom" at the bottom (as
+ // "total" rows, for example). This is B&R, since what you're supposed
+ // to do is put them in a tfoot. So, if there are sortbottom rows,
+ // for backwards compatibility, move them to tfoot (creating it if needed).
+ sortbottomrows = [];
+ for (var i=0; i<table.rows.length; i++) {
+ if (table.rows[i].className.search(/\bsortbottom\b/) != -1) {
+ sortbottomrows[sortbottomrows.length] = table.rows[i];
+ }
+ }
+ if (sortbottomrows) {
+ if (table.tFoot == null) {
+ // table doesn't have a tfoot. Create one.
+ tfo = document.createElement('tfoot');
+ table.appendChild(tfo);
+ }
+ for (var i=0; i<sortbottomrows.length; i++) {
+ tfo.appendChild(sortbottomrows[i]);
+ }
+ delete sortbottomrows;
+ }
+
+ // work through each column and calculate its type
+ headrow = table.tHead.rows[0].cells;
+ for (var i=0; i<headrow.length; i++) {
+ // manually override the type with a sorttable_type attribute
+ if (!headrow[i].className.match(/\bsorttable_nosort\b/)) { // skip this col
+ mtch = headrow[i].className.match(/\bsorttable_([a-z0-9]+)\b/);
+ if (mtch) { override = mtch[1]; }
+ if (mtch && typeof sorttable["sort_"+override] == 'function') {
+ headrow[i].sorttable_sortfunction = sorttable["sort_"+override];
+ } else {
+ headrow[i].sorttable_sortfunction = sorttable.guessType(table,i);
+ }
+ // make it clickable to sort
+ headrow[i].sorttable_columnindex = i;
+ headrow[i].sorttable_tbody = table.tBodies[0];
+ dean_addEvent(headrow[i],"click", function(e) {
+
+ if (this.className.search(/\bsorttable_sorted\b/) != -1) {
+ // if we're already sorted by this column, just
+ // reverse the table, which is quicker
+ sorttable.reverse(this.sorttable_tbody);
+ this.className = this.className.replace('sorttable_sorted',
+ 'sorttable_sorted_reverse');
+ this.removeChild(document.getElementById('sorttable_sortfwdind'));
+ sortrevind = document.createElement('span');
+ sortrevind.id = "sorttable_sortrevind";
+ sortrevind.innerHTML = stIsIE ? '&nbsp<font face="webdings">5</font>' : '&nbsp;&#x25B4;';
+ this.appendChild(sortrevind);
+ return;
+ }
+ if (this.className.search(/\bsorttable_sorted_reverse\b/) != -1) {
+ // if we're already sorted by this column in reverse, just
+ // re-reverse the table, which is quicker
+ sorttable.reverse(this.sorttable_tbody);
+ this.className = this.className.replace('sorttable_sorted_reverse',
+ 'sorttable_sorted');
+ this.removeChild(document.getElementById('sorttable_sortrevind'));
+ sortfwdind = document.createElement('span');
+ sortfwdind.id = "sorttable_sortfwdind";
+ sortfwdind.innerHTML = stIsIE ? '&nbsp<font face="webdings">6</font>' : '&nbsp;&#x25BE;';
+ this.appendChild(sortfwdind);
+ return;
+ }
+
+ // remove sorttable_sorted classes
+ theadrow = this.parentNode;
+ forEach(theadrow.childNodes, function(cell) {
+ if (cell.nodeType == 1) { // an element
+ cell.className = cell.className.replace('sorttable_sorted_reverse','');
+ cell.className = cell.className.replace('sorttable_sorted','');
+ }
+ });
+ sortfwdind = document.getElementById('sorttable_sortfwdind');
+ if (sortfwdind) { sortfwdind.parentNode.removeChild(sortfwdind); }
+ sortrevind = document.getElementById('sorttable_sortrevind');
+ if (sortrevind) { sortrevind.parentNode.removeChild(sortrevind); }
+
+ this.className += ' sorttable_sorted';
+ sortfwdind = document.createElement('span');
+ sortfwdind.id = "sorttable_sortfwdind";
+ sortfwdind.innerHTML = stIsIE ? '&nbsp<font face="webdings">6</font>' : '&nbsp;&#x25BE;';
+ this.appendChild(sortfwdind);
+
+ // build an array to sort. This is a Schwartzian transform thing,
+ // i.e., we "decorate" each row with the actual sort key,
+ // sort based on the sort keys, and then put the rows back in order
+ // which is a lot faster because you only do getInnerText once per row
+ row_array = [];
+ col = this.sorttable_columnindex;
+ rows = this.sorttable_tbody.rows;
+ for (var j=0; j<rows.length; j++) {
+ row_array[row_array.length] = [sorttable.getInnerText(rows[j].cells[col]), rows[j]];
+ }
+ /* If you want a stable sort, uncomment the following line */
+ //sorttable.shaker_sort(row_array, this.sorttable_sortfunction);
+ /* and comment out this one */
+ row_array.sort(this.sorttable_sortfunction);
+
+ tb = this.sorttable_tbody;
+ for (var j=0; j<row_array.length; j++) {
+ tb.appendChild(row_array[j][1]);
+ }
+
+ delete row_array;
+ });
+ }
+ }
+ },
+
+ guessType: function(table, column) {
+ // guess the type of a column based on its first non-blank row
+ sortfn = sorttable.sort_alpha;
+ for (var i=0; i<table.tBodies[0].rows.length; i++) {
+ text = sorttable.getInnerText(table.tBodies[0].rows[i].cells[column]);
+ if (text != '') {
+ if (text.match(/^-?[£$¤]?[\d,.]+%?$/)) {
+ return sorttable.sort_numeric;
+ }
+ // check for a date: dd/mm/yyyy or dd/mm/yy
+ // can have / or . or - as separator
+ // can be mm/dd as well
+ possdate = text.match(sorttable.DATE_RE)
+ if (possdate) {
+ // looks like a date
+ first = parseInt(possdate[1]);
+ second = parseInt(possdate[2]);
+ if (first > 12) {
+ // definitely dd/mm
+ return sorttable.sort_ddmm;
+ } else if (second > 12) {
+ return sorttable.sort_mmdd;
+ } else {
+ // looks like a date, but we can't tell which, so assume
+ // that it's dd/mm (English imperialism!) and keep looking
+ sortfn = sorttable.sort_ddmm;
+ }
+ }
+ }
+ }
+ return sortfn;
+ },
+
+ getInnerText: function(node) {
+ // gets the text we want to use for sorting for a cell.
+ // strips leading and trailing whitespace.
+ // this is *not* a generic getInnerText function; it's special to sorttable.
+ // for example, you can override the cell text with a customkey attribute.
+ // it also gets .value for <input> fields.
+
+ hasInputs = (typeof node.getElementsByTagName == 'function') &&
+ node.getElementsByTagName('input').length;
+
+ if (node.getAttribute("sorttable_customkey") != null) {
+ return node.getAttribute("sorttable_customkey");
+ }
+ else if (typeof node.textContent != 'undefined' && !hasInputs) {
+ return node.textContent.replace(/^\s+|\s+$/g, '');
+ }
+ else if (typeof node.innerText != 'undefined' && !hasInputs) {
+ return node.innerText.replace(/^\s+|\s+$/g, '');
+ }
+ else if (typeof node.text != 'undefined' && !hasInputs) {
+ return node.text.replace(/^\s+|\s+$/g, '');
+ }
+ else {
+ switch (node.nodeType) {
+ case 3:
+ if (node.nodeName.toLowerCase() == 'input') {
+ return node.value.replace(/^\s+|\s+$/g, '');
+ }
+ case 4:
+ return node.nodeValue.replace(/^\s+|\s+$/g, '');
+ break;
+ case 1:
+ case 11:
+ var innerText = '';
+ for (var i = 0; i < node.childNodes.length; i++) {
+ innerText += sorttable.getInnerText(node.childNodes[i]);
+ }
+ return innerText.replace(/^\s+|\s+$/g, '');
+ break;
+ default:
+ return '';
+ }
+ }
+ },
+
+ reverse: function(tbody) {
+ // reverse the rows in a tbody
+ newrows = [];
+ for (var i=0; i<tbody.rows.length; i++) {
+ newrows[newrows.length] = tbody.rows[i];
+ }
+ for (var i=newrows.length-1; i>=0; i--) {
+ tbody.appendChild(newrows[i]);
+ }
+ delete newrows;
+ },
+
+ /* sort functions
+ each sort function takes two parameters, a and b
+ you are comparing a[0] and b[0] */
+ sort_numeric: function(a,b) {
+ aa = parseFloat(a[0].replace(/[^0-9.-]/g,''));
+ if (isNaN(aa)) aa = 0;
+ bb = parseFloat(b[0].replace(/[^0-9.-]/g,''));
+ if (isNaN(bb)) bb = 0;
+ return aa-bb;
+ },
+ sort_alpha: function(a,b) {
+ if (a[0]==b[0]) return 0;
+ if (a[0]<b[0]) return -1;
+ return 1;
+ },
+ sort_ddmm: function(a,b) {
+ mtch = a[0].match(sorttable.DATE_RE);
+ y = mtch[3]; m = mtch[2]; d = mtch[1];
+ if (m.length == 1) m = '0'+m;
+ if (d.length == 1) d = '0'+d;
+ dt1 = y+m+d;
+ mtch = b[0].match(sorttable.DATE_RE);
+ y = mtch[3]; m = mtch[2]; d = mtch[1];
+ if (m.length == 1) m = '0'+m;
+ if (d.length == 1) d = '0'+d;
+ dt2 = y+m+d;
+ if (dt1==dt2) return 0;
+ if (dt1<dt2) return -1;
+ return 1;
+ },
+ sort_mmdd: function(a,b) {
+ mtch = a[0].match(sorttable.DATE_RE);
+ y = mtch[3]; d = mtch[2]; m = mtch[1];
+ if (m.length == 1) m = '0'+m;
+ if (d.length == 1) d = '0'+d;
+ dt1 = y+m+d;
+ mtch = b[0].match(sorttable.DATE_RE);
+ y = mtch[3]; d = mtch[2]; m = mtch[1];
+ if (m.length == 1) m = '0'+m;
+ if (d.length == 1) d = '0'+d;
+ dt2 = y+m+d;
+ if (dt1==dt2) return 0;
+ if (dt1<dt2) return -1;
+ return 1;
+ },
+
+ shaker_sort: function(list, comp_func) {
+ // A stable sort function to allow multi-level sorting of data
+ // see: http://en.wikipedia.org/wiki/Cocktail_sort
+ // thanks to Joseph Nahmias
+ var b = 0;
+ var t = list.length - 1;
+ var swap = true;
+
+ while(swap) {
+ swap = false;
+ for(var i = b; i < t; ++i) {
+ if ( comp_func(list[i], list[i+1]) > 0 ) {
+ var q = list[i]; list[i] = list[i+1]; list[i+1] = q;
+ swap = true;
+ }
+ } // for
+ t--;
+
+ if (!swap) break;
+
+ for(var i = t; i > b; --i) {
+ if ( comp_func(list[i], list[i-1]) < 0 ) {
+ var q = list[i]; list[i] = list[i-1]; list[i-1] = q;
+ swap = true;
+ }
+ } // for
+ b++;
+
+ } // while(swap)
+ }
+}
+
+/* ******************************************************************
+ Supporting functions: bundled here to avoid depending on a library
+ ****************************************************************** */
+
+// Dean Edwards/Matthias Miller/John Resig
+
+/* for Mozilla/Opera9 */
+if (document.addEventListener) {
+ document.addEventListener("DOMContentLoaded", sorttable.init, false);
+}
+
+/* for Internet Explorer */
+/*@cc_on @*/
+/*@if (@_win32)
+ document.write("<script id=__ie_onload defer src=javascript:void(0)><\/script>");
+ var script = document.getElementById("__ie_onload");
+ script.onreadystatechange = function() {
+ if (this.readyState == "complete") {
+ sorttable.init(); // call the onload handler
+ }
+ };
+/*@end @*/
+
+/* for Safari */
+if (/WebKit/i.test(navigator.userAgent)) { // sniff
+ var _timer = setInterval(function() {
+ if (/loaded|complete/.test(document.readyState)) {
+ sorttable.init(); // call the onload handler
+ }
+ }, 10);
+}
+
+/* for other browsers */
+window.onload = sorttable.init;
+
+// written by Dean Edwards, 2005
+// with input from Tino Zijdel, Matthias Miller, Diego Perini
+
+// http://dean.edwards.name/weblog/2005/10/add-event/
+
+function dean_addEvent(element, type, handler) {
+ if (element.addEventListener) {
+ element.addEventListener(type, handler, false);
+ } else {
+ // assign each event handler a unique ID
+ if (!handler.$$guid) handler.$$guid = dean_addEvent.guid++;
+ // create a hash table of event types for the element
+ if (!element.events) element.events = {};
+ // create a hash table of event handlers for each element/event pair
+ var handlers = element.events[type];
+ if (!handlers) {
+ handlers = element.events[type] = {};
+ // store the existing event handler (if there is one)
+ if (element["on" + type]) {
+ handlers[0] = element["on" + type];
+ }
+ }
+ // store the event handler in the hash table
+ handlers[handler.$$guid] = handler;
+ // assign a global event handler to do all the work
+ element["on" + type] = handleEvent;
+ }
+};
+// a counter used to create unique IDs
+dean_addEvent.guid = 1;
+
+function removeEvent(element, type, handler) {
+ if (element.removeEventListener) {
+ element.removeEventListener(type, handler, false);
+ } else {
+ // delete the event handler from the hash table
+ if (element.events && element.events[type]) {
+ delete element.events[type][handler.$$guid];
+ }
+ }
+};
+
+function handleEvent(event) {
+ var returnValue = true;
+ // grab the event object (IE uses a global event object)
+ event = event || fixEvent(((this.ownerDocument || this.document || this).parentWindow || window).event);
+ // get a reference to the hash table of event handlers
+ var handlers = this.events[event.type];
+ // execute each event handler
+ for (var i in handlers) {
+ this.$$handleEvent = handlers[i];
+ if (this.$$handleEvent(event) === false) {
+ returnValue = false;
+ }
+ }
+ return returnValue;
+};
+
+function fixEvent(event) {
+ // add W3C standard event methods
+ event.preventDefault = fixEvent.preventDefault;
+ event.stopPropagation = fixEvent.stopPropagation;
+ return event;
+};
+fixEvent.preventDefault = function() {
+ this.returnValue = false;
+};
+fixEvent.stopPropagation = function() {
+ this.cancelBubble = true;
+}
+
+// Dean's forEach: http://dean.edwards.name/base/forEach.js
+/*
+ forEach, version 1.0
+ Copyright 2006, Dean Edwards
+ License: http://www.opensource.org/licenses/mit-license.php
+*/
+
+// array-like enumeration
+if (!Array.forEach) { // mozilla already supports this
+ Array.forEach = function(array, block, context) {
+ for (var i = 0; i < array.length; i++) {
+ block.call(context, array[i], i, array);
+ }
+ };
+}
+
+// generic enumeration
+Function.prototype.forEach = function(object, block, context) {
+ for (var key in object) {
+ if (typeof this.prototype[key] == "undefined") {
+ block.call(context, object[key], key, object);
+ }
+ }
+};
+
+// character enumeration
+String.forEach = function(string, block, context) {
+ Array.forEach(string.split(""), function(chr, index) {
+ block.call(context, chr, index, string);
+ });
+};
+
+// globally resolve forEach enumeration
+var forEach = function(object, block, context) {
+ if (object) {
+ var resolve = Object; // default
+ if (object instanceof Function) {
+ // functions have a "length" property
+ resolve = Function;
+ } else if (object.forEach instanceof Function) {
+ // the object implements a custom forEach method so use that
+ object.forEach(block, context);
+ return;
+ } else if (typeof object == "string") {
+ // the object is a string
+ resolve = String;
+ } else if (typeof object.length == "number") {
+ // the object is array-like
+ resolve = Array;
+ }
+ resolve.forEach(object, block, context);
+ }
+};
+
View
131 tools/allocation_tracking/track_allocations.py
@@ -0,0 +1,131 @@
+import numpy as np
+import inspect
+from alloc_hook import NumpyAllocHook
+
+class AllocationTracker(object):
+ def __init__(self, threshold=0):
+ '''track numpy allocations of size threshold bytes or more.'''
+
+ self.threshold = threshold
+
+ # The total number of bytes currently allocated with size above
+ # threshold
+ self.total_bytes = 0
+
+ # We buffer requests line by line and move them into the allocation
+ # trace when a new line occurs
+ self.current_line = None
+ self.pending_allocations = []
+
+ self.blocksizes = {}
+
+ # list of (lineinfo, bytes allocated, bytes freed, # allocations, #
+ # frees, maximum memory usage, long-lived bytes allocated)
+ self.allocation_trace = []
+
+ self.numpy_hook = NumpyAllocHook(self.hook)
+
+ def __enter__(self):
+ self.numpy_hook.__enter__()
+
+ def __exit__(self, type, value, traceback):
+ self.check_line_changed() # forces pending events to be handled
+ self.numpy_hook.__exit__()
+
+ def hook(self, inptr, outptr, size):
+ if outptr == 0: # it's a free
+ self.free_cb(inptr)
+ elif inptr != 0: # realloc
+ self.realloc_cb(inptr, outptr, size)
+ else: # malloc
+ self.alloc_cb(outptr, size)
+
+ def alloc_cb(self, ptr, size):
+ if size >= self.threshold:
+ self.check_line_changed()
+ self.blocksizes[ptr] = size
+ self.pending_allocations.append(size)
+
+ def free_cb(self, ptr):
+ size = self.blocksizes.pop(ptr, 0)
+ if size:
+ self.check_line_changed()
+ self.pending_allocations.append(-size)
+
+ def realloc_cb(self, newptr, oldptr, size):
+ if (size >= self.threshold) or (oldptr in self.blocksizes):
+ self.check_line_changed()
+ oldsize = self.blocksizes.pop(oldptr, 0)
+ self.pending_allocations.append(size - oldsize)
+ self.blocksizes[newptr] = size
+
+ def get_code_line(self):
+ # first frame is this line, then check_line_changed(), then 2 callbacks,
+ # then actual code.
+ try:
+ return inspect.stack()[4][1:]
+ except:
+ return inspect.stack()[0][1:]
+
+ def check_line_changed(self):
+ line = self.get_code_line()
+ if line != self.current_line and (self.current_line is not None):
+ # move pending events into the allocation_trace
+ max_size = self.total_bytes
+ bytes_allocated = 0
+ bytes_freed = 0
+ num_allocations = 0
+ num_frees = 0
+ before_size = self.total_bytes
+ for allocation in self.pending_allocations:
+ self.total_bytes += allocation
+ if allocation > 0:
+ bytes_allocated += allocation
+ num_allocations += 1
+ else:
+ bytes_freed += -allocation
+ num_frees += 1
+ max_size = max(max_size, self.total_bytes)
+ long_lived = max(self.total_bytes - before_size, 0)
+ self.allocation_trace.append((self.current_line, bytes_allocated,
+ bytes_freed, num_allocations,
+ num_frees, max_size, long_lived))
+ # clear pending allocations
+ self.pending_allocations = []
+ # move to the new line
+ self.current_line = line
+
+ def write_html(self, filename):
+ f = open(filename, "w")
+ f.write('<HTML><HEAD><script src="sorttable.js"></script></HEAD><BODY>\n')
+ f.write('<TABLE class="sortable" width=100%>\n')
+ f.write("<TR>\n")
+ cols = "event#,lineinfo,bytes allocated,bytes freed,#allocations,#frees,max memory usage,long lived bytes".split(',')
+ for header in cols:
+ f.write(" <TH>{0}</TH>".format(header))
+ f.write("\n</TR>\n")
+ for idx, event in enumerate(self.allocation_trace):
+ f.write("<TR>\n")
+ event = [idx] + list(event)
+ for col, val in zip(cols, event):
+ if col == 'lineinfo':
+ # special handling
+ try:
+ filename, line, module, code, index = val
+ val = "{0}({1}): {2}".format(filename, line, code[index])
+ except:
+ # sometimes this info is not available (from eval()?)
+ val = str(val)
+ f.write(" <TD>{0}</TD>".format(val))
+ f.write("\n</TR>\n")
+ f.write("</TABLE></BODY></HTML>\n")
+ f.close()
+
+
+if __name__ == '__main__':
+ tracker = AllocationTracker(1000)
+ with tracker:
+ for i in range(100):
+ np.zeros(i * 100)
+ np.zeros(i * 200)
+ tracker.write_html("allocations.html")
Please sign in to comment.
Something went wrong with that request. Please try again.