Skip to content
Find file
Fetching contributors…
Cannot retrieve contributors at this time
110 lines (90 sloc) 3.35 KB
Conversion, validation, and size measurement utilities for Python's
built-in types.
import re
import sys
from math import log, log10, ceil
_postImportVars = vars().keys()
_64bitPy = sys.maxint > 2**31 - 1
_bytesPerWord = 8 if _64bitPy else 4
_gcHeaderSize = 24 if _64bitPy else 12 # Just a guess
_UCS4 = sys.maxunicode > 2**16 - 1
_bytesPerCodePoint = 4 if _UCS4 else 2
def basicGetSizeOf(obj):
Works like L{sys.getsizeof}, but only returns reasonable numbers for
a limited set of types: str, unicode, list, tuple, dict, set, frozenset,
bool, NoneType, int, float, long.
Many of these numbers are guesses. Don't use this if L{sys.getsizeof}
is available.
if isinstance(obj, str):
return _gcHeaderSize + len(obj)
elif isinstance(obj, unicode):
return _gcHeaderSize + _bytesPerCodePoint * len(obj)
elif isinstance(obj, int):
return _gcHeaderSize + _bytesPerWord
elif isinstance(obj, float):
return _gcHeaderSize + 8
elif isinstance(obj, long):
return _gcHeaderSize + int(ceil(log(obj, 2)))
elif isinstance(obj, (list, tuple)):
return _gcHeaderSize + _bytesPerWord + _bytesPerWord * len(obj)
elif isinstance(obj, dict):
return _gcHeaderSize + _bytesPerWord * (750 + 8 * len(obj))
elif isinstance(obj, (set, frozenset)):
return _gcHeaderSize + _bytesPerWord * (750 + 5 * len(obj))
return _gcHeaderSize + _bytesPerWord
# TODO: handle C{complex}es
from sys import getsizeof
except ImportError:
getsizeof = basicGetSizeOf
def totalSizeOf(obj, _alreadySeen=None):
Get the size of object C{obj} using L{sys.getsizeof} or L{basicGetSizeOf}
on the object itself and all of its children recursively. If the same
object appears more than once inside C{obj}, it is counted only once.
This only works properly if C{obj} is a str, unicode, list, tuple, dict,
set, frozenset, bool, NoneType, int, complex, float, long, or any nested
combination of the above. C{obj} is allowed to have circular references.
This is particularly useful for getting a good estimate of how much
memory a JSON-decoded object is using after receiving it.
Design notes: L{sys.getsizeof} or L{basicGetSizeOf} return reasonable
numbers, but do not recurse into the object's children. As we recurse
into the children, we keep track of objects we've already counted for two
- If we've already counted the object's memory usage, we don't
want to count it again.
- As a bonus, we handle circular references gracefully.
This function assumes that containers do not modify their children as
they are traversed.
If your Python is < 2.6, the returned size will be less accurate, because
L{basicGetSizeOf} is used instead of L{sys.getsizeof}.
if _alreadySeen is None:
_alreadySeen = set()
total = getsizeof(obj)
if isinstance(obj, dict):
# Count the memory usage of both the keys and values.
for k, v in obj.iteritems():
if not id(k) in _alreadySeen:
total += totalSizeOf(k, _alreadySeen)
if not id(v) in _alreadySeen:
total += totalSizeOf(v, _alreadySeen)
iterator = obj.__iter__()
except (TypeError, AttributeError):
for item in iterator:
if not id(item) in _alreadySeen:
total += totalSizeOf(item, _alreadySeen)
return total
try: from refbinder.api import bindRecursive
except ImportError: pass
else: bindRecursive(sys.modules[__name__], _postImportVars)
Jump to Line
Something went wrong with that request. Please try again.