From cc12387ce81f799075378030815680bbd9e105b4 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 16 Sep 2011 09:35:32 -0700 Subject: [PATCH 01/21] Update README to reflect new type size field --- README.rst | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 7bcc004..37c3378 100644 --- a/README.rst +++ b/README.rst @@ -94,11 +94,11 @@ Unbuffered structures have ff in the write buffer field. Buffered -------- -+----------------+----------------------+----------+------------------+-------------+ -| ``label size`` | ``label`` | ``type`` | ``write buffer`` | ``buffers`` | -+================+======================+==========+==================+=============+ -| ``ushort`` | ``label size chars`` | ``char`` | ``byte`` | ``varies`` | -+----------------+----------------------+----------+------------------+-------------+ ++----------------+------------+---------------+------------+------------------+-------------+ +| ``label size`` | ``label`` | ``type size`` | ``type`` | ``write buffer`` | ``buffers`` | ++================+============+===============+============+==================+=============+ +| ``ushort`` | ``char[]`` | ``ushort`` | ``char[]`` | ``byte`` | ``varies`` | ++----------------+------------+---------------+------------+------------------+-------------+ The buffers field length = sizeof(type) * buffers. @@ -110,10 +110,11 @@ TODO: field for total number of buffers? Unbuffered ---------- -+----------------+----------------------+----------+------------------+-------------+ -| ``label size`` | ``label`` | ``type`` | ``write buffer`` | ``value`` | -+================+======================+==========+==================+=============+ -| ``ushort`` | ``label size chars`` | ``char`` | ``ff`` | ``varies`` | -+----------------+----------------------+----------+------------------+-------------+ + ++----------------+------------+---------------+------------+------------------+-------------+ +| ``label size`` | ``label`` | ``type size`` | ``type`` | ``write buffer`` | ``value`` | ++================+============+===============+============+==================+=============+ +| ``ushort`` | ``char[]`` | ``ushort`` | ``char[]`` | ``ff`` | ``varies`` | ++----------------+------------+---------------+------------+------------------+-------------+ The value field length = sizeof(type). From bb604a6153b29eebb40ca240beb9d1a3947a6425 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Wed, 21 Sep 2011 00:00:21 -0500 Subject: [PATCH 02/21] Use ftruncate instead of write('\x00'*size) --- mmstats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmstats.py b/mmstats.py index 20c2614..6b05901 100644 --- a/mmstats.py +++ b/mmstats.py @@ -40,7 +40,7 @@ def _init_mmap(path=None, filename=None, size=PAGESIZE): size = PAGESIZE # Zero out the file - os.write(fd, '\x00' * size) + os.ftruncate(fd, size) m = mmap.mmap(fd, size, mmap.MAP_SHARED, mmap.PROT_WRITE) return (full_path, size, m) From 30f1156140a4a246a2090aa3e8d5183ceea0beed Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Wed, 21 Sep 2011 00:10:20 -0500 Subject: [PATCH 03/21] Add some more mmap related tests --- tests/test_mmap.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_mmap.py b/tests/test_mmap.py index 6045243..9af386e 100644 --- a/tests/test_mmap.py +++ b/tests/test_mmap.py @@ -19,3 +19,35 @@ def test_init_alt_name(self): path=self.path, filename='mmstats-test_init_alt_name') self.assertEqual(fn, expected_fn) self.assertTrue(os.path.exists(fn)) + + def test_size_adjusting1(self): + """mmapped files must be at least PAGESIZE in size""" + _, sz, m = mmstats._init_mmap(path=self.path, + filename='mmstats-test_size_adjusting-1', size=1) + + self.assertEqual(sz, mmstats.PAGESIZE) + self.assertEqual(m[:], '\x00' * mmstats.PAGESIZE) + + def test_size_adjusting2(self): + """mmapped files must be multiples of PAGESIZE""" + _, sz, m = mmstats._init_mmap( + path=self.path, + filename='mmstats-test_size_adjusting-2', + size=(mmstats.PAGESIZE+1) + ) + + self.assertEqual(sz, mmstats.PAGESIZE * 2) + self.assertEqual(m[:], '\x00' * mmstats.PAGESIZE * 2) + + def test_truncate(self): + """mmapped files must be initialized with null bytes""" + fn, sz, m = mmstats._init_mmap( + path=self.path, + filename='mmstats-test_truncate', + ) + + m[0] = 'X' + + reopened_file = open(fn) + self.assertEqual(reopened_file.read(1), 'X') + self.assertEqual(reopened_file.read(1), '\x00') From 7d0f8366b0c215f9d84dbd30cbbc6430994f0587 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Tue, 20 Sep 2011 22:20:15 -0700 Subject: [PATCH 04/21] Add a little helper script for the flask example --- run_flask_example | 4 ++++ 1 file changed, 4 insertions(+) create mode 100755 run_flask_example diff --git a/run_flask_example b/run_flask_example new file mode 100755 index 0000000..7f5671c --- /dev/null +++ b/run_flask_example @@ -0,0 +1,4 @@ +#!/bin/sh +echo "Running a basic flask app via gunicorn with 4 processes to demonstrate" +echo "cross-process metric aggregation" +gunicorn -b 0.0.0.0:5001 -w 4 examples.basic_flask From 8ae120f9947b96a2adc5a265a7c56f7337b2361e Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Tue, 20 Sep 2011 22:27:59 -0700 Subject: [PATCH 05/21] Add subclassing tests --- mmstats.py | 8 ++++++-- tests/test_mmstats.py | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/mmstats.py b/mmstats.py index 6b05901..696ad28 100644 --- a/mmstats.py +++ b/mmstats.py @@ -111,7 +111,7 @@ def _init(self, state, mm, offset): class ReadOnlyStat(Stat): def __get__(self, inst, owner): if inst is None: - return owner + return self return inst._fields[self.key]._struct.value def __init__(self, label=None, value=None): @@ -138,7 +138,7 @@ def _init(self, state, mm, offset): class ReadWriteStat(Stat): def __get__(self, inst, owner): if inst is None: - return owner + return self return inst._fields[self.key]._struct.value def __set__(self, inst, value): @@ -168,6 +168,8 @@ def _init(self, state, mm, offset): return offset + ctypes.sizeof(state._StructCls) def __get__(self, inst, owner): + if inst is None: + return self state = inst._fields[self.key] # Get from the read buffer return state._struct.buffers[state._struct.write_buffer ^ 1] @@ -221,6 +223,8 @@ class BoolStat(ReadWriteStat): type_signature = '?' def __get__(self, inst, owner): + if inst is None: + return self return inst._fields[self.key]._struct.value == 1 def __set__(self, inst, value): diff --git a/tests/test_mmstats.py b/tests/test_mmstats.py index e984d1f..11a987c 100644 --- a/tests/test_mmstats.py +++ b/tests/test_mmstats.py @@ -67,3 +67,28 @@ class BigStats(mmstats.MmStats): self.assertEqual(bs.f1, 0) self.assertEqual(bs.f2, 0) self.assertEqual(bs.f3, 0) + + def test_subclassing(self): + class ParentStats(mmstats.MmStats): + a = mmstats.UIntStat() + b = mmstats.UIntStat() + + class ChildAStats(ParentStats): + a = mmstats.BoolStat() + c = mmstats.UIntStat() + + class ChildBStats(ChildAStats): + b = mmstats.BoolStat() + c = mmstats.BoolStat() + + self.assertTrue(isinstance(ParentStats.a, mmstats.UIntStat)) + self.assertTrue(isinstance(ParentStats.b, mmstats.UIntStat)) + self.assertRaises(AttributeError, getattr, ParentStats, 'c') + + self.assertTrue(isinstance(ChildAStats.a, mmstats.BoolStat)) + self.assertTrue(isinstance(ChildAStats.b, mmstats.UIntStat)) + self.assertTrue(isinstance(ChildAStats.c, mmstats.UIntStat)) + + self.assertTrue(isinstance(ChildBStats.a, mmstats.BoolStat)) + self.assertTrue(isinstance(ChildBStats.b, mmstats.BoolStat)) + self.assertTrue(isinstance(ChildBStats.c, mmstats.BoolStat)) From a934d6c8834a147944fac9fc2c4af51b89dcffe5 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Thu, 22 Sep 2011 15:57:51 -0700 Subject: [PATCH 06/21] Rename stat fields from Stat to Field --- README.rst | 8 ++-- examples/basic.py | 12 +++--- examples/basic_flask.py | 6 +-- mmstats.py | 82 ++++++++++++++++++++--------------------- tests/test_mmstats.py | 46 +++++++++++------------ tests/test_types.py | 32 ++++++++-------- 6 files changed, 93 insertions(+), 93 deletions(-) diff --git a/README.rst b/README.rst index 37c3378..c3bb0de 100644 --- a/README.rst +++ b/README.rst @@ -47,10 +47,10 @@ Using :: class WebStats(mmstats.MmStats): - status2xx = mmstats.UIntStat(label='status.2XX') - status3xx = mmstats.UIntStat(label='status.3XX') - status4xx = mmstats.UIntStat(label='status.4XX') - status5xx = mmstats.UIntStat(label='status.5XX') + status2xx = mmstats.UIntField(label='status.2XX') + status3xx = mmstats.UIntField(label='status.3XX') + status4xx = mmstats.UIntField(label='status.4XX') + status5xx = mmstats.UIntField(label='status.5XX') 4. Instantiate it once per thread/process: diff --git a/examples/basic.py b/examples/basic.py index 9b0b380..1549751 100644 --- a/examples/basic.py +++ b/examples/basic.py @@ -8,12 +8,12 @@ class MyStats(mmstats.BaseMmStats): tid = mmstats.StaticInt64Field(label="sys.tid", value=libgettid.gettid) uid = mmstats.StaticUInt64Field(label="sys.uid", value=os.getuid) gid = mmstats.StaticUInt64Field(label="sys.gid", value=os.getgid) - errors = mmstats.UIntStat(label="com.urbanairship.app.errors") - warnings = mmstats.UIntStat(label="com.urbanairship.app.warnings") - queries = mmstats.UIntStat(label="com.urbanairship.app.queries") - cache_hits = mmstats.UIntStat(label="com.urbanairship.app.cache_hits") - cache_misses = mmstats.UIntStat(label="com.urbanairship.app.cache_misses") - degraded = mmstats.BoolStat(label="com.urbanairship.app.degraded") + errors = mmstats.UIntField(label="com.urbanairship.app.errors") + warnings = mmstats.UIntField(label="com.urbanairship.app.warnings") + queries = mmstats.UIntField(label="com.urbanairship.app.queries") + cache_hits = mmstats.UIntField(label="com.urbanairship.app.cache_hits") + cache_misses = mmstats.UIntField(label="com.urbanairship.app.cache_misses") + degraded = mmstats.BoolField(label="com.urbanairship.app.degraded") foo = mmstats.StaticTextField( label="com.idealist.app.name", value="webapp") diff --git a/examples/basic_flask.py b/examples/basic_flask.py index bffc963..6657d39 100644 --- a/examples/basic_flask.py +++ b/examples/basic_flask.py @@ -8,9 +8,9 @@ class Stats(mmstats.MmStats): - ok = mmstats.UIntStat(label="mmstats.example.ok") - bad = mmstats.UIntStat(label="mmstats.example.bad") - working = mmstats.BoolStat(label="mmstats.example.working") + ok = mmstats.UIntField(label="mmstats.example.ok") + bad = mmstats.UIntField(label="mmstats.example.bad") + working = mmstats.BoolField(label="mmstats.example.working") stats = Stats() diff --git a/mmstats.py b/mmstats.py index 696ad28..0fb59b9 100644 --- a/mmstats.py +++ b/mmstats.py @@ -9,13 +9,13 @@ PAGESIZE = mmap.PAGESIZE BUFFER_IDX_TYPE = ctypes.c_byte -LABEL_SZ_TYPE = ctypes.c_ushort +SIZE_TYPE = ctypes.c_ushort WRITE_BUFFER_UNUSED = 255 DEFAULT_PATH = os.environ.get('MMSTATS_PATH', tempfile.gettempdir()) -class DuplicateStatName(Exception): - """Cannot add 2 stats with the same name to MmStat instances""" +class DuplicateFieldName(Exception): + """Cannot add 2 fields with the same name to MmStat instances""" def _init_mmap(path=None, filename=None, size=PAGESIZE): @@ -52,9 +52,9 @@ def _create_struct(label, type_, type_signature, buffers=None): label = label.encode('utf8') fields = [ - ('label_sz', LABEL_SZ_TYPE), + ('label_sz', SIZE_TYPE), ('label', ctypes.c_char * len(label)), - ('type_sig_sz', ctypes.c_ushort), + ('type_sig_sz', SIZE_TYPE), ('type_signature', ctypes.c_char * len(type_signature)), ('write_buffer', ctypes.c_ubyte), ] @@ -70,7 +70,7 @@ def _create_struct(label, type_, type_signature, buffers=None): ) -class Stat(object): +class Field(object): initial = 0 def __init__(self, label=None): @@ -81,7 +81,7 @@ def __init__(self, label=None): self.label = None def _new(self, state, label_prefix, attrname, buffers=None): - """Creates new data structure for stat in state instance""" + """Creates new data structure for field in state instance""" # Key is used to reference field state on the parent instance self.key = attrname @@ -97,7 +97,7 @@ def _new(self, state, label_prefix, attrname, buffers=None): return state.size def _init(self, state, mm, offset): - """Initializes value of stat's data structure""" + """Initializes value of field's data structure""" state._struct = state._StructCls.from_buffer(mm, offset) state._struct.label_sz = len(state.label) state._struct.label = state.label @@ -108,14 +108,14 @@ def _init(self, state, mm, offset): return offset + ctypes.sizeof(state._StructCls) -class ReadOnlyStat(Stat): +class ReadOnlyField(Field): def __get__(self, inst, owner): if inst is None: return self return inst._fields[self.key]._struct.value def __init__(self, label=None, value=None): - super(ReadOnlyStat, self).__init__(label=label) + super(ReadOnlyField, self).__init__(label=label) self.value = value def _init(self, state, mm, offset): @@ -127,7 +127,7 @@ def _init(self, state, mm, offset): self.value = self.value() # Call super to do standard initialization - new_offset = super(ReadOnlyStat, self)._init(state, mm, offset) + new_offset = super(ReadOnlyField, self)._init(state, mm, offset) # Set the static field now state._struct.value = self.value @@ -135,7 +135,7 @@ def _init(self, state, mm, offset): return new_offset -class ReadWriteStat(Stat): +class ReadWriteField(Field): def __get__(self, inst, owner): if inst is None: return self @@ -152,9 +152,9 @@ def __repr__(self): return '%s(label=%r)' % (self.__class__.__name__, self.label) -class DoubleBufferedStat(ReadWriteStat): +class DoubleBufferedField(ReadWriteField): def _new(self, state, label_prefix, attrname): - return super(DoubleBufferedStat, self)._new( + return super(DoubleBufferedField, self)._new( state, label_prefix, attrname, buffers=2) def _init(self, state, mm, offset): @@ -183,40 +183,40 @@ def __set__(self, inst, value): class FieldState(object): - """Holds field state for each stat instance""" + """Holds field state for each Field instance""" - def __init__(self, stat): - self.stat = stat + def __init__(self, field): + self.field = field -class UIntStat(DoubleBufferedStat): +class UIntField(DoubleBufferedField): """32bit Double Buffered Unsigned Integer field""" buffer_type = ctypes.c_uint32 type_signature = 'I' -class IntStat(DoubleBufferedStat): +class IntField(DoubleBufferedField): """32bit Double Buffered Signed Integer field""" buffer_type = ctypes.c_int32 type_signature = 'i' -class ShortStat(DoubleBufferedStat): +class ShortField(DoubleBufferedField): """16bit Double Buffered Signed Integer field""" buffer_type = ctypes.c_int16 -class UShortStat(DoubleBufferedStat): +class UShortField(DoubleBufferedField): """16bit Double Buffered Unsigned Integer field""" buffer_type = ctypes.c_uint16 -class ByteStat(ReadWriteStat): +class ByteField(ReadWriteField): """8bit Signed Integer Field""" buffer_type = ctypes.c_byte -class BoolStat(ReadWriteStat): +class BoolField(ReadWriteField): """Boolean Field""" # Avoid potential ambiguity and marshal bools to 0/1 manually buffer_type = ctypes.c_byte @@ -231,37 +231,37 @@ def __set__(self, inst, value): inst._fields[self.key]._struct.value = 1 if value else 0 -class StaticUIntField(ReadOnlyStat): +class StaticUIntField(ReadOnlyField): """Unbuffered read-only 32bit Unsigned Integer field""" buffer_type = ctypes.c_uint32 type_signature = 'I' -class StaticInt64Field(ReadOnlyStat): +class StaticInt64Field(ReadOnlyField): """Unbuffered read-only 64bit Unsigned Integer field""" buffer_type = ctypes.c_uint64 type_signature = 'l' -class StaticUInt64Field(ReadOnlyStat): +class StaticUInt64Field(ReadOnlyField): """Unbuffered read-only 64bit Unsigned Integer field""" buffer_type = ctypes.c_uint64 type_signature = 'L' -class StaticTextField(ReadOnlyStat): +class StaticTextField(ReadOnlyField): """Unbuffered read-only UTF-8 encoded String field""" initial = '' buffer_type = ctypes.c_char * 256 type_signature = '256s' -class StaticListField(ReadOnlyStat): +class StaticListField(ReadOnlyField): """Unbuffered read-only List field""" #TODO -class StaticMappingField(ReadOnlyStat): +class StaticMappingField(ReadOnlyField): """Unbuffered read-only List field""" #TODO @@ -282,30 +282,30 @@ def __init__(self, filename=None, label_prefix=None): #FIXME This is the *wrong* way to initialize stat fields for cls in self.__class__.__mro__: for attrname, attrval in cls.__dict__.items(): - if attrname not in self._fields and isinstance(attrval, Stat): - total_size += self._add_stat(attrname, attrval) + if attrname not in self._fields and isinstance(attrval, Field): + total_size += self._add_field(attrname, attrval) self._filename, self._size, self._mmap = _init_mmap( filename=filename, size=total_size) self._mmap[0] = '\x01' # Stupid version number # Finally initialize thes stats - self._init_stats(total_size) + self._init_fields(total_size) - def _add_stat(self, name, stat): - """Given a name and Stat instance, add this field and retun size""" + def _add_field(self, name, field): + """Given a name and Field instance, add this field and retun size""" # Stats need a place to store their per Mmstats instance state - state = self._fields[name] = FieldState(stat) + state = self._fields[name] = FieldState(field) - # Call stat._new to determine size - return stat._new(state, self.label_prefix, name) + # Call field._new to determine size + return field._new(state, self.label_prefix, name) - def _init_stats(self, total_size): - """Once all stats have been added, initialize them in mmap""" + def _init_fields(self, total_size): + """Once all fields have been added, initialize them in mmap""" for state in self._fields.values(): - # 2nd Call stat._init to initialize new stat - self._offset = state.stat._init(state, self._mmap, self._offset) + # 2nd Call field._init to initialize new stat + self._offset = state.field._init(state, self._mmap, self._offset) @property def filename(self): diff --git a/tests/test_mmstats.py b/tests/test_mmstats.py index 11a987c..1c51e0c 100644 --- a/tests/test_mmstats.py +++ b/tests/test_mmstats.py @@ -7,8 +7,8 @@ class TestMmStats(base.MmstatsTestCase): def test_class_instances(self): """You can have 2 instances of an MmStats model without shared state""" class LaserStats(mmstats.MmStats): - blue = mmstats.UIntStat() - red = mmstats.UIntStat() + blue = mmstats.UIntField() + red = mmstats.UIntField() a = LaserStats(filename='mmstats-test-laserstats-a') b = LaserStats(filename='mmstats-test-laserstats-b') @@ -24,8 +24,8 @@ class LaserStats(mmstats.MmStats): def test_label_prefix(self): class StatsA(mmstats.MmStats): - f2 = mmstats.UIntStat(label='f.secondary') - f1 = mmstats.UIntStat() + f2 = mmstats.UIntField(label='f.secondary') + f1 = mmstats.UIntField() a = StatsA(filename='mmstats-test-label-prefix1') b = StatsA(filename='mmstats-test-label-prefix2', @@ -50,17 +50,17 @@ class StatsA(mmstats.MmStats): def test_mmap_resize1(self): class BigStats(mmstats.MmStats): - f1 = mmstats.BoolStat(label='f1'*(mmstats.PAGESIZE / 2)) - f2 = mmstats.BoolStat(label='f2'*(mmstats.PAGESIZE / 2)) + f1 = mmstats.BoolField(label='f1'*(mmstats.PAGESIZE / 2)) + f2 = mmstats.BoolField(label='f2'*(mmstats.PAGESIZE / 2)) bs = BigStats() self.assertEqual(bs.size, mmstats.PAGESIZE * 3) def test_mmap_resize2(self): class BigStats(mmstats.MmStats): - f1 = mmstats.UIntStat(label='f'+('o'*mmstats.PAGESIZE)) - f2 = mmstats.UIntStat(label='f'+('0'*mmstats.PAGESIZE)) - f3 = mmstats.UIntStat(label='f'+('1'*mmstats.PAGESIZE)) + f1 = mmstats.UIntField(label='f'+('o'*mmstats.PAGESIZE)) + f2 = mmstats.UIntField(label='f'+('0'*mmstats.PAGESIZE)) + f3 = mmstats.UIntField(label='f'+('1'*mmstats.PAGESIZE)) bs = BigStats(filename='mmstats-test-resize1') self.assertEqual(bs.size, mmstats.PAGESIZE * 4) @@ -70,25 +70,25 @@ class BigStats(mmstats.MmStats): def test_subclassing(self): class ParentStats(mmstats.MmStats): - a = mmstats.UIntStat() - b = mmstats.UIntStat() + a = mmstats.UIntField() + b = mmstats.UIntField() class ChildAStats(ParentStats): - a = mmstats.BoolStat() - c = mmstats.UIntStat() + a = mmstats.BoolField() + c = mmstats.UIntField() class ChildBStats(ChildAStats): - b = mmstats.BoolStat() - c = mmstats.BoolStat() + b = mmstats.BoolField() + c = mmstats.BoolField() - self.assertTrue(isinstance(ParentStats.a, mmstats.UIntStat)) - self.assertTrue(isinstance(ParentStats.b, mmstats.UIntStat)) + self.assertTrue(isinstance(ParentStats.a, mmstats.UIntField)) + self.assertTrue(isinstance(ParentStats.b, mmstats.UIntField)) self.assertRaises(AttributeError, getattr, ParentStats, 'c') - self.assertTrue(isinstance(ChildAStats.a, mmstats.BoolStat)) - self.assertTrue(isinstance(ChildAStats.b, mmstats.UIntStat)) - self.assertTrue(isinstance(ChildAStats.c, mmstats.UIntStat)) + self.assertTrue(isinstance(ChildAStats.a, mmstats.BoolField)) + self.assertTrue(isinstance(ChildAStats.b, mmstats.UIntField)) + self.assertTrue(isinstance(ChildAStats.c, mmstats.UIntField)) - self.assertTrue(isinstance(ChildBStats.a, mmstats.BoolStat)) - self.assertTrue(isinstance(ChildBStats.b, mmstats.BoolStat)) - self.assertTrue(isinstance(ChildBStats.c, mmstats.BoolStat)) + self.assertTrue(isinstance(ChildBStats.a, mmstats.BoolField)) + self.assertTrue(isinstance(ChildBStats.b, mmstats.BoolField)) + self.assertTrue(isinstance(ChildBStats.c, mmstats.BoolField)) diff --git a/tests/test_types.py b/tests/test_types.py index 691d310..f04bb6b 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -6,9 +6,9 @@ class TestTypes(base.MmstatsTestCase): def test_ints(self): class MyStats(mmstats.MmStats): - zebras = mmstats.IntStat() - apples = mmstats.UIntStat() - oranges = mmstats.UIntStat() + zebras = mmstats.IntField() + apples = mmstats.UIntField() + oranges = mmstats.UIntField() mmst = MyStats(filename='mmstats-test-ints') @@ -37,11 +37,11 @@ class MyStats(mmstats.MmStats): self.assertEqual(mmst.apples, (2**32)-100) def test_shorts(self): - class ShortStats(mmstats.MmStats): - a = mmstats.ShortStat() - b = mmstats.UShortStat() + class ShortFields(mmstats.MmStats): + a = mmstats.ShortField() + b = mmstats.UShortField() - s = ShortStats(filename='mmstats-test-shorts') + s = ShortFields(filename='mmstats-test-shorts') self.assertEqual(s.a, 0, s.a) self.assertEqual(s.b, 0, s.b) s.a = -1 @@ -55,11 +55,11 @@ class ShortStats(mmstats.MmStats): self.assertEqual(s.b, (2**16)-2, s.b) def test_bools(self): - class BoolStats(mmstats.MmStats): - a = mmstats.BoolStat() - b = mmstats.BoolStat() + class BoolFields(mmstats.MmStats): + a = mmstats.BoolField() + b = mmstats.BoolField() - s = BoolStats(filename='mmstats-test-bools') + s = BoolFields(filename='mmstats-test-bools') self.assertTrue('a\x01\x00?\xff\x00' in s._mmap[:], repr(s._mmap[:30])) self.assertTrue('b\x01\x00?\xff\x00' in s._mmap[:], repr(s._mmap[:30])) self.assertTrue(s.a is False, s.a) @@ -84,11 +84,11 @@ class StringStats(mmstats.BaseMmStats): def test_mixed(self): class MixedStats(mmstats.MmStats): - a = mmstats.UIntStat() - b = mmstats.BoolStat() - c = mmstats.IntStat() - d = mmstats.BoolStat(label='The Bool') - e = mmstats.ShortStat(label='shortie') + a = mmstats.UIntField() + b = mmstats.BoolField() + c = mmstats.IntField() + d = mmstats.BoolField(label='The Bool') + e = mmstats.ShortField(label='shortie') m1 = MixedStats(label_prefix='m1::', filename='mmstats-test-m1') m2 = MixedStats(label_prefix='m2::', filename='mmstats-test-m2') From 8e86251b7ae49cefbb8faa715ec2ccc7a530635a Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Thu, 22 Sep 2011 17:21:47 -0700 Subject: [PATCH 07/21] mmstats "Fields of Gold" 0.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bebeeb7..fb77b10 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='mmstats', - version='0.1', + version='0.2', license='BSD', author='Michael Schurter', author_email='m@schmichael.com', From b701bdab14de0a8c4f90cee0d3f36947bfd31ecb Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 23 Sep 2011 14:54:46 -0700 Subject: [PATCH 08/21] Add UInt64Field and alias it to CounterField Wanted to make the most common use case as obvious as possible --- mmstats.py | 10 ++++++++++ tests/test_types.py | 12 ++++++++++++ 2 files changed, 22 insertions(+) diff --git a/mmstats.py b/mmstats.py index 0fb59b9..ad9fc72 100644 --- a/mmstats.py +++ b/mmstats.py @@ -189,6 +189,16 @@ def __init__(self, field): self.field = field +class UInt64Field(DoubleBufferedField): + """Unbuffered read-only 64bit Unsigned Integer field""" + buffer_type = ctypes.c_uint64 + type_signature = 'L' + + +# Alias UInt64Field to CounterField for simplicity +CounterField = UInt64Field + + class UIntField(DoubleBufferedField): """32bit Double Buffered Unsigned Integer field""" buffer_type = ctypes.c_uint32 diff --git a/tests/test_types.py b/tests/test_types.py index f04bb6b..7fc7396 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -118,3 +118,15 @@ class MixedStats(mmstats.MmStats): self.assertTrue(m2.d is True, m2.d) self.assertEqual(m1.e, 1, m1.e) self.assertEqual(m2.e, 90, m2.e) + + def test_counter(self): + class SimpleCounter(mmstats.MmStats): + counter = mmstats.CounterField() + + s = SimpleCounter(filename='mmstats-test_counter') + self.assertEqual(s.counter, 0) + s.counter += 1 + self.assertEqual(s.counter, 1) + s.counter -= 2 + self.assertNotEqual(s.counter, -1) + From bc84593b3d68f0c71ce0026e07c28e7b4df1bc50 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 23 Sep 2011 15:11:06 -0700 Subject: [PATCH 09/21] mmstats "Counting Coppers" 0.2.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fb77b10..052d212 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='mmstats', - version='0.2', + version='0.2.1', license='BSD', author='Michael Schurter', author_email='m@schmichael.com', From 67a49e1798a16119f85012c9c391f4f78c808688 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 23 Sep 2011 15:13:22 -0700 Subject: [PATCH 10/21] Update TODO with timer field & field api --- TODO.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TODO.rst b/TODO.rst index f5cca9e..3f2354c 100644 --- a/TODO.rst +++ b/TODO.rst @@ -3,6 +3,8 @@ TODO ==== * Add every simple type (strings, ints, floats, bools, etc) +* Add timer field/contextmanager +* Add API to dynamically add fields to MmStat classes * Test severity of race conditions * Test performance From d540e4c93a7747158680e7ff768d3a2a7f11b134 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 23 Sep 2011 15:50:29 -0700 Subject: [PATCH 11/21] Add some ideas from talking with Niall --- TODO.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TODO.rst b/TODO.rst index 3f2354c..5f91c9c 100644 --- a/TODO.rst +++ b/TODO.rst @@ -4,6 +4,8 @@ TODO * Add every simple type (strings, ints, floats, bools, etc) * Add timer field/contextmanager +* Add memory usage field/contextmanager +* Vary filename based on class name * Add API to dynamically add fields to MmStat classes * Test severity of race conditions * Test performance From ca8198a50b82cdc52d8e632ad99288b2d73c7058 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Sat, 24 Sep 2011 12:47:57 -0700 Subject: [PATCH 12/21] Add basic AUTHORS file --- AUTHORS | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 AUTHORS diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..77c7383 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,6 @@ +Developers: + Dan Colish + Michael Schurter (creator) + +Additional help: + Niall Kelly (reviewer) From 77f31ca103972109d3077ae4b8684de12953bfd4 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Sat, 24 Sep 2011 12:51:49 -0700 Subject: [PATCH 13/21] Add alternative writer API idea to TODO --- TODO.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/TODO.rst b/TODO.rst index 5f91c9c..4d4ec9a 100644 --- a/TODO.rst +++ b/TODO.rst @@ -7,6 +7,7 @@ TODO * Add memory usage field/contextmanager * Vary filename based on class name * Add API to dynamically add fields to MmStat classes +* Add alternative procedural writer API (vs existing declarative models) * Test severity of race conditions * Test performance From 552803895a44862da5420d107021d80ec1f4301b Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Sat, 24 Sep 2011 13:28:03 -0700 Subject: [PATCH 14/21] Refactor descriptor methods into mixins This should help facilitate non-descriptor fields such as context managers and compound/complex types. --- mmstats.py | 101 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 60 insertions(+), 41 deletions(-) diff --git a/mmstats.py b/mmstats.py index ad9fc72..d7d4a4a 100644 --- a/mmstats.py +++ b/mmstats.py @@ -107,13 +107,54 @@ def _init(self, state, mm, offset): state._struct.value = self.initial return offset + ctypes.sizeof(state._StructCls) + @property + def type_signature(self): + return self.buffer_type._type_ + + def __repr__(self): + return '%s(label=%r)' % (self.__class__.__name__, self.label) + + +class NonDataDescriptorMixin(object): + """Mixin to add single buffered __get__ method""" -class ReadOnlyField(Field): def __get__(self, inst, owner): if inst is None: return self return inst._fields[self.key]._struct.value + +class DataDescriptorMixin(object): + """Mixin to add single buffered __set__ method""" + + def __set__(self, inst, value): + inst._fields[self.key]._struct.value = value + + +class BufferedDescriptorMixin(object): + """\ + Mixin to add double buffered descriptor methods + + Always read/write as double buffering doesn't make sense for readonly + fields + """ + + def __get__(self, inst, owner): + if inst is None: + return self + state = inst._fields[self.key] + # Get from the read buffer + return state._struct.buffers[state._struct.write_buffer ^ 1] + + def __set__(self, inst, value): + state = inst._fields[self.key] + # Set the write buffer + state._struct.buffers[state._struct.write_buffer] = value + # Swap the write buffer + state._struct.write_buffer ^= 1 + + +class ReadOnlyField(Field, NonDataDescriptorMixin): def __init__(self, label=None, value=None): super(ReadOnlyField, self).__init__(label=label) self.value = value @@ -135,24 +176,12 @@ def _init(self, state, mm, offset): return new_offset -class ReadWriteField(Field): - def __get__(self, inst, owner): - if inst is None: - return self - return inst._fields[self.key]._struct.value - - def __set__(self, inst, value): - inst._fields[self.key]._struct.value = value - - @property - def type_signature(self): - return self.buffer_type._type_ - - def __repr__(self): - return '%s(label=%r)' % (self.__class__.__name__, self.label) +class ReadWriteField(Field, NonDataDescriptorMixin, DataDescriptorMixin): + """Base class for simple writable fields""" -class DoubleBufferedField(ReadWriteField): +class DoubleBufferedField(Field): + """Base class for double buffered writable fields""" def _new(self, state, label_prefix, attrname): return super(DoubleBufferedField, self)._new( state, label_prefix, attrname, buffers=2) @@ -167,29 +196,12 @@ def _init(self, state, mm, offset): state._struct.buffers = 0, 0 return offset + ctypes.sizeof(state._StructCls) - def __get__(self, inst, owner): - if inst is None: - return self - state = inst._fields[self.key] - # Get from the read buffer - return state._struct.buffers[state._struct.write_buffer ^ 1] - - def __set__(self, inst, value): - state = inst._fields[self.key] - # Set the write buffer - state._struct.buffers[state._struct.write_buffer] = value - # Swap the write buffer - state._struct.write_buffer ^= 1 - - -class FieldState(object): - """Holds field state for each Field instance""" - def __init__(self, field): - self.field = field +class BufferedDescriptorField(DoubleBufferedField, BufferedDescriptorMixin): + """Base class for double buffered descriptor fields""" -class UInt64Field(DoubleBufferedField): +class UInt64Field(BufferedDescriptorField): """Unbuffered read-only 64bit Unsigned Integer field""" buffer_type = ctypes.c_uint64 type_signature = 'L' @@ -199,24 +211,24 @@ class UInt64Field(DoubleBufferedField): CounterField = UInt64Field -class UIntField(DoubleBufferedField): +class UIntField(BufferedDescriptorField): """32bit Double Buffered Unsigned Integer field""" buffer_type = ctypes.c_uint32 type_signature = 'I' -class IntField(DoubleBufferedField): +class IntField(BufferedDescriptorField): """32bit Double Buffered Signed Integer field""" buffer_type = ctypes.c_int32 type_signature = 'i' -class ShortField(DoubleBufferedField): +class ShortField(BufferedDescriptorField): """16bit Double Buffered Signed Integer field""" buffer_type = ctypes.c_int16 -class UShortField(DoubleBufferedField): +class UShortField(BufferedDescriptorField): """16bit Double Buffered Unsigned Integer field""" buffer_type = ctypes.c_uint16 @@ -276,6 +288,13 @@ class StaticMappingField(ReadOnlyField): #TODO +class FieldState(object): + """Holds field state for each Field instance""" + + def __init__(self, field): + self.field = field + + class BaseMmStats(object): """Stats models should inherit from this""" From fcb912555749449d396338e8e0bfc8106d6c1007 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Sat, 24 Sep 2011 13:51:43 -0700 Subject: [PATCH 15/21] Backwards incompatible change to CounterField --- mmstats.py | 36 ++++++++++++++++++++++++++++++++---- tests/test_types.py | 10 +++++----- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/mmstats.py b/mmstats.py index d7d4a4a..8ad376f 100644 --- a/mmstats.py +++ b/mmstats.py @@ -197,6 +197,38 @@ def _init(self, state, mm, offset): return offset + ctypes.sizeof(state._StructCls) +class _Counter(object): + """Internal counter class used by CounterFields""" + def __init__(self, state): + self._struct = state._struct + + @property + def value(self): + return self._struct.buffers[self._struct.write_buffer ^ 1] + + def inc(self, n=1): + # Set the write buffer + self._struct.buffers[self._struct.write_buffer] = self.value + n + # Swap the write buffer + self._struct.write_buffer ^= 1 + + +class CounterField(DoubleBufferedField): + """Counter field supporting an inc() method and value attribute""" + buffer_type = ctypes.c_uint64 + type_signature = 'L' + + def _init(self, state, mm, offset): + offset = super(CounterField, self)._init(state, mm, offset) + state.counter = _Counter(state) + return offset + + def __get__(self, inst, owner): + if inst is None: + return self + return inst._fields[self.key].counter + + class BufferedDescriptorField(DoubleBufferedField, BufferedDescriptorMixin): """Base class for double buffered descriptor fields""" @@ -207,10 +239,6 @@ class UInt64Field(BufferedDescriptorField): type_signature = 'L' -# Alias UInt64Field to CounterField for simplicity -CounterField = UInt64Field - - class UIntField(BufferedDescriptorField): """32bit Double Buffered Unsigned Integer field""" buffer_type = ctypes.c_uint32 diff --git a/tests/test_types.py b/tests/test_types.py index 7fc7396..bca6f5e 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -124,9 +124,9 @@ class SimpleCounter(mmstats.MmStats): counter = mmstats.CounterField() s = SimpleCounter(filename='mmstats-test_counter') - self.assertEqual(s.counter, 0) - s.counter += 1 - self.assertEqual(s.counter, 1) - s.counter -= 2 - self.assertNotEqual(s.counter, -1) + self.assertEqual(s.counter.value, 0) + s.counter.inc() + self.assertEqual(s.counter.value, 1) + s.counter.inc(-2) + self.assertNotEqual(s.counter.value, -1) From 0e69f29e15fd69717de1dfdcbd85bae68398f5a1 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Sat, 24 Sep 2011 17:14:18 -0500 Subject: [PATCH 16/21] Explain why my initial approach to compound fields failed --- TODO.rst | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/TODO.rst b/TODO.rst index 4d4ec9a..defa0be 100644 --- a/TODO.rst +++ b/TODO.rst @@ -15,6 +15,51 @@ TODO Scrapped Ideas ============== +--------------------------------------------------------------- +Compounds Fields where 1 Writer Field = Many Mmap/Reader Fields +--------------------------------------------------------------- + +This seemed like a honking great idea at first. Compound fields would look just +like a mini-MmStat model: + +:: + + class SamplingCounterField(CompoundField): + """Records increments per ms every N increments""" + counter = CounterField() + per_ms = UInt64Field() + + class _Counter(object): + """Implement counter/rate-sampling logic here""" + + def __get__(self, inst, owner): + if inst is None: + return self + return inst._fields[self.key]._counter_instance + +The blocker is that there's no way to atomically update all of the compound +fields. The only way to accomplish this is for compound fields to appear as a +single double buffered field with each component field as a type in the type +signature: + +:: + + class SamplingCounterField(DoubleBufferedField): + initial = ( + CounterField.initial, + UInt64Field.initial, + ) + buffer_type = ( + CounterField.buffer_type, + UInt64Field.buffer_type, + ) + type_signature = ( + CounterField.type_signature + UInt64Field.type_signature + ) + +Obviously an actual implementation should remove the redundant references to +the component types. + ------------------------ Metadata metaprogramming ------------------------ From 4468d483d0e98c3438e34aa24116985c4c315544 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Thu, 29 Sep 2011 10:59:17 -0700 Subject: [PATCH 17/21] mmstats "Adventure Update" 0.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 052d212..b255010 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='mmstats', - version='0.2.1', + version='0.3.0', license='BSD', author='Michael Schurter', author_email='m@schmichael.com', From f4c223ede9c09d1933fcd5d475594ca0e13b87e1 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Thu, 29 Sep 2011 11:35:52 -0700 Subject: [PATCH 18/21] Allow setting defaults on BoolFields (defaulted to False before) --- mmstats.py | 4 ++++ tests/test_types.py | 16 ++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/mmstats.py b/mmstats.py index 8ad376f..be65e71 100644 --- a/mmstats.py +++ b/mmstats.py @@ -272,6 +272,10 @@ class BoolField(ReadWriteField): buffer_type = ctypes.c_byte type_signature = '?' + def __init__(self, initial=False, **kwargs): + self.initial = initial + super(BoolField, self).__init__(**kwargs) + def __get__(self, inst, owner): if inst is None: return self diff --git a/tests/test_types.py b/tests/test_types.py index bca6f5e..5ecdec5 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -57,23 +57,23 @@ class ShortFields(mmstats.MmStats): def test_bools(self): class BoolFields(mmstats.MmStats): a = mmstats.BoolField() - b = mmstats.BoolField() + b = mmstats.BoolField(initial=True) s = BoolFields(filename='mmstats-test-bools') self.assertTrue('a\x01\x00?\xff\x00' in s._mmap[:], repr(s._mmap[:30])) - self.assertTrue('b\x01\x00?\xff\x00' in s._mmap[:], repr(s._mmap[:30])) + self.assertTrue('b\x01\x00?\xff\x01' in s._mmap[:], repr(s._mmap[:30])) self.assertTrue(s.a is False, s.a) - self.assertTrue(s.b is False, s.b) + self.assertTrue(s.b is True, s.b) s.a = 'Anything truthy at all' self.assertTrue(s.a is True, s.a) - self.assertTrue(s.b is False, s.b) + self.assertTrue(s.b is True, s.b) s.a = [] # Anything falsey self.assertTrue(s.a is False, s.a) - self.assertTrue(s.b is False, s.b) - s.b = 1 - s.a = s.b - self.assertTrue(s.a is True, s.a) self.assertTrue(s.b is True, s.b) + s.b = False + s.a = s.b + self.assertTrue(s.a is False, s.a) + self.assertTrue(s.b is False, s.b) def test_strings(self): class StringStats(mmstats.BaseMmStats): From e4b21878e9451e1bf824d642d3b762efde48484a Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Thu, 29 Sep 2011 11:50:55 -0700 Subject: [PATCH 19/21] Add ability to remove mmstats files --- mmstats.py | 16 ++++++++++++++++ tests/test_mmap.py | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/mmstats.py b/mmstats.py index be65e71..671a300 100644 --- a/mmstats.py +++ b/mmstats.py @@ -331,6 +331,9 @@ class BaseMmStats(object): """Stats models should inherit from this""" def __init__(self, filename=None, label_prefix=None): + """\ + Optionally given a filename or label_prefix, create an MmStats instance + """ # Setup label prefix self._label_prefix = '' if label_prefix is None else label_prefix @@ -380,6 +383,19 @@ def label_prefix(self): def size(self): return self._mmap.size() + def flush(self): + """Flush mmapped file to disk""" + #TODO Handle Windows return values: + # http://docs.python.org/library/mmap#mmap.flush + self._mmap.flush() + + def remove(self): + """Close and remove mmap file - No further stats updates will work""" + self.flush() + self._mmap.close() + os.remove(self.filename) + # Remove fields to prevent segfaults + self._fields = {} class MmStats(BaseMmStats): pid = StaticUIntField(label="sys.pid", value=os.getpid) diff --git a/tests/test_mmap.py b/tests/test_mmap.py index 9af386e..78cc96d 100644 --- a/tests/test_mmap.py +++ b/tests/test_mmap.py @@ -51,3 +51,19 @@ def test_truncate(self): reopened_file = open(fn) self.assertEqual(reopened_file.read(1), 'X') self.assertEqual(reopened_file.read(1), '\x00') + + def test_remove(self): + """Calling remove() on an MmStat instance should remove the file""" + class TestStat(mmstats.MmStats): + b = mmstats.BoolField() + + fn = os.path.join(self.path, 'mmstats-test_remove') + ts = TestStat(filename=fn) + ts.b = True + self.assertTrue(ts.b) + self.assertTrue(os.path.exists(fn)) + ts.remove() + self.assertFalse(os.path.exists(fn)) + # Trying to access the mmap after it's been removed should raise an + # exception but *not* segault + self.assertRaises(Exception, getattr, ts, 'b') From 85b348707066f5e2e1498ae1f5447603b38f994d Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Thu, 29 Sep 2011 12:01:47 -0700 Subject: [PATCH 20/21] Convinced Adam to peek at it --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 77c7383..7115a2a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -4,3 +4,4 @@ Developers: Additional help: Niall Kelly (reviewer) + Adam Lowry (reviewer) From 3319d247359c9e3f589b8e687bbffb406c8b412b Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Thu, 29 Sep 2011 12:02:00 -0700 Subject: [PATCH 21/21] mmstats "Now with 100% more suicide" 0.3.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b255010..377674c 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='mmstats', - version='0.3.0', + version='0.3.1', license='BSD', author='Michael Schurter', author_email='m@schmichael.com',