Skip to content

Commit

Permalink
cosmetic changes to keep code climate happy.
Browse files Browse the repository at this point in the history
  • Loading branch information
c0c0n3 committed Dec 9, 2020
1 parent b2fd29c commit 8af5e67
Show file tree
Hide file tree
Showing 4 changed files with 21 additions and 14 deletions.
2 changes: 1 addition & 1 deletion src/server/telemetry/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,4 @@
module to see how to write one, finally you can use the implementation of
the ``monitor`` module as a starting point for wiring together the building
blocks to make them fit for your use case.
"""
"""
22 changes: 14 additions & 8 deletions src/server/telemetry/observation.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,15 @@
# NOTE. Performance. Using a bare tuple instead of a class to keep memory
# footprint low. In fact, there could be quite a number of these little
# guys in memory during a sampling session...
# TODO. Consider using NumPy
# Unfortunately integers and floats aren't as light on memory as in other
# languages---e.g. an int is actually an object taking up at least 28 bytes
# on a 64-bit box. See:
# - https://pythonspeed.com/articles/python-integers-memory/
# - https://stackoverflow.com/questions/10365624
#
# NumPy would possibly have a much smaller footprint, but we'd like to keep
# this package self-contained so not to impose any external lib dependency
# on users.

OBSERVATION_MIN_SZ = getsizeof((0, 0.0))
"""
Expand Down Expand Up @@ -136,8 +139,8 @@ def observe_many(*labelled_measurements: (str, float)) \

ObservationStore = Dict[str, ObservationSeries]
"""
A collection of labelled observation time series. Each label uniquely identifies
a time series.
A collection of labelled observation time series. Each label uniquely
identifies a time series.
"""
# NOTE. Memory footprint. It looks like that using a dict with list values
# shouldn't be too bad:
Expand All @@ -154,9 +157,9 @@ def _extend_series(store: ObservationStore, label: str, obs: [Observation]):
# pre-allocating a list with an initial capacity. I have my doubts
# about this though and the effect of append on GC---i.e. what if
# the list grows in too small chunks? Is there any data structure
# we could use?
#
# TODO figure out to what extent this affects GC and how to optimise.
# we could use? Simple benchmarks show that we shouldn't have an
# issue here, but I'd still like to figure out to what extent this
# affects GC and how to optimise.


def observation_store(*ts: LabelledObservation) -> ObservationStore:
Expand Down Expand Up @@ -221,11 +224,14 @@ def merge_observation_stores(*ts: ObservationStore) -> ObservationStore:
[5.0]
"""
merged = {}
for t in ts: # TODO implement efficient merge algo
for t in ts:
for k in t:
_extend_series(merged, k, t[k])

return merged
# NOTE. Efficient algo. This function is only used for examples and testing
# so performance isn't really critical. But it'd be nice to implement an
# efficient algorithm.


def tabulate(store: ObservationStore) -> \
Expand Down Expand Up @@ -265,7 +271,7 @@ class ObservationBuffer:
>>> tot_bytes = buf.estimate_memory_lower_bound()
>>> 150 < tot_bytes < 200
True
>>> store = buf.flush()
>>> buf.size() == 0
True
Expand Down
4 changes: 2 additions & 2 deletions src/server/telemetry/pandas_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@

import pandas as pd

from server.telemetry.flush import TIMEPOINT_CSV_FIELD, MEASUREMENT_CSV_FIELD, \
LABEL_CSV_FIELD, PID_CSV_FIELD
from server.telemetry.flush import TIMEPOINT_CSV_FIELD, \
MEASUREMENT_CSV_FIELD, LABEL_CSV_FIELD, PID_CSV_FIELD
from server.telemetry.monitor import DURATION_FILE_PREFIX, RUNTIME_FILE_PREFIX
from server.telemetry.sampler import GC_COLLECTIONS, GC_COLLECTED, \
GC_UNCOLLECTABLE, PROC_MAX_RSS, PROC_SYSTEM_TIME, PROC_USER_TIME
Expand Down
7 changes: 4 additions & 3 deletions src/server/telemetry/sampler.py
Original file line number Diff line number Diff line change
Expand Up @@ -280,9 +280,10 @@ def sample(self):
class RuntimeBackgroundSampler:
"""
Convenience class to sample GC and OS metrics at regular intervals in a
background daemon thread. The thread goes on forever until the program
exits, calling ``GCSampler`` and ``ProcSampler`` every ``sampling_interval``
seconds to collect GC and OS-level metrics using a bucket you specify.
background daemon thread.
The thread goes on forever until the program exits, calling ``GCSampler``
and ``ProcSampler`` every ``sampling_interval`` seconds to collect GC and
OS-level metrics using a bucket you specify.
Just before the program exits, you should call the bucket's ``empty``
method to make sure any left over sampled data still in the memory buffer
gets processed by the bucket's empty action.
Expand Down

0 comments on commit 8af5e67

Please sign in to comment.