Sampling

README.md

@@ -250,6 +250,14 @@ A custom ingestion endpoint to stream log lines into.
 
 List of fields out of `record` object to include in the `meta` object. By default, `args`, `name`, `pathname`, and `lineno` will be included.
 
+##### sampling_instance
+
+* _Optional_
+* Type: Sampling class instance
+* Default: [`Sampling()`](logdna/sampling.py), which sends everything
+
+Instance of a "sampling class". Used to decide if a log should be sent via a random selection over some distribution.  The default sends everything.  However, the `UniformSampling` class is included and extension of `Sampling` is welcome.
+
 ### log(line, [options])
 
 #### line

logdna/logdna.py

@@ -9,6 +9,7 @@
 
 from .configs import defaults
 from .utils import sanitize_meta, get_ip, normalize_list_option
+from .sampling import Sampling
 
 
 class LogDNAHandler(logging.Handler):
@@ -77,6 +78,9 @@ def __init__(self, key, options={}):
         self.setLevel(logging.DEBUG)
         self.lock = threading.RLock()
 
+        # Set up sampling (class instance).  Defaults to send everything via base class
+        self.sampling_instance = options.get('sampling_instance', Sampling())
+
     def start_flusher(self):
         if not self.flusher:
             self.flusher = threading.Timer(self.flush_interval_secs,
@@ -178,6 +182,9 @@ def try_request(self):
             self.exception_flag = True
 
     def send_request(self, data):
+        # If sampling function's send_check fails, drop the log.  Naive approach, probably a better place to put this.
+        if not self.sampling_instance.send_check(): return True
+
         try:
             response = requests.post(url=self.url,
                                      json=data,

logdna/sampling.py

@@ -0,0 +1,39 @@
+import random
+
+_DEV = False
+
+class Sampling:
+    """
+    Sampling class boilerplate that returns True no matter what.
+
+    send_check is required.  It is the "decision interface".
+    """
+
+    def __init__(self): pass
+
+    def send_check(self):
+        """
+        Decide if a log should be sent using a random selection over some distribution.
+        """
+        return True
+
+
+class UniformSampling(Sampling):
+    """
+    Uniform distribution
+    """
+
+    def __init__(self, send_ratio=1.0):
+        """
+        send_ratio: percentage of logs to let through.  Defaults to always send
+        """
+        if send_ratio > 1.0 or send_ratio < 0.0:
+            send_ratio = 1.0 # set to 1.0 if out of bounds: [0,1]
+        self.send_ratio = send_ratio
+
+    def send_check(self):
+        """
+        Binary send decision based on a uniform distribution.
+        """
+        # Send if random number is less than send_ratio
+        return True if random.uniform(0.0,1.0) < self.send_ratio else False