Add max_age_expected to model and pep8

backdrop/core/bucket.py

@@ -9,6 +9,7 @@
 
 
 class Bucket(object):
+
     def __init__(self, db, config):
         self.bucket_name = config.name
         self.repository = db.get_repository(config.name)
@@ -50,14 +51,15 @@ def _generate_id(self, datum):
 _BucketConfig = namedtuple(
     "_BucketConfig",
     "name data_group data_type raw_queries_allowed bearer_token upload_format "
-    "upload_filters auto_ids queryable realtime capped_size")
+    "upload_filters auto_ids queryable realtime capped_size max_age_expected")
 
 
 class BucketConfig(_BucketConfig):
+
     def __new__(cls, name, data_group, data_type, raw_queries_allowed=False,
                 bearer_token=None, upload_format="csv", upload_filters=None,
                 auto_ids=None, queryable=True, realtime=False,
-                capped_size=5040):
+                capped_size=5040, max_age_expected=2678400):
         if not bucket_is_valid(name):
             raise ValueError("Bucket name is not valid")
 
@@ -71,8 +73,9 @@ def __new__(cls, name, data_group, data_type, raw_queries_allowed=False,
                                                 bearer_token, upload_format,
                                                 upload_filters, auto_ids,
                                                 queryable, realtime,
-                                                capped_size)
+                                                capped_size, max_age_expected)
 
     @property
     def max_age(self):
+        """ Set cache-control header length based on type of bucket. """
         return 120 if self.realtime else 1800

migrations/006_add_last_time_expected.py

@@ -0,0 +1,66 @@
+"""
+Add last time we expected to get data to buckets - max age expected
+
+data types at the time of writing this
+
+    "journey",
+    "monitoring",
+    "realtime",
+    "channels",
+    "customer-satisfaction",
+    "failures",
+    "services",
+    "volumetrics",
+    "annotations",
+    "volumes",
+    "application",
+    "test",
+    "search-volumetrics",
+    "annual-mortgage-lending",
+    "monthly-mortgage-lending",
+    "dwellings-completes",
+    "dwellings-starts",
+    "first-time-buyer",
+    "house-price-index",
+    "residential-transactions"
+
+"""
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def up(db):
+    all_buckets = db['buckets'].find()
+    for bucket in all_buckets:
+        if 'max_age_expected' in bucket:
+            continue
+        max_age_expected = calculate_max_age(bucket)
+        bucket['max_age_expected'] = max_age_expected
+        log.info("Adding max age of: %s to %s bucket" %
+                 (max_age_expected, bucket['name']))
+        db['buckets'].save(bucket)
+
+
+def calculate_max_age(bucket):
+    dt = bucket['data_type']
+
+    return {
+        'realtime': minutes(5),  # 5mins for realtime buckets
+        'journey': hours(25),  # Journey buckets should update daily
+        # Jobs run every hour, we're giving 2hrs tolerance
+        'monitoring': hours(2),
+    }.get(dt, months(1))  # default to a month
+
+
+def months(months):
+    # Possibly not the most accurate comparison ever written.
+    return months * hours(24) * 31
+
+
+def hours(hours):
+    return hours * (60 * 60)
+
+
+def minutes(minutes):
+    return minutes * 60