This repository has been archived by the owner on Mar 24, 2021. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #173 from alphagov/max_age_expected
Add max_age_expected to model and pep8
- Loading branch information
Showing
2 changed files
with
72 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
""" | ||
Add last time we expected to get data to buckets - max age expected | ||
data types at the time of writing this | ||
"journey", | ||
"monitoring", | ||
"realtime", | ||
"channels", | ||
"customer-satisfaction", | ||
"failures", | ||
"services", | ||
"volumetrics", | ||
"annotations", | ||
"volumes", | ||
"application", | ||
"test", | ||
"search-volumetrics", | ||
"annual-mortgage-lending", | ||
"monthly-mortgage-lending", | ||
"dwellings-completes", | ||
"dwellings-starts", | ||
"first-time-buyer", | ||
"house-price-index", | ||
"residential-transactions" | ||
""" | ||
import logging | ||
|
||
log = logging.getLogger(__name__) | ||
|
||
|
||
def up(db): | ||
all_buckets = db['buckets'].find() | ||
for bucket in all_buckets: | ||
if 'max_age_expected' in bucket: | ||
continue | ||
max_age_expected = calculate_max_age(bucket) | ||
bucket['max_age_expected'] = max_age_expected | ||
log.info("Adding max age of: %s to %s bucket" % | ||
(max_age_expected, bucket['name'])) | ||
db['buckets'].save(bucket) | ||
|
||
|
||
def calculate_max_age(bucket): | ||
dt = bucket['data_type'] | ||
|
||
return { | ||
'realtime': minutes(5), # 5mins for realtime buckets | ||
'journey': hours(25), # Journey buckets should update daily | ||
# Jobs run every hour, we're giving 2hrs tolerance | ||
'monitoring': hours(2), | ||
}.get(dt, months(1)) # default to a month | ||
|
||
|
||
def months(months): | ||
# Possibly not the most accurate comparison ever written. | ||
return months * hours(24) * 31 | ||
|
||
|
||
def hours(hours): | ||
return hours * (60 * 60) | ||
|
||
|
||
def minutes(minutes): | ||
return minutes * 60 |