/
count-sandbox-logs.py
executable file
·92 lines (84 loc) · 2.32 KB
/
count-sandbox-logs.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Count router logs per org per sandbox per month over the last three months.
"""
import os
import csv
import datetime
import requests
from dateutil import relativedelta
query = {
'size': 0,
'query': {
'bool': {
'must': [
{
'prefix': {
'@cf.org': 'sandbox-'
}
},
{
'term': {
'@source.type': 'RTR'
}
}
]
}
},
'aggs': {
'count': {
'terms': {
'field': '@cf.org',
'size': 999
},
'aggs': {
'count': {
'terms': {
'field': '@cf.space',
'size': '999'
},
'aggs': {
'count': {
'date_histogram': {
'field': '@timestamp',
'interval': 'month'
}
}
}
}
}
}
}
}
today = datetime.date.today()
first = datetime.date(today.year, today.month, 1)
indices = [
(first - relativedelta.relativedelta(months=idx)).strftime('logs-app-%Y.%m.*')
for idx in reversed(range(3))
]
months = [
(first - relativedelta.relativedelta(months=idx)).strftime('%Y-%m-%d')
for idx in reversed(range(3))
]
result = requests.get(
'http://{}:9200/{}/_search'.format(os.environ['ES_HOST'], ','.join(indices)),
json=query,
).json()
rows = []
for org_bucket in result['aggregations']['count']['buckets']:
org = org_bucket['key']
for space_bucket in org_bucket['count']['buckets']:
space = space_bucket['key']
row = {
'org': org,
'space': space,
}
for month_bucket in space_bucket['count']['buckets']:
month = month_bucket['key_as_string']
row[month.split('T')[0]] = month_bucket['doc_count']
rows.append(row)
with open('summary.csv', 'w') as fp:
writer = csv.DictWriter(fp, fieldnames=['org', 'space'] + months)
writer.writeheader()
writer.writerows(rows)