/
database_logger.py
126 lines (107 loc) · 4.7 KB
/
database_logger.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
# -*- coding: utf-8 -*-
import logging
import time
import psycopg2
from c2cwsgiutils import stats
logger = logging.getLogger(__name__)
class DatabaseLoggerCommon(object): # pragma: no cover
def __init__(self, config, daemon):
db_params = config['database']
while True:
try:
self.connection = psycopg2.connect(
dbname=db_params['dbname'],
host=db_params.get('host'),
port=db_params.get('port'),
user=db_params.get('user'),
password=db_params.get('password')
)
break
except psycopg2.OperationalError as e:
if daemon:
logger.warning("Failed connecting to the database. Will try again in 1s", exc_info=True)
time.sleep(1)
else:
exit("Cannot connect to the database: " + str(e))
if '.' in db_params['table']:
schema, table = db_params['table'].split('.')
else:
schema = 'public'
table = db_params['table']
with self.connection.cursor() as cursor:
cursor.execute(
"SELECT EXISTS(SELECT 1 FROM pg_tables WHERE schemaname=%s AND tablename=%s)",
(schema, table)
)
schema = psycopg2.extensions.quote_ident(schema, self.connection)
table = psycopg2.extensions.quote_ident(table, self.connection)
if not cursor.fetchone()[0]:
try:
cursor.execute(
'CREATE TABLE {}.{} ('
' id BIGSERIAL PRIMARY KEY,'
' layer CHARACTER VARYING(80) NOT NULL,'
' run INTEGER NOT NULL,'
' action CHARACTER VARYING(7) NOT NULL,'
' tile TEXT NOT NULL,'
' UNIQUE (layer, run, tile))'.format(schema, table)
)
self.connection.commit()
except psycopg2.DatabaseError:
logging.error('Unable to create table %s.%s', schema, table, exc_info=1)
exit(1)
else:
try:
cursor.execute(
'INSERT INTO {}.{}(layer, run, action, tile) '
'VALUES (%s, %s, %s, %s)'.format(schema, table),
('test_layer', -1, 'test', '-1x-1')
)
except psycopg2.DatabaseError:
logging.error('Unable to insert logging data into %s.%s',
schema, table, exc_info=1)
exit(1)
finally:
self.connection.rollback()
self.full_table = '{}.{}'.format(schema, table)
class DatabaseLoggerInit(DatabaseLoggerCommon): # pragma: no cover
def __init__(self, config, daemon):
super(DatabaseLoggerInit, self).__init__(config, daemon)
with self.connection.cursor() as cursor:
cursor.execute('SELECT COALESCE(MAX(run), 0) + 1 FROM {}'.format(self.full_table))
self.run, = cursor.fetchone()
def __call__(self, tile):
tile.metadata['run'] = self.run
return tile
class DatabaseLogger(DatabaseLoggerCommon): # pragma: no cover
def __call__(self, tile):
if tile is None:
logger.warning("The tile is None")
return None
if tile.error:
action = 'error'
elif tile.data:
action = 'create'
else:
action = 'delete'
layer = tile.metadata.get('layer', '- No layer -')
run = tile.metadata.get('run', -1)
with stats.timer_context(['db_logger', 'insert']):
with self.connection.cursor() as cursor:
try:
cursor.execute(
'INSERT INTO {} (layer, run, action, tile) '
'VALUES (%(layer)s, %(run)s, %(action)s::varchar(7), %(tile)s)'.
format(self.full_table),
{'layer': layer, 'action': action, 'tile': str(tile.tilecoord), 'run': run}
)
except psycopg2.IntegrityError:
self.connection.rollback()
cursor.execute(
'UPDATE {} SET action = %(action)s '
'WHERE layer = %(layer)s AND run = %(run)s AND tile = %(tile)s'.
format(self.full_table),
{'layer': layer, 'action': action, 'tile': str(tile.tilecoord), 'run': run}
)
self.connection.commit()
return tile