Skip to content
Permalink
Browse files

Added label filtering feature, add tests

  • Loading branch information...
MichaelVL committed Sep 8, 2016
1 parent c478961 commit 90eae62caf88475e4b35bc8a71814c0bb2e26f90
@@ -1,7 +1,6 @@
import datetime, pytz
import HumanTime
import OsmDiff as osmdiff
import OsmChangeset as oc
import osm.changeset as oc
import pprint

class Backend(object):
@@ -20,7 +19,8 @@ def print_state(self, state):
self.print_chgsets(state.area_chgsets,
state.area_chgsets_info)

def print_chgsets(self, csets, chginfo, print_tags=False):
# FIXME:Delete
def xxprint_chgsets(self, csets, chginfo, print_tags=False):
for chgid in csets[::-1]:
meta = chginfo[chgid]['meta']
#print '>>cset=', pprint.pprint(meta)
@@ -7,7 +7,7 @@
class Backend(Backend.Backend):
def __init__(self, config, subcfg):
super(Backend, self).__init__(config, subcfg)
self.print_meta = getattr(subcfg, 'print_meta', False)
self.print_meta = subcfg.get('print_meta', False)

def print_state(self, db):
strfmt = '%Y:%m:%d %H:%M:%S'
@@ -3,20 +3,20 @@
import json
import datetime
import logging
import tempfilewriter

logger = logging.getLogger(__name__)

class Backend(Backend.Backend):
def __init__(self, config, subcfg):
super(Backend, self).__init__(config, subcfg)
self.list_fname = config.getpath('path', 'BackendGeoJson')+'/'+subcfg['filename']
def __init__(self, globalconfig, subcfg):
super(Backend, self).__init__(globalconfig, subcfg)
self.list_fname = globalconfig.getpath('path', 'tracker')+'/'+subcfg['filename']
self.click_url = subcfg['click_url']
self.print_chgsets(None)

def print_state(self, db):
if self.generation != db.generation:
self.generation = db.generation
if db.chgsets.count() > 0:
if db.chgsets_count() > 0:
self.print_chgsets(db)

def pprint(self, txt):
@@ -73,7 +73,6 @@ def print_chgsets(self, db):
for c in db.chgsets_find(state=db.STATE_DONE):
self.add_cset_bbox(c, db, geoj)

self.start_file(self.list_fname)
logger.debug('Data sent to json file={}'.format(geoj))
self.pprint(json.dumps(geoj))
self.end_file()
with tempfilewriter.TempFileWriter(self.list_fname) as f:
f.write(json.dumps(geoj))
@@ -1,38 +1,36 @@
# -*- coding: utf-8 -*-

from __future__ import print_function
import Backend
import datetime, pytz
import HumanTime
import OsmDiff as osmdiff
import OsmChangeset as oc
import osm.diff as osmdiff
import osm.changeset as oc
import operator
import os
import logging
import jinja2
import tempfilewriter

logger = logging.getLogger(__name__)

class Backend(Backend.Backend):

def __init__(self, config, subcfg):
super(Backend, self).__init__(config, subcfg)
self.list_fname = config.getpath('path', 'BackendHtml')+'/'+subcfg['filename']
def __init__(self, globalconfig, subcfg):
super(Backend, self).__init__(globalconfig, subcfg)
self.list_fname = globalconfig.getpath('path', 'tracker')+'/'+subcfg['filename']
self.template_name = subcfg['template']
self.labels = subcfg.get('labels', None)
self.title = subcfg.get('title', None)

self.show_details = getattr(subcfg, 'show_details', True)
self.show_comments = getattr(subcfg, 'show_comments', True)
self.show_details = subcfg.get('show_details', True)
self.show_comments = subcfg.get('show_comments', True)
self.last_chg_seen = None
self.last_update = datetime.datetime.now()

self.env = jinja2.Environment(loader=jinja2.FileSystemLoader(config.getpath('template_path', 'tracker')))
self.env = jinja2.Environment(loader=jinja2.FileSystemLoader(globalconfig.getpath('template_path', 'tracker')),
trim_blocks=True, lstrip_blocks=True)
self.env.filters['js_datetime'] = self._js_datetime_filter
self.env.filters['utc_datetime'] = self._utc_datetime_filter

self.start_page(self.list_fname)
self.no_items()
self.end_page()

def _js_datetime_filter(self, value):
'''Jinja2 filter formatting timestamps in format understood by javascript'''
# See javascript date/time format: http://tools.ietf.org/html/rfc2822#page-14
@@ -56,19 +54,23 @@ def print_state(self, db):
if self.generation != db.generation:
self.generation = db.generation

self.start_page(self.list_fname)
template = self.env.get_template(self.template_name)
ctx = { 'csets': [],
'csets_err': [],
'csetmeta': {},
'csetinfo': {},
'page_title': self.title,
'show_details': self.show_details,
'show_comments': self.show_comments }
notes = 0
csets_w_notes = 0
csets_w_addr_changes = 0
for c in db.chgsets_find(state=[db.STATE_CLOSED, db.STATE_OPEN, db.STATE_ANALYZING2,
db.STATE_REANALYZING, db.STATE_DONE]):
logger.debug('Cset={}'.format(c))
logger.debug('Backend labels {}, cset labels {}'.format(self.labels, c['labels']))
if self.labels and not set(c['labels']).intersection(self.labels):
continue
cid = c['cid']
ctx['csets'].append(c)
info = db.chgset_get_info(cid)
@@ -86,39 +88,20 @@ def print_state(self, db):
csets_w_notes += 1
if c['state'] != db.STATE_DONE:
continue
if not 'dk_address_node_changes' in info['misc']:
logger.error('No dk_address_node_changes key found')
logger.error('cset: {}'.format(c))
logger.error('meta: {}'.format(meta))
logger.error('info: {}'.format(info))

if int(info['misc']['dk_address_node_changes'])>0:
if 'address_node_changes' in c['labels']:
csets_w_addr_changes += 1
ctx['csets_with_notes'] = csets_w_notes
ctx['csets_with_addr_changes'] = csets_w_addr_changes
logger.debug('Data passed to template: {}'.format(ctx))
self.pprint(template.render(ctx))
self.end_page()

def pprint(self, txt):
print(txt.encode('utf8'), file=self.f)
#print('*'+txt)

def start_page(self, fname, tmp_suffix='_tmp'):
self.filename = fname
self.filename_tmp = fname+tmp_suffix
self.f = open(self.filename_tmp, 'w', os.O_TRUNC)
self.pprint('<!-- Generated by OpenStreetMap Analytic Difference Engine -->')

def end_page(self):
self.f.close()
os.rename(self.filename_tmp, self.filename)
self.f = None
with tempfilewriter.TempFileWriter(self.list_fname) as f:
f.write('<!-- Generated by OpenStreetMap Analytic Difference Engine -->\n')
f.write(template.render(ctx))

def no_items(self, state=None):
if state:
time = state.timestamp.strftime('%Y:%m:%d %H:%M:%S')
else:
time = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
time = time.strftime('%Y:%m:%d %H:%M:%S')
self.pprint('<p>No changesets at '+time+' (UTC)</p>')
return '<p>No changesets at '+time+' (UTC)</p>'
@@ -6,113 +6,115 @@
import operator
import logging
import jinja2
import tempfilewriter

logger = logging.getLogger(__name__)

class Backend(BackendHtml.Backend):

def __init__(self, config, subcfg):
super(Backend, self).__init__(config, subcfg)
self.list_fname = config.getpath('path', 'BackendHtmlSummary')+'/'+subcfg['filename']
def __init__(self, globalconfig, subcfg):
super(Backend, self).__init__(globalconfig, subcfg)
self.list_fname = globalconfig.getpath('path', 'tracker')+'/'+subcfg['filename']
self.template_name = subcfg['template']
self.env = jinja2.Environment(loader=jinja2.FileSystemLoader(config.getpath('template_path', 'tracker')))
self.env = jinja2.Environment(loader=jinja2.FileSystemLoader(globalconfig.getpath('template_path', 'tracker')),
trim_blocks=True, lstrip_blocks=True)
self.env.filters['js_datetime'] = self._js_datetime_filter
self.env.filters['utc_datetime'] = self._utc_datetime_filter
self.print_state(None)
self.horizon_hours = globalconfig.get('horizon_hours','tracker')

def print_state(self, db):
force = True # Because otherwise 'summary_created' timestamp below is not updated
if not db or self.generation != db.generation or force:
self.start_page(self.list_fname)
if not db or not db.pointer:
self.pprint('Nothing here yet')
else:
template = self.env.get_template(self.template_name)
self.generation = db.generation
now = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
dbptr = db.pointer
data = {
'csets': [],
'track_starttime': dbptr['first_pointer']['timestamp'],
'track_endtime': dbptr['timestamp'],
'tracked_hours': (dbptr['timestamp']-dbptr['first_pointer']['timestamp']).total_seconds()/3600,
'summary_created': now,
'pointer_timestamp': dbptr['timestamp'],
'first_seqno': dbptr['first_pointer']['seqno'],
'latest_seqno': dbptr['seqno']-1,
'generation': self.generation
}
cset_tracked_hours = 0
users = {}
notes = 0
csets_w_notes = 0
csets_w_addr_changes = 0
for c in db.chgsets_find(state=[db.STATE_CLOSED, db.STATE_OPEN, db.STATE_ANALYZING2,
db.STATE_REANALYZING, db.STATE_DONE]):
data['csets'].append(c)
cid = c['cid']
meta = db.chgset_get_meta(cid)
info = db.chgset_get_info(cid)
user = meta['user']
users[user] = users.get(user,0) + 1
if meta['open'] or (info and 'truncated' in info['state']):
continue
notecnt = int(meta['comments_count'])
if notecnt > 0:
notes += int(meta['comments_count'])
csets_w_notes += 1
if c['state'] != db.STATE_DONE:
continue
if int(info['misc']['dk_address_node_changes'])>0:
csets_w_addr_changes += 1
data['csets_with_notes'] = csets_w_notes
data['csets_with_addr_changes'] = csets_w_addr_changes
return
template = self.env.get_template(self.template_name)
self.generation = db.generation
now = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
dbptr = db.pointer
data = {
'csets': [],
'track_starttime': dbptr['first_pointer']['timestamp'],
'track_endtime': dbptr['timestamp'],
'tracked_hours': (dbptr['timestamp']-dbptr['first_pointer']['timestamp']).total_seconds()/3600,
'summary_created': now,
'pointer_timestamp': dbptr['timestamp'],
'first_seqno': dbptr['first_pointer']['seqno'],
'latest_seqno': dbptr['seqno']-1,
'generation': self.generation
}
cset_tracked_hours = (dbptr['timestamp']-dbptr['first_pointer']['timestamp']).total_seconds()/3600
cset_tracked_hours = min(cset_tracked_hours, self.horizon_hours)
users = {}
notes = 0
csets_w_notes = 0
csets_w_addr_changes = 0
for c in db.chgsets_find(state=[db.STATE_CLOSED, db.STATE_OPEN, db.STATE_ANALYZING2,
db.STATE_REANALYZING, db.STATE_DONE]):
data['csets'].append(c)
cid = c['cid']
meta = db.chgset_get_meta(cid)
info = db.chgset_get_info(cid)
user = meta['user']
users[user] = users.get(user,0) + 1
if meta['open'] or (info and 'truncated' in info['state']):
continue
notecnt = int(meta['comments_count'])
if notecnt > 0:
notes += int(meta['comments_count'])
csets_w_notes += 1
if c['state'] != db.STATE_DONE:
continue
if 'address-node-change' in c['labels']: #FIXME: does not belong here - this is configuration
csets_w_addr_changes += 1
data['csets_with_notes'] = csets_w_notes
data['csets_with_addr_changes'] = csets_w_addr_changes

# Summarize edits and mileage - we don't do this incrementally
# since csets can be split over multiple diffs
edits = {'node': {'create':0, 'modify':0, 'delete':0},
'way': {'create':0, 'modify':0, 'delete':0},
'relation': {'create':0, 'modify':0, 'delete':0}}
mileage = {}
for c in db.chgsets_find(state=db.STATE_DONE):
cid = c['cid']
info = db.chgset_get_info(cid)
if 'truncated' in info['state']:
continue
summary = info['summary']
for action in ['create', 'modify', 'delete']:
if summary['_'+action] > 0:
for type in ['node', 'way', 'relation']:
edits[type][action] += summary[action][type]
self.merge_int_dict(mileage, info['mileage_m'])
data['edits'] = edits
data['users'] = users
data['notes'] = notes
logger.debug('Accumulated mileage: {}'.format(mileage))
# Summarize edits and mileage - we don't do this incrementally
# since csets can be split over multiple diffs
edits = {'node': {'create':0, 'modify':0, 'delete':0},
'way': {'create':0, 'modify':0, 'delete':0},
'relation': {'create':0, 'modify':0, 'delete':0}}
mileage = {}
for c in db.chgsets_find(state=db.STATE_DONE):
cid = c['cid']
info = db.chgset_get_info(cid)
if 'truncated' in info['state']:
continue
summary = info['summary']
for action in ['create', 'modify', 'delete']:
if summary['_'+action] > 0:
for type in ['node', 'way', 'relation']:
edits[type][action] += summary[action][type]
self.merge_int_dict(mileage, info['mileage_m'])
data['edits'] = edits
data['users'] = users
data['notes'] = notes
logger.debug('Accumulated mileage: {}'.format(mileage))

mileage_bytype = []
if mileage:
sum = 0
num_items = 0
by_type = mileage['by_type']
for cat in by_type.keys():
mi = [(t,int(by_type[cat][t])) for t in by_type[cat].keys()]
mi = sorted(mi, key=lambda x: x[1], reverse=True)
for typ,m in mi:
if num_items < 13:
mileage_bytype.append((cat, typ, self._i2s(m)))
sum += m
num_items += 1
data['mileage_bytype'] = mileage_bytype
data['mileage_meter'] = self._i2s(sum)
# Rounding means that if cset_tracked_hours=0, then we pretend the current
# metrics are for one hours
data['mileage_meter_per_hour'] = self._i2s(int(sum/max(1,cset_tracked_hours)))
mileage_bytype = []
if mileage:
sum = 0
num_items = 0
by_type = mileage['by_type']
for cat in by_type.keys():
mi = [(t,int(by_type[cat][t])) for t in by_type[cat].keys()]
mi = sorted(mi, key=lambda x: x[1], reverse=True)
for typ,m in mi:
if num_items < 13:
mileage_bytype.append((cat, typ, self._i2s(m)))
sum += m
num_items += 1
data['mileage_bytype'] = mileage_bytype
data['mileage_meter'] = self._i2s(sum)
# Rounding means that if cset_tracked_hours=0, then we pretend the current
# metrics are for one hours
data['mileage_meter_per_hour'] = self._i2s(int(sum/max(1,cset_tracked_hours)))

#if hasattr(state, 'pointer'): # FIXME
# lag = now-state.pointer.timestamp()
# data['lag_seconds'] = int(lag.seconds)
#if hasattr(state, 'pointer'): # FIXME
# lag = now-state.pointer.timestamp()
# data['lag_seconds'] = int(lag.seconds)

logger.debug('Data passed to template: {}'.format(data))
self.pprint(template.render(data))
self.end_page()
logger.debug('Data passed to template: {}'.format(data))
with tempfilewriter.TempFileWriter(self.list_fname) as f:
f.write('<!-- Generated by OpenStreetMap Analytic Difference Engine -->')
f.write(template.render(data))
Oops, something went wrong.

0 comments on commit 90eae62

Please sign in to comment.
You can’t perform that action at this time.