Permalink
Browse files

Playing around to get sophox to be one of the sources (besides .pbf)

  • Loading branch information...
stalker314314 committed Dec 1, 2017
1 parent 609e8d4 commit f4c76a4a104dbd461f1d5a5fcd72a98f136afa95
View
@@ -8,8 +8,9 @@ However, it is not constrained to just Serbian checks on OSM, it _is_ general pu
It features:
* Support for Windows/Linux/Mac, Python 3.x
* Support for both (fast) [PyOsmium](http://osmcode.org/pyosmium/) library,
* Support for reading PBF files using (fast) [PyOsmium](http://osmcode.org/pyosmium/) library,
as well as [osmread](https://github.com/dezhin/osmread) library (PyOsmium seems 2x-5x faster than osmread, but I dare you to compile it on Windows:)
* Support for [Sophox SPARQL queries](https://wiki.openstreetmap.org/wiki/Sophox)
* extensive configuration support
* support for both executing reports on checks and automated/semi-automated OSM editing
* fully tested and documented
View
@@ -67,5 +67,11 @@
"checks.LatinNameSameAsCyrillicCheck",
"checks.LatinNameNotInCyrillicCheck"
]
},
"Serbia-AddingSrLatn": {
"location": "sparql/adding_sr_latn.sparql",
"checks": [
"checks.GenericSophoxCheck"
]
}
}
View
@@ -7,4 +7,5 @@ mwparserfromhell==0.5
requests==2.18.4
urllib3==1.22
osmium==2.13.0
simplejson==3.11.1
simplejson==3.11.1
SPARQLWrapper==1.8.0
@@ -0,0 +1,41 @@
#defaultView:Editor{ "comment":"Automatsko prebacivanje u cirilicu", "vote":false }
SELECT
?id ?name ?loc
('name:sr-Latn' AS ?tag_1) (?correctLatinName as ?val_1)
WHERE {
?id osmt:name ?name ;
osmt:highway ?highway ; # Ovim se filtriraju samo ulice. Zakomentarisati za sve entitete
osmm:type ?mtype .
FILTER (regex(str(?name), "а|б|в|г|д|ђ|е|ж|з|и|ј|к|л|љ|м|н|њ|о|п|р|с|т|ћ|у|ф|х|ц|ч|џ|ш", "i")) . # Ovako filtriramo samo cirilicna imena
BIND(
REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(
REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(
REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(
REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(
REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(
REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(
?name,
# Mala slova
'а', 'a'), 'б', 'b'), 'в', 'v'), 'г', 'g'), 'д', 'd'), 'ђ', 'đ'), 'е', 'e'), 'ж', 'ž'), 'з', 'z'), 'и', 'i'),
'ј', 'j'), 'к', 'k'), 'л', 'l'), 'љ', 'lj'), 'м', 'm'), 'н', 'n'), 'њ', 'nj'), 'о', 'o'), 'п', 'p'), 'р', 'r'),
'с', 's'), 'т', 't'), 'ћ', 'ć'), 'у', 'u'), 'ф', 'f'), 'х', 'h'), 'ц', 'c'), 'ч', 'č'), 'џ', 'dž'), 'ш', 'š'),
# Velika slova
'А', 'A'), 'Б', 'B'), 'В', 'V'), 'Г', 'G'), 'Д', 'D'), 'Ђ', 'Đ'), 'Е', 'E'), 'Ж', 'Ž'), 'З', 'Z'), 'И', 'I'),
'Ј', 'J'), 'К', 'K'), 'Л', 'L'), 'Љ', 'Lj'), 'М', 'M'), 'Н', 'N'), 'Њ', 'Nj'), 'О', 'O'), 'П', 'P'), 'Р', 'R'),
'С', 'S'), 'Т', 'T'), 'Ћ', 'Ć'), 'У', 'U'), 'Ф', 'F'), 'Х', 'H'), 'Ц', 'C'), 'Ч', 'Č'), 'Џ', 'Dž'), 'Ш', 'Š')
as ?correctLatinName)
SERVICE wikibase:around {
?id osmm:loc ?loc .
# Koordinata centra Beograda, zameni da dobijes krug sa centrom u drugoj tacki
bd:serviceParam wikibase:center "Point(20.45340 44.79338)"^^geo:wktLiteral.
# Precnik kruga u kilometrima, povecaj/smanji po zelji
bd:serviceParam wikibase:radius "15" .
}
OPTIONAL { ?id osmt:name:sr-Latn ?currentLatinName . } # Uzmi trenutno latinicno ime, ako postoji
FILTER (NOT EXISTS { ?id osmt:name:sr-Latn ?namesrlatn . } || ?correctLatinName != ?currentLatinName) # Napravi ime ako ne postoji ili ako se ne slaze
# FILTER (?mtype = 'w') . # Stavite 'n', 'w' ili 'r' ako zelite da filtrirate po tipu entiteta
}
View
@@ -38,4 +38,13 @@ class Village(Applicability):
"""
@staticmethod
def is_entity_applicable(entity):
return entity and 'place' in entity.tags and entity.tags['place'] == 'village'
return entity and 'place' in entity.tags and entity.tags['place'] == 'village'
class SophoxEntity(Applicability):
"""
Check that entity is coming form Sophox, no other questions asked
"""
@staticmethod
def is_entity_applicable(entity):
return entity.origin == 'sophox'
View
@@ -3,7 +3,7 @@
import pywikibot
import tools
from applicability import City, Town, Village
from applicability import City, Town, Village, SophoxEntity
from exceptions import CalculateDistanceException
from haversine import haversine
from transliteration import at_least_some_in_cyrillic, cyr2lat
@@ -175,6 +175,12 @@ def ask_confirmation(self, input_text, entity):
:return: True if user confirmed, False otherwise
"""
for k, v in entity.tags.items():
# Ignore following (coming from Sophox), just polluting output
if k == 'metadata':
continue
if k.startswith('tag_') or k.startswith('val_'):
continue
print('{0}: {1}'.format(k, v))
print('https://www.openstreetmap.org/{0}/{1}'.format(entity.entity_type, entity.id))
response = input('[{0}] {1} (Y/n)?'.format(self.map, input_text))
@@ -690,3 +696,88 @@ def fix(self, entity, api):
api.NodeUpdate(node)
return 'is_in:country for node {0} was missing, added it to be "{1}"'.format(name, 'Serbia')
return ''
class GenericSophoxCheck(AbstractCheck):
"""
Check semantically same to Sophox service. Will also fix things if there are (tag_N, val_N) pairs.
"""
applicable_on = [SophoxEntity]
is_fixable = True
def __init__(self, entity_context):
super(GenericSophoxCheck, self).__init__(entity_context)
def do_check(self, entity):
# By definition, everything returned from Sophox is not passing check
name = entity.tags['name'] if 'name' in entity.tags else entity.id
comment = entity.tags['metadata']['comment'] if 'comment' in entity.tags['metadata'] else 'no description'
return 'Entity {0}: {1}'.format(name, comment)
def fix(self, entity, api):
name = entity.tags['name'] if 'name' in entity.tags else entity.id
changed_anything = False
if entity.entity_type == 'way':
osm_entity = api.WayGet(entity.id)
elif entity.entity_type == 'node':
osm_entity = api.NodeGet(entity.id)
else:
osm_entity = api.RelationGet(entity.id)
# Check for all suggestions
suggestion_id = 1
while True:
tag_key = 'tag_{0}'.format(suggestion_id)
value_key = 'val_{0}'.format(suggestion_id)
if tag_key not in entity.tags: # If there is no this tag, we reached the end
break
tag = entity.tags[tag_key]['value']
val = entity.tags[value_key]['value']
if val == 'false' and entity.tags[value_key]['datatype'] == 'http://www.w3.org/2001/XMLSchema#boolean':
val = None
if val is not None:
# We are adding/modifying value
if tag not in osm_entity['tag']:
# Case that it doesn't exist
question = 'Are you really sure you want to add tag "{0}" with value "{1}" to entity "{2}"'.format(
tag, val, name
)
else:
existing_value = osm_entity['tag']
if existing_value != val:
# Case that we need to modify value
question = 'Are you really sure you want to change tag "{0}" from value {1} to value "{2}" ' \
'for entity "{3}"'.format(tag, osm_entity['tag'][tag], val, name)
else:
# Case that value is already set as per query, so no changes needed
question = ''
else:
# We are trying to delete value
if tag in osm_entity['tag']:
question = 'Are you really sure you want to delete tag "{0}" with value {1} for entity "{2}"'\
.format(tag, osm_entity['tag'][tag], name)
else:
# Already deleted
question = ''
if question != '' and self.ask_confirmation(question, entity):
changed_anything = True
if val is not None:
osm_entity['tag'][tag] = val
else:
del osm_entity['tag'][tag]
if not self.dry_run:
if entity.entity_type == 'way':
api.WayUpdate(osm_entity)
elif entity.entity_type == 'node':
api.NodeUpdate(osm_entity)
else:
api.RelationUpdate(osm_entity)
suggestion_id = suggestion_id + 1
if changed_anything:
return 'Fixes made'
else:
return ''
Oops, something went wrong.

0 comments on commit f4c76a4

Please sign in to comment.