Permalink
Switch branches/tags
Nothing to show
Find file
Fetching contributors…
Cannot retrieve contributors at this time
executable file 280 lines (241 sloc) 10.2 KB
#!/usr/bin/env python
# Take a file and of descriptions, multi-line and separated by blank lines,
# and turn it into a collection of GPX waypoints
# suitable for import into Osmand, PyTopo or other mapping programs.
# Copyright 2013 by Akkana Peck <akkana@shallowsky.com>.
# Please share and enjoy under the GPL v2 or later.
import sys, os
import re
import cgi
import datetime
import time
# import googlemaps # need this for exceptions
# from googlemaps import GoogleMaps
import urllib.request, urllib.parse, urllib.error, json, csv
def write_gpx_file(entries, filename, omit_address=True):
'''Write the list of entries -- each entry is [lat, long, desc] --
to a GPX file as separate waypoints.
'''
fp = open(filename, 'w')
fp.write('''<?xml version="1.0" encoding="UTF-8"?>
<gpx
version="1.0"
creator="makeway v. 0.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://www.topografix.com/GPX/1/0"
xsi:schemaLocation="http://www.topografix.com/GPX/1/0 http://www.topografix.com/GPX/1/0/gpx.xsd">
''')
fp.write('<time>%s</time>\n' % datetime.datetime.now().isoformat())
# Calculate our bounds:
minlat = 91
maxlat = -90
minlon = 1000
maxlon = -1000
for ent in entries:
if ent[0] < minlat:
minlat = ent[0]
if ent[0] > maxlat:
maxlat = ent[0]
if ent[1] < minlon:
minlon = ent[1]
if ent[1] > maxlon:
maxlon = ent[1]
fp.write('<bounds minlat="%f" minlon="%f" maxlat="%f" maxlon="%f"/>\n' \
% (minlat, minlon, maxlat, maxlon))
for ent in entries:
print('<wpt lat="%f" lon="%f">' % (ent[0], ent[1]), file=fp)
if omit_address:
addy = '\n'.join(ent[2].split('\n')[2:])
else:
addy = ent[2]
print('<name>%s</name>' % addy, file=fp)
print('</wpt>', file=fp)
fp.write('</gpx>\n')
#
# Replacement for googlemaps API:
# http://stackoverflow.com/questions/18807114/http-error-403-with-api-id-in-accessing-google-maps
#
over_query_limit = False
# Explanation of google's query limit:
# https://developers.google.com/maps/documentation/business/articles/usage_limits
def geocode(addr):
'''Use Google's geocoding URL to convert a string address
to latitude and longitude.
Returns a (lat, lon) pair, or None, None.
'''
global over_query_limit
if over_query_limit:
time.sleep(2)
else:
time.sleep(.1)
# Currently google's limit is 10 queries/second, so if we sleep
# for 1/10 second we should hopefully stay under the rate limit.
# https://developers.google.com/maps/documentation/business/faq#usage_limits
url = "http://maps.googleapis.com/maps/api/geocode/json?address=%s&sensor=false" % (urllib.parse.quote(addr.replace(' ', '+')))
try:
data = urllib.request.urlopen(url).read()
loaded = json.loads(data)
if loaded['status'] == 'OVER_QUERY_LIMIT':
print("Hit the Google Maps query limit; sleeping for 2 seconds between queries")
over_query_limit = True
time.sleep(2.5)
data = urllib.request.urlopen(url).read()
loaded = json.loads(data)
# If we've already waited the 2 seconds and Google still says
# we're over limit, then we've hit the (unspecified) daily limit
# and need to give up.
if over_query_limit and loaded['status'] == 'OVER_QUERY_LIMIT':
print("Hit Google's daily quota; giving up")
sys.exit(1)
location = loaded["results"][0]["geometry"]["location"]
except IndexError as e:
print("Error reading JSON for %s" % addr)
print(loaded)
return None, None
return location['lat'], location['lng']
def read_description_file(filename):
'''Read a file filled with multi-line descriptions, blank line separated.
The first line of each description is a latitude and longitude,
separated by whitespace.
The rest is free-form description.
Returns a list of entries, where each entry is a list:
[ latitude, longitude, text ]
'''
entries = []
cur_ent = []
# In case we need to look up an address, we'll need to initialize
# Google Maps, but we only want to do that once.
gmaps = None
fp = open(filename)
for line in fp:
line = line.strip()
if not line: # end of a record. Save the current entry and move on
if cur_ent:
# print "Appending entry for", cur_ent[2]
entries.append(cur_ent)
cur_ent = []
continue
if not cur_ent:
numeric = '[\+\-\d\.]'
# re.search doesn't work if you put the % expression in the
# search call, but it does work if you store the
# intermediate string first:
twonums = '^(%s+)\s+(%s+)$' % (numeric, numeric)
match = re.search(twonums, line)
if match:
# Okay, they may be numbers, but that doesn't mean
# they're coordinates. Consider 23054 7250 Rd.
# So let's do a sanity check:
lat = float(match.group(1))
lon = float(match.group(2))
if lat >= -90 and lat <= 90 and lon >= -180 and lon <= 360:
cur_ent.append(lat)
cur_ent.append(lon)
# Start cur_ent[2] with a null string:
cur_ent.append('')
continue
# If the numbers didn't pass the sanity check,
# fall through to the address parser.
# Now either the first line, or the first two lines,
# are an address. But we should be able to tell them apart:
# The last two fields of an address are a state
# (2 uppercase letters) followed by a 5-digit zip code.
statezip = '.*[A-Z]{2}\s+\d{5}$'
match = re.search(statezip, line)
# If the state/zip wasn't in the first line, try the second:
line2 = None
if not match:
# Try guards against StopIteration, i.e. end of file
try:
line2 = next(fp).strip()
if not line2:
# a blank line here means the previous line
# wasn't meant to be the start of an entry anyway --
# probably just a stray URL or something.
continue
match = re.search(statezip, line2)
if not match:
print("Couldn't find coordinates OR address in '%s' or in '%s'" % (line, line2))
print("Skipping this entry")
while True:
line = next(fp).strip()
if not line:
break
# Now continue the outer loop --
# don't try to process this blank line as an address.
continue
except StopIteration:
# StopIteration means the end of the file.
# Since this clause is only to start a new entry,
# that means we can return without doing any cleanup
# except closing the file.
# (montrose.txt is a good test case --
# or anything that ends with a blank line.)
print("StopIteration")
fp.close()
return entries
# There's a match! Either single or double line.
# Either way, look it up and add it to the desc.
addr = line
if line2:
addr += ' ' + line2
print("Found an address! %s" % addr)
# Google Maps has shut off access to this API,
# perhaps only temporarily:
# Look up an address using map search.
# This requires a Google Maps API key and
# the Python GoogleMaps package (pip install googlemaps).
# http://py-googlemaps.sourceforge.net/
#if not gmaps:
# # We only want to initialize Gmaps with the API key once.
# print "Initializing Google Maps API"
# gmaps = GoogleMaps('YOUR GOOGLE MAPS API KEY HERE')
#
# try:
# lat, lon = gmaps.address_to_latlng(addr)
# except googlemaps.GoogleMapsError, e:
# print("Oh, no! Couldn't geocode for %s" % addr)
# print(e)
# So instead, use a local version:
lat, lon = geocode(addr)
if not lat:
cur_ent = []
continue
cur_ent.append(lat)
cur_ent.append(lon)
# XXX Need to remove special characters XML can't handle:
# ? & ( ) ' "
# and append the address as the first part of the description:
if line2:
cur_ent.append(cgi.escape(line) + '\n' + cgi.escape(line2))
else:
cur_ent.append(cgi.escape(line))
continue
# Else we have a non-null line AND we have a current entry,
# so we're just appending to cur_ent[2].
# But skip lines that have any long words that are likely
# too long to wrap on a phone display (they're probably URLs).
if re.search('\S{27,}', line):
print("Skipping long line: '%s'" % line)
continue
if cur_ent[2]:
cur_ent[2] += '\n' + cgi.escape(line)
else:
cur_ent[2] += cgi.escape(line)
if cur_ent:
entries.append(cur_ent)
fp.close()
return entries
def Usage():
print("Usage: %s infile.txt outfile.gpx" % os.path.basename(sys.argv[0]))
sys.exit(1)
if __name__ == "__main__" :
if len(sys.argv) < 3:
Usage()
# It would be relatively easy to mess up with autocomplete and
# run makeway foo.txt foo.txt. That would be bad.
if not(sys.argv[2].endswith('.gpx')):
print("Output file %s doesn't end with .gpx" % sys.argv[2])
Usage()
entries = read_description_file(sys.argv[1])
write_gpx_file(entries, sys.argv[2])