Altitude (#1645)

* Altitude

* generating random altitude

* Travis

* Update adjust proper range

* Update for Spawn Point Scanning

* Update Elevation Request

* Attempt to receive altitude via Elevation

* Updated utils

* Altitude for SpeedScan

* Extract repeated code into get_altitude

* Make altitude range param conform with the other params

* Store location altitudes in database

* Rename altitude range to altitude variance

* Try to keep unrelated things as they were originally

* Log error whe get_gmaps_altitude fails

* Replace repeated code in runserver

* Get nearby altitudes

* Improve args help

* Add option to not cache altitudes

* Change default altitude

Default based on the average elevation of cities around the world.
Source: https://www.wikiwand.com/en/List_of_cities_by_elevation

* Fix grammar

* Move altitude stuff to its own file

* Rename no altitude cache argument

* Stop the map if elevation is not set in GMaps API

* Good bye double quotes

* Fix wrong call, should be no_altitude_cache

* Update documentation with Elevation API info

docs/basic-install/google-maps.md

@@ -25,7 +25,7 @@ This project uses Google Maps. Each instance of Google Maps requires an API key
    ![API Browser Key](../_static/img/csEFWKd.png)
    ![API Browser Key](../_static/img/6upJVIr.png)
 
-5. Enable two Google Maps APIs
+5. Enable three Google Maps APIs
    - Google Maps Javascript API - Enables Displaying of Map
      - Click on 'Library'
      - Click on Google Maps Javascript API
@@ -35,6 +35,11 @@ This project uses Google Maps. Each instance of Google Maps requires an API key
      - Type 'Places' into the search box ' Search all 100+ APIs'
      - Choose Google Places API Web Service
      - Click 'ENABLE'
+   - Google Maps Elevation API - Enables fetching of altitude
+     - Click on 'Library'
+     - Type 'Elevation' into the search box ' Search all 100+ APIs'
+     - Choose Google Maps Elevation API
+     - Click 'ENABLE'
 
 ## Using the API Key
 

pogom/altitude.py

@@ -0,0 +1,80 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+import logging
+import requests
+import random
+from .models import LocationAltitude
+
+log = logging.getLogger(__name__)
+
+# Altitude used when no_altitude_cache is enabled
+fallback_altitude = None
+
+
+def get_gmaps_altitude(lat, lng, gmaps_key):
+    try:
+        r_session = requests.Session()
+        response = r_session.get((
+            'https://maps.googleapis.com/maps/api/elevation/json?' +
+            'locations={},{}&key={}').format(lat, lng, gmaps_key))
+        response = response.json()
+        status = response['status']
+        results = response.get('results', [])
+        result = results[0] if results else {}
+        altitude = result.get('elevation', None)
+    except:
+        log.error('Unable to retrieve altitude from Google APIs.')
+        status = 'UNKNOWN_ERROR'
+        altitude = None
+
+    return (altitude, status)
+
+
+def randomize_altitude(altitude, altitude_variance):
+    if altitude_variance > 0:
+        altitude = (altitude +
+                    random.randrange(-1 * altitude_variance,
+                                     altitude_variance) +
+                    float(format(random.random(), '.13f')))
+    else:
+        altitude = altitude + float(format(random.random(), '.13f'))
+
+    return altitude
+
+
+# Only once fetched altitude
+def get_fallback_altitude(args, loc):
+    global fallback_altitude
+
+    if fallback_altitude is None:
+        (fallback_altitude, status) = get_gmaps_altitude(loc[0], loc[1],
+                                                         args.gmaps_key)
+
+    return fallback_altitude
+
+
+# Get altitude from the db or try to fetch from gmaps api,
+# otherwise, default altitude
+def cached_get_altitude(args, loc):
+    altitude = LocationAltitude.get_nearby_altitude(loc)
+
+    if altitude is None:
+        (altitude, status) = get_gmaps_altitude(loc[0], loc[1], args.gmaps_key)
+        if altitude is not None:
+            LocationAltitude.save_altitude(loc, altitude)
+
+    return altitude
+
+
+# Get altitude main method
+def get_altitude(args, loc):
+    if args.no_altitude_cache:
+        altitude = get_fallback_altitude(args, loc)
+    else:
+        altitude = cached_get_altitude(args, loc)
+
+    if altitude is None:
+        altitude = args.altitude
+
+    return randomize_altitude(altitude, args.altitude_variance)

pogom/models.py

@@ -36,7 +36,7 @@
 flaskDb = FlaskDB()
 cache = TTLCache(maxsize=100, ttl=60 * 5)
 
-db_schema_version = 12
+db_schema_version = 13
 
 
 class MyRetryDB(RetryOperationalError, PooledMySQLDatabase):
@@ -690,6 +690,51 @@ def get_gym(id):
         return result
 
 
+class LocationAltitude(BaseModel):
+    cellid = CharField(primary_key=True, max_length=50)
+    latitude = DoubleField()
+    longitude = DoubleField()
+    last_modified = DateTimeField(index=True, default=datetime.utcnow,
+                                  null=True)
+    altitude = DoubleField()
+
+    class Meta:
+        indexes = ((('latitude', 'longitude'), False),)
+
+    # DB format of a new location altitude
+    @staticmethod
+    def new_loc(loc, altitude):
+        return {'cellid': cellid(loc),
+                'latitude': loc[0],
+                'longitude': loc[1],
+                'altitude': altitude}
+
+    # find a nearby altitude from the db
+    # looking for one within 140m
+    @classmethod
+    def get_nearby_altitude(cls, loc):
+        n, e, s, w = hex_bounds(loc, radius=0.14)  # 140m
+
+        # Get all location altitudes in that box.
+        query = (cls
+                 .select()
+                 .where((cls.latitude <= n) &
+                        (cls.latitude >= s) &
+                        (cls.longitude >= w) &
+                        (cls.longitude <= e))
+                 .dicts())
+
+        altitude = None
+        if len(list(query)):
+            altitude = query[0]['altitude']
+
+        return altitude
+
+    @classmethod
+    def save_altitude(cls, loc, altitude):
+        InsertQuery(cls, rows=[cls.new_loc(loc, altitude)]).upsert().execute()
+
+
 class ScannedLocation(BaseModel):
     cellid = CharField(primary_key=True, max_length=50)
     latitude = DoubleField()
@@ -2231,8 +2276,8 @@ def create_tables(db):
     verify_database_schema(db)
     db.create_tables([Pokemon, Pokestop, Gym, ScannedLocation, GymDetails,
                       GymMember, GymPokemon, Trainer, MainWorker, WorkerStatus,
-                      SpawnPoint, ScanSpawnPoint, SpawnpointDetectionData],
-                     safe=True)
+                      SpawnPoint, ScanSpawnPoint, SpawnpointDetectionData,
+                      LocationAltitude], safe=True)
     db.close()
 
 
@@ -2241,7 +2286,8 @@ def drop_tables(db):
     db.drop_tables([Pokemon, Pokestop, Gym, ScannedLocation, Versions,
                     GymDetails, GymMember, GymPokemon, Trainer, MainWorker,
                     WorkerStatus, SpawnPoint, ScanSpawnPoint,
-                    SpawnpointDetectionData, Versions], safe=True)
+                    SpawnpointDetectionData, LocationAltitude, Versions],
+                   safe=True)
     db.close()
 
 

pogom/schedulers.py

@@ -58,9 +58,10 @@
 from operator import itemgetter
 from datetime import datetime, timedelta
 from .transform import get_new_coords
-from .models import (hex_bounds, Pokemon, SpawnPoint,
-                     ScannedLocation, ScanSpawnPoint)
+from .models import (hex_bounds, Pokemon, SpawnPoint, ScannedLocation,
+                     ScanSpawnPoint)
 from .utils import now, cur_sec, cellid, date_secs, equi_rect_distance
+from .altitude import get_altitude
 
 log = logging.getLogger(__name__)
 
@@ -130,8 +131,8 @@ def next_item(self, search_items_queue):
                 step_location[0], step_location[1], remain),
             'late': 'Too late for location {:6f},{:6f}; skipping.'.format(
                 step_location[0], step_location[1]),
-            'search': 'Searching at {:6f},{:6f}.'.format(
-                step_location[0], step_location[1]),
+            'search': 'Searching at {:6f},{:6f},{:6f}.'.format(
+                step_location[0], step_location[1], step_location[2]),
             'invalid': ('Invalid response at {:6f},{:6f}, ' +
                         'abandoning location.').format(step_location[0],
                                                        step_location[1])
@@ -170,7 +171,6 @@ def __init__(self, queues, status, args):
             self.step_distance = 0.070
 
         self.step_limit = args.step_limit
-
         # This will hold the list of locations to scan so it can be reused,
         # instead of recalculating on each loop.
         self.locations = False
@@ -276,7 +276,9 @@ def _generate_locations(self):
         # Add the required appear and disappear times.
         locationsZeroed = []
         for step, location in enumerate(results, 1):
-            locationsZeroed.append((step, (location[0], location[1], 0), 0, 0))
+            altitude = get_altitude(self.args, location)
+            locationsZeroed.append(
+                (step, (location[0], location[1], altitude), 0, 0))
         return locationsZeroed
 
     # Schedule the work to be done.
@@ -432,10 +434,10 @@ def _generate_locations(self):
         # locations = [((lat, lng, alt), ts_appears, ts_leaves),...]
         retset = []
         for step, location in enumerate(self.locations, 1):
-            retset.append((step,
-                           (location['lat'], location['lng'], 40.32),
-                           location['appears'],
-                           location['leaves']))
+            altitude = get_altitude(self.args, [location['lat'],
+                                    location['lng']])
+            retset.append((step, (location['lat'], location['lng'], altitude),
+                           location['appears'], location['leaves']))
 
         return retset
 
@@ -598,8 +600,12 @@ def _generate_locations(self):
                 loc = get_new_coords(loc, xdist, WEST)
                 results.append((loc[0], loc[1], 0))
 
-        return [(step, (location[0], location[1], 0), 0, 0)
-                for step, location in enumerate(results)]
+        generated_locations = []
+        for step, location in enumerate(results):
+            altitude = get_altitude(self.args, location)
+            generated_locations.append(
+                (step, (location[0], location[1], altitude), 0, 0))
+        return generated_locations
 
     def getsize(self):
         return len(self.queues[0])

pogom/utils.py

@@ -98,6 +98,19 @@ def get_args():
                         type=int, default=1)
     parser.add_argument('-l', '--location', type=parse_unicode,
                         help='Location, can be an address or coordinates.')
+    # Default based on the average elevation of cities around the world.
+    # Source: https://www.wikiwand.com/en/List_of_cities_by_elevation
+    parser.add_argument('-alt', '--altitude',
+                        help='Default altitude in meters.',
+                        type=int, default=507)
+    parser.add_argument('-altv', '--altitude-variance',
+                        help='Variance for --altitude in meters',
+                        type=int, default=1)
+    parser.add_argument('-nac', '--no-altitude-cache',
+                        help=('Do not cache fetched altitudes in the' +
+                              'database. This implies fetching the altitude ' +
+                              'only once for the running instance.'),
+                        action='store_true', default=False)
     parser.add_argument('-nj', '--no-jitter',
                         help=("Don't apply random -9m to +9m jitter to " +
                               "location."),

runserver.py

@@ -7,7 +7,6 @@
 import logging
 import time
 import re
-import requests
 import ssl
 import json
 
@@ -22,6 +21,7 @@
 from pogom import config
 from pogom.app import Pogom
 from pogom.utils import get_args, now
+from pogom.altitude import get_gmaps_altitude
 
 from pogom.search import search_overseer_thread
 from pogom.models import (init_database, create_tables, drop_tables,
@@ -196,15 +196,23 @@ def main():
     if position is None or not any(position):
         log.error("Location not found: '{}'".format(args.location))
         sys.exit()
+
     # Use the latitude and longitude to get the local altitude from Google.
-    try:
-        url = ('https://maps.googleapis.com/maps/api/elevation/json?' +
-               'locations={},{}').format(str(position[0]), str(position[1]))
-        altitude = requests.get(url).json()[u'results'][0][u'elevation']
+    (altitude, status) = get_gmaps_altitude(position[0], position[1],
+                                            args.gmaps_key)
+    if altitude is not None:
         log.debug('Local altitude is: %sm', altitude)
         position = (position[0], position[1], altitude)
-    except (requests.exceptions.RequestException, IndexError, KeyError):
-        log.error('Unable to retrieve altitude from Google APIs; setting to 0')
+    else:
+        if status == 'REQUEST_DENIED':
+            log.error(
+                'Google API Elevation request was denied. You probably ' +
+                'forgot to enable elevation api in https://console.' +
+                'developers.google.com/apis/api/elevation_backend/')
+            sys.exit()
+        else:
+            log.error('Unable to retrieve altitude from Google APIs' +
+                      'setting to 0')
 
     log.info('Parsed location is: %.4f/%.4f/%.4f (lat/lng/alt)',
              position[0], position[1], position[2])