Merge pull request #86 from HomebyTwo/improvement/hb2-85

HB2-85 Improve elevation data on Strava routes

CHANGELOG.md

@@ -1,4 +1,7 @@
 # Change Log
+## [0.12.1] - 2020-10-23 Remove extreme gradient values from imported routes
+- Routes from Strava now have a much more realistic schedule
+
 ## [0.12.0] - 2020-10-15 Improve route update
 - Import page can update routes
 - Update forms tries to retain existing checkpoints

config/settings/base.py

@@ -264,11 +264,3 @@
 GARMIN_CONNECT_USERNAME = get_env_variable("GARMIN_CONNECT_USERNAME", "")
 GARMIN_CONNECT_PASSWORD = get_env_variable("GARMIN_CONNECT_PASSWORD", "")
 GARMIN_ACTIVITY_URL = get_env_variable("GARMIN_ACTIVITY_URL", "")
-
-##############################
-# Swiss public transport API #
-##############################
-
-# http://transport.opendata.ch/docs.html
-
-SWISS_PUBLIC_TRANSPORT_API_URL = get_env_variable("SWISS_PUBLIC_TRANSPORT_API_URL", "")

homebytwo/conftest.py

@@ -1,21 +1,26 @@
+import json
 from functools import partial
 from pathlib import Path
 
 from django.core.files.uploadedfile import SimpleUploadedFile
+from django.shortcuts import resolve_url
 
 import responses
 from pytest import fixture
 from requests.exceptions import ConnectionError
 
+from .importers.models.switzerlandmobilityroute import parse_route_data
 from .utils.factories import AthleteFactory
 from .utils.tests import open_data
 
-METHODS = {
+HTTP_METHODS = {
     "get": responses.GET,
     "post": responses.POST,
     "delete": responses.DELETE,
 }
 
+STRAVA_API_BASE_URL = "https://www.strava.com/api/v3/"
+
 
 @fixture(autouse=True)
 def media_storage(settings, tmpdir):
@@ -29,6 +34,23 @@ def athlete(db, client):
     return athlete
 
 
+@fixture
+def celery(settings):
+    settings.celery_task_always_eager = True
+    settings.celery_task_eager_propagates = True
+
+
+@fixture
+def coda(settings):
+    settings.CODA_API_KEY = "coda_key"
+    settings.CODA_DOC_ID = "doc_id"
+    settings.CODA_TABLE_ID = "grid-table_id"
+    api_url = "https://coda.io/apis/v1"
+    doc_url = api_url + f"/docs/{settings.CODA_DOC_ID}/"
+    table_url = doc_url + f"tables/{settings.CODA_TABLE_ID}"
+    yield {"doc_url": doc_url, "table_url": table_url}
+
+
 @fixture
 def data_dir_path(request):
     return Path(request.module.__file__).parent.resolve()
@@ -51,20 +73,30 @@ def _read_file(file, binary=False):
 
 
 @fixture
-def celery(settings):
-    settings.celery_task_always_eager = True
-    settings.celery_task_eager_propagates = True
+def read_json_file(read_file):
+    def _read_json_file(file):
+        return json.loads(read_file(file))
+
+    return _read_json_file
 
 
 @fixture
-def coda(settings):
-    settings.CODA_API_KEY = "coda_key"
-    settings.CODA_DOC_ID = "doc_id"
-    settings.CODA_TABLE_ID = "grid-table_id"
-    api_url = "https://coda.io/apis/v1"
-    doc_url = api_url + f"/docs/{settings.CODA_DOC_ID}/"
-    table_url = doc_url + f"tables/{settings.CODA_TABLE_ID}"
-    yield {"doc_url": doc_url, "table_url": table_url}
+def uploaded_file(read_file):
+    def _uploaded_file(file):
+        return SimpleUploadedFile(
+            name=file,
+            content=read_file(file, binary=True),
+        )
+
+    return _uploaded_file
+
+
+@fixture
+def switzerland_mobility_data_from_json(read_json_file):
+    def _parse_data(file):
+        return parse_route_data(read_json_file(file))
+
+    return _parse_data
 
 
 @fixture
@@ -73,20 +105,139 @@ def mocked_responses():
         yield response
 
 
+@fixture
+def mock_json_response(read_file, mocked_responses):
+    def _mock_json_response(
+        url,
+        response_json,
+        method="get",
+        status=200,
+    ):
+        mocked_responses.add(
+            HTTP_METHODS.get(method),
+            url=url,
+            content_type="application/json",
+            body=read_file(response_json),
+            status=status,
+        )
+
+    return _mock_json_response
+
+
+@fixture
+def mock_strava_streams_response(settings, mock_json_response):
+    settings.STRAVA_ROUTE_URL = (
+        settings.STRAVA_ROUTE_URL or "https://strava.org/route/%d"
+    )
+
+    def _mock_strava_streams_response(
+        source_id,
+        streams_json="strava_streams_run.json",
+        api_streams_status=200,
+    ):
+        mock_json_response(
+            STRAVA_API_BASE_URL + "routes/%d/streams" % source_id,
+            response_json=streams_json,
+            status=api_streams_status,
+        )
+
+    return _mock_strava_streams_response
+
+
+@fixture
+def mock_route_details_responses(
+    settings, mock_json_response, mock_strava_streams_response
+):
+    settings.SWITZERLAND_MOBILITY_ROUTE_DATA_URL = (
+        settings.SWITZERLAND_MOBILITY_ROUTE_DATA_URL
+        or "https://switzerland_mobility.org/route/%d/data"
+    )
+    settings.SWITZERLAND_MOBILITY_ROUTE_URL = (
+        settings.SWITZERLAND_MOBILITY_ROUTE_URL
+        or "https://switzerland_mobility.org/route/%d"
+    )
+
+    def _mock_route_details_responses(
+        data_source,
+        source_ids,
+        api_response_json=None,
+        api_response_status=200,
+        api_streams_json="strava_streams_run.json",
+        api_streams_status=200,
+    ):
+
+        # intercept the API call
+        api_request_url = {
+            "strava": STRAVA_API_BASE_URL + "routes/%d",
+            "switzerland_mobility": settings.SWITZERLAND_MOBILITY_ROUTE_DATA_URL,
+        }
+        default_api_response_json = {
+            "strava": "strava_route_run.json",
+            "switzerland_mobility": "switzerland_mobility_route.json",
+        }
+
+        api_response_file = api_response_json or default_api_response_json[data_source]
+
+        for source_id in source_ids:
+            mock_json_response(
+                url=api_request_url[data_source] % source_id,
+                response_json=api_response_file,
+                status=api_response_status,
+            )
+
+            if data_source == "strava":
+                mock_strava_streams_response(
+                    source_id, api_streams_json, api_streams_status
+                )
+
+    return _mock_route_details_responses
+
+
+@fixture
+def mock_route_details_response(mock_route_details_responses):
+    def _mock_route_details_response(data_source, source_id, *args, **kwargs):
+        source_ids = [source_id]
+        return mock_route_details_responses(data_source, source_ids, *args, **kwargs)
+
+    return _mock_route_details_response
+
+
+@fixture
+def mock_routes_response(settings, mock_json_response):
+    def _mock_routes_response(athlete, data_source, response_json=None, status=200):
+        response_jsons = {
+            "strava": "strava_route_list.json",
+            "switzerland_mobility": "tracks_list.json",
+        }
+        routes_urls = {
+            "strava": (STRAVA_API_BASE_URL + "athletes/%s/routes" % athlete.strava_id),
+            "switzerland_mobility": settings.SWITZERLAND_MOBILITY_LIST_URL
+            or "https://switzerland_mobility.org/tracks",
+        }
+        mock_json_response(
+            url=routes_urls[data_source],
+            response_json=response_json or response_jsons[data_source],
+            method="get",
+            status=status,
+        )
+
+    return _mock_routes_response
+
+
 @fixture
 def mock_call_response(mocked_responses):
     def _mock_call_response(
         call,
         url,
+        body,
         method="get",
-        body=None,
         content_type="application/json",
         status=200,
         *args,
         **kwargs,
     ):
         mocked_responses.add(
-            method=METHODS[method],
+            method=HTTP_METHODS[method],
             url=url,
             body=body,
             content_type=content_type,
@@ -97,17 +248,6 @@ def _mock_call_response(
     return _mock_call_response
 
 
-@fixture
-def uploaded_file(read_file):
-    def _uploaded_file(file):
-        return SimpleUploadedFile(
-            name=file,
-            content=read_file(file, binary=True),
-        )
-
-    return _uploaded_file
-
-
 @fixture
 def mock_call_json_response(read_file, mock_call_response):
     def _mock_call_json_response(
@@ -131,7 +271,7 @@ def mock_call_json_responses(read_file, mocked_responses):
     def _mock_call_json_responses(call, response_mocks, *args, **kwargs):
         for response in response_mocks:
             mocked_responses.add(
-                METHODS.get(response.get("method")) or responses.GET,
+                HTTP_METHODS.get(response.get("method")) or responses.GET,
                 response["url"],
                 body=read_file(response["response_json"]),
                 status=response.get("status") or 200,
@@ -143,15 +283,42 @@ def _mock_call_json_responses(call, response_mocks, *args, **kwargs):
 
 
 @fixture
-def connection_error(mock_call_response):
+def mock_import_route_call_response(client, mock_route_details_response):
+    def _mock_import_route_call_response(
+        data_source,
+        source_id,
+        method="get",
+        post_data=None,
+        follow_redirect=False,
+        **kwargs,
+    ):
+
+        mock_route_details_response(
+            data_source,
+            source_id,
+            **kwargs,
+        )
+
+        # call import_route url
+        url = resolve_url("import_route", data_source=data_source, source_id=source_id)
+        if method == "get":
+            return client.get(url, follow=follow_redirect)
+        if method == "post":
+            return client.post(url, post_data, follow=follow_redirect)
+
+    return _mock_import_route_call_response
+
+
+@fixture
+def mock_connection_error(mock_call_response):
     return partial(mock_call_response, body=ConnectionError("Connection error."))
 
 
 @fixture
-def server_error(mock_call_json_response):
+def mock_server_error(mock_call_json_response):
     return partial(mock_call_json_response, status=500)
 
 
 @fixture
-def not_found(mock_call_json_response):
+def mock_not_found_error(mock_call_json_response):
     return partial(mock_call_json_response, status=404)

homebytwo/importers/decorators.py

@@ -83,7 +83,7 @@ def _wrapped_view(request, *args, **kwargs):
             if match.url_name == "import_route":
                 route_id = match.kwargs["source_id"]
                 params = urlencode({"route_id": route_id})
-                return redirect(f'{login_url}?{params}')
+                return redirect(f"{login_url}?{params}")
             else:
                 return redirect(login_url)
 

homebytwo/importers/models/stravaroute.py

@@ -100,9 +100,11 @@ def get_route_details(self, cookies=None):
 
     def get_route_data(self, cookies=None):
         """
-        convert raw streams into the route geom and a pandas DataFrame
+        convert raw streams into the route LineString and a pandas DataFrame
         with columns for distance and altitude.
 
+        :param cookies: switzerland mobility cookies from the athlete session.
+
         the stravalib client creates a list of dicts:
         `[stream_type: <Stream object>, stream_type: <Stream object>, ...]`
         """

homebytwo/importers/models/switzerlandmobilityroute.py

@@ -1,4 +1,4 @@
-from ast import literal_eval
+import json
 
 from django.conf import settings
 from django.contrib.gis.geos import LineString
@@ -57,6 +57,22 @@ def get_remote_routes_list(athlete, cookies=None):
             return []
 
 
+def parse_route_data(raw_route_json):
+    data = DataFrame(
+        json.loads(raw_route_json["properties"]["profile"]),
+        columns=["lat", "lng", "altitude", "distance"],
+    )
+
+    # create geom from lat, lng data columns
+    coords = list(zip(data["lat"], data["lng"]))
+    geom = LineString(coords, srid=21781)
+
+    # remove redundant lat, lng columns in data
+    data.drop(columns=["lat", "lng"], inplace=True)
+
+    return geom, data
+
+
 class SwitzerlandMobilityRoute(Route):
 
     """
@@ -126,16 +142,4 @@ def get_route_data(self, cookies=None, raw_route_json=None):
             raw_route_json = request_json(self.route_data_url, cookies)
 
         if raw_route_json:
-            data = DataFrame(
-                literal_eval(raw_route_json["properties"]["profile"]),
-                columns=["lat", "lng", "altitude", "distance"],
-            )
-
-            # create geom from lat, lng data columns
-            coords = list(zip(data["lat"], data["lng"]))
-            geom = LineString(coords, srid=21781)
-
-            # remove redundant lat, lng columns in data
-            data.drop(columns=["lat", "lng"], inplace=True)
-
-            return geom, data
+            return parse_route_data(raw_route_json)