Merge remote-tracking branch 'origin/master' into improvement/hb2-79

# Conflicts:
#	CHANGELOG.md
#	homebytwo/importers/views.py
#	homebytwo/routes/views.py
#	homebytwo/templates/routes/_route_card.html

.editorconfig

@@ -12,5 +12,8 @@ max_line_length = 120
 [*.{py,php,md}]
 indent_size = 4
 
+[*.py]
+max_line_length = 88
+
 [*.md]
 trim_trailing_whitespace = false

CHANGELOG.md

@@ -1,9 +1,15 @@
 # Change Log
-## [0.11.0] - 2020-10-15 Update routes from the import app
-- Update from the import form is now allowed
+## [0.12.0] - 2020-10-15 Improve route update
+- Import page can update routes
 - Update forms tries to retain existing checkpoints
-- Improved route page structure and design
-- Enforced Strava brand guidelines
+- Redirect after Switzerland Mobility login
+- Improved route page structure and design and enforced Strava guidelines
+
+## [0.11.0] - 2020-10-07 Import GPX files
+- Create new routes from GPX files 
+## [0.10.4] - 2020-10-13 Slightly improve admin security
+- Move admin to another domain
+- Adopt increased security settings 
 
 ## [0.10.3] - 2020-10-07 Report usage data to coda.io
 - Report athlete, routes and activities count on a daily basis 

config/settings/prod.py

@@ -25,6 +25,13 @@
 SESSION_COOKIE_SECURE = True
 CSRF_COOKIE_SECURE = True
 
+#####################
+# Improved Security #
+#####################
+
+X_FRAME_OPTIONS = "DENY"
+SECURE_CONTENT_TYPE_NOSNIFF = True
+
 #########################
 # File upload permission#
 #########################
@@ -45,7 +52,7 @@
 # Force Celery Broker URL #
 ###########################
 
-CELERY_BROKER_URL = get_env_variable("CELERY_BROKER_URL")
+celery_broker_url = get_env_variable("CELERY_BROKER_URL")
 
 ###########################
 # Force Mailchimp API Key #

config/urls.py

@@ -17,7 +17,7 @@
     # import/
     path("import/", include("homebytwo.importers.urls")),
     # admin/
-    path("admin/", admin.site.urls),
+    path("control/", admin.site.urls),
     path("", include("django.contrib.auth.urls")),
 ]
 

fabfile.py

@@ -5,25 +5,13 @@
 from pathlib import Path
 
 import dj_database_url
-from config import get_project_root_path
-from fabric.api import (
-    cd,
-    env,
-    execute,
-    get,
-    local,
-    put,
-    require,
-    run,
-    settings,
-    shell_env,
-    sudo,
-    task,
-)
+from fabric.api import cd, env, execute, get, local, put, require, run, settings, shell_env, sudo, task
 from fabric.context_managers import quiet
 from fabric.operations import prompt
 from gitric import api as gitric
 
+from config import get_project_root_path
+
 # This is the definition of your environments. Every item of the ENVIRONMENTS
 # dict will be made available as a fabric task and the properties you put in a
 # particular environment will be made available in the `env` variable.
@@ -154,6 +142,14 @@ def restart_processes():
         sudo("/bin/systemctl restart {}.service".format(service), shell=False)
 
 
+def check_deployment_settings():
+    """
+    Check the entire Django project for potential problems with deployment settings
+    """
+    with cd(get_project_root()):
+        run_python("manage.py check --deploy")
+
+
 def generate_secret_key():
     """
     Generate a random secret key, suitable to be used as a SECRET_KEY setting.
@@ -260,9 +256,9 @@ def deploy(tag):
     execute(git_push, commit="@")
     dump_db(get_backups_root())
     execute(install_requirements)
+    execute(check_deployment_settings)
     execute(collect_static)
     execute(migrate_database)
-
     execute(restart_processes)
     execute(clean_old_database_backups, nb_backups_to_keep=10)
 

homebytwo/conftest.py

@@ -1,6 +1,8 @@
 from functools import partial
 from pathlib import Path
 
+from django.core.files.uploadedfile import SimpleUploadedFile
+
 import responses
 from pytest import fixture
 from requests.exceptions import ConnectionError
@@ -95,12 +97,22 @@ def _mock_call_response(
     return _mock_call_response
 
 
+@fixture
+def uploaded_file(read_file):
+    def _uploaded_file(file):
+        return SimpleUploadedFile(
+            name=file,
+            content=read_file(file, binary=True),
+        )
+
+    return _uploaded_file
+
+
 @fixture
 def mock_call_json_response(read_file, mock_call_response):
     def _mock_call_json_response(
         call, url, response_json, method="get", status=200, *args, **kwargs
     ):
-
         return mock_call_response(
             call,
             url,

homebytwo/importers/forms.py

@@ -3,9 +3,15 @@
 from django import forms
 from django.conf import settings
 from django.contrib import messages
+from django.contrib.gis.geos import LineString, Point
 
+import gpxpy
+from pandas import DataFrame
 from requests import Session, codes
 
+from homebytwo.routes.models import Route
+from homebytwo.routes.utils import get_distances
+
 
 class SwitzerlandMobilityLogin(forms.Form):
     """
@@ -86,3 +92,68 @@ def retrieve_authorization_cookie(self, request):
                 )
                 messages.error(request, message)
                 return False
+
+
+class GpxUploadForm(forms.Form):
+    """
+    Athletes create a new route uploading a GPS exchange format file.
+    """
+
+    gpx = forms.FileField()
+
+    def clean_gpx(self):
+        gpx_file = self.cleaned_data["gpx"]
+        try:
+            gpx = gpxpy.parse(gpx_file)
+        except (
+            gpxpy.gpx.GPXXMLSyntaxException,
+            gpxpy.gpx.GPXException,
+            ValueError,  # namespace declaration error in Swisstopo app exports
+        ) as error:
+            raise forms.ValidationError(
+                "Your file does not appear to be a valid GPX file"
+                f'(error was: "{str(error)}") '
+            )
+
+        # check that we can create a lineString from the file
+        if len(list(gpx.walk(only_points=True))) > 1:
+            return gpx
+        else:
+            raise forms.ValidationError("Your file does not contain a valid route.")
+
+    def save(self, commit=True):
+        gpx = self.cleaned_data["gpx"]
+        route = Route(source_id=None)
+
+        # use the `name` in the GPX file as proposition for the route name
+        route.name = gpx.name if gpx.name else ""
+
+        # assume we want to use all points of all tracks in the GPX file
+        points = list(gpx.walk(only_points=True))
+
+        # create route geometry from coords
+        coords = [(point.longitude, point.latitude) for point in points]
+        route.geom = LineString(coords, srid=4326).transform(21781, clone=True)
+
+        # calculate total distance and elevation differences
+        route.total_distance = gpx.length_2d()
+        (
+            route.total_elevation_gain,
+            route.total_elevation_loss,
+        ) = gpx.get_uphill_downhill()
+
+        # create route DataFrame with distance and elevation
+        distances = get_distances([Point(p) for p in route.geom])
+        route_data = [
+            {
+                "distance": distance,
+                "altitude": point.elevation,
+            }
+            for point, distance in zip(points, distances)
+        ]
+        route.data = DataFrame(route_data, columns=["distance", "altitude"])
+
+        if commit:
+            route.save()
+
+        return route

homebytwo/importers/models/stravaroute.py

@@ -115,8 +115,7 @@ def get_route_data(self, cookies=None):
         for key, stream in route_streams.items():
             # create route geom from latlng stream
             if key == "latlng":
-                coords = [(lng, lat) for lat, lng in stream.data]
-                geom = LineString(coords, srid=4326).transform(21781, clone=True)
+                geom = LineString(stream.data, srid=4326).transform(21781, clone=True)
 
             # import other streams
             else:

homebytwo/importers/models/switzerlandmobilityroute.py

@@ -132,7 +132,7 @@ def get_route_data(self, cookies=None, raw_route_json=None):
             )
 
             # create geom from lat, lng data columns
-            coords = [(lat, lng) for lat, lng in zip(data["lat"], data["lng"])]
+            coords = list(zip(data["lat"], data["lng"]))
             geom = LineString(coords, srid=21781)
 
             # remove redundant lat, lng columns in data

homebytwo/importers/tests/data/bad_schemaLocation.gpx

@@ -0,0 +1,6 @@
+<?xml version="1.0"?>
+<gpx version="1.1" creator="Generated by Swisstopo App. See https://www.swisstopo.ch/app for more information."
+     xmlns="http://www.topografix.com/GPX/1/1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+     xmlns:swisstopo="https://prod-static.swisstopo-app.ch/xmlschemas/SwisstopoExtensions"
+     xmlns:schemaLocation="http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd https://swisstopo-app.ch/xmlschemas/SwisstopoExtensions https://prod-static.swisstopo-app.ch/xmlschemas/SwisstopoExtensions.xsd">
+</gpx>

homebytwo/importers/tests/data/empty.gpx

@@ -0,0 +1,6 @@
+<?xml version="1.0"?>
+<gpx version="1.1" creator="Generated by Swisstopo App. See https://www.swisstopo.ch/app for more information."
+     xmlns="http://www.topografix.com/GPX/1/1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+     xmlns:swisstopo="https://prod-static.swisstopo-app.ch/xmlschemas/SwisstopoExtensions"
+     xsi:schemaLocation="http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd https://swisstopo-app.ch/xmlschemas/SwisstopoExtensions https://prod-static.swisstopo-app.ch/xmlschemas/SwisstopoExtensions.xsd">
+</gpx>