Merge pull request #26 from communitiesuk/fs-1116-update-logging

FS-1116 update logging

.github/workflows/deploy.yml

@@ -23,8 +23,8 @@ jobs:
         with:
           python-version: 3.10.1
       - name: install dependencies
-        run: python -m pip install --upgrade pip && python -m pip install -r requirements.txt
-      - name: build swagger
+        run: python -m pip install --upgrade pip && python -m pip install -r requirements-dev.txt
+      - name: build govuk assets
         run: python build.py
       - name: run unit tests
         run: python -m pip install pytest && python -m pytest .
@@ -59,7 +59,7 @@ jobs:
         run: python -m venv .venv
       - name: install dependencies
         run: source .venv/bin/activate && python -m pip install --upgrade pip && pip install -r requirements.txt
-      - name: build swagger and govuk assets
+      - name: build govuk assets
         run: source .venv/bin/activate && python build.py
       - name: Deploy to Gov PaaS
         uses: citizen-of-planet-earth/cf-cli-action@v2
@@ -92,7 +92,7 @@ jobs:
         with:
           python-version: 3.10.1
       - name: install dependencies
-        run: python -m pip install --upgrade pip && python -m pip install -r requirements.txt
+        run: python -m pip install --upgrade pip && python -m pip install -r requirements-dev.txt
       - name: Bandit
         run: bandit -r ./auth ./demo
       - name: Safety

README.md

@@ -36,7 +36,15 @@ Built with Flask.
 ### Install dependencies
 From the top-level directory enter the command to install pip and the dependencies of the project
 
-    python3 -m pip install --upgrade pip && pip install -r requirements.txt
+    python3 -m pip install --upgrade pip && pip install -r requirements-dev.txt
+
+NOTE: requirements-dev.txt and requirements.txt are updated using [pip-tools pip-compile](https://github.com/jazzband/pip-tools)
+To update requirements please manually add the dependencies in the .in files (not the requirements.txt files)
+Then run:
+
+    pip-compile requirements.in
+
+    pip-compile requirements-dev.in
 
 ## How to use
 Enter the virtual environment as described above, then:
@@ -76,7 +84,7 @@ First set the FLASK_ENV environment you wish to test eg:
 
 Then run gunicorn using the following command:
 
-    gunicorn wsgi:app -c gunicorn.config.local.py
+    gunicorn wsgi:app -c run/gunicorn/local.py
 
 # Configuration
 
@@ -105,8 +113,6 @@ Performance tests are stored in a separate repository which is then run in the p
 This repo comes with a .pre-commit-config.yaml, if you wish to use this do
 the following while in your virtual enviroment:
 
-    pip install pre-commit black
-
     pre-commit install
 
 Once the above is done you will have autoformatting and pep8 compliance built

app.py

@@ -14,10 +14,10 @@
 from flask_session import Session
 from flask_talisman import Talisman
 from frontend.assets import compile_static_assets
+from fsd_utils.logging import logging
 from jinja2 import ChoiceLoader
 from jinja2 import PackageLoader
 from jinja2 import PrefixLoader
-from utils import logging
 
 redis_mlinks = FlaskRedis(config_prefix="REDIS_MLINKS")
 
@@ -30,21 +30,26 @@ def get_bundled_specs(main_file: Path) -> Dict[str, Any]:
 
 def create_app() -> Flask:
 
-    options = {
+    # Initialise Connexion Flask App
+    connexion_options = {
         "swagger_path": Config.FLASK_ROOT + "/swagger/dist",
         "swagger_url": "/docs",
         "swagger_ui_template_arguments": {},
     }
-
     connexion_app = connexion.FlaskApp(
-        __name__,
+        "Authenticator",
         specification_dir="/openapi/",
-        options=options,
+        options=connexion_options,
+    )
+    connexion_app.add_api(
+        get_bundled_specs(Config.FLASK_ROOT + "/openapi/api.yml"),
+        validate_responses=True,
+        resolver=MethodViewResolver("api"),
     )
 
+    # Configure Flask App
     flask_app = connexion_app.app
     flask_app.config.from_object("config.Config")
-
     flask_app.static_folder = "frontend/static/dist/"
     flask_app.jinja_loader = ChoiceLoader(
         [
@@ -54,31 +59,23 @@ def create_app() -> Flask:
             ),
         ]
     )
-
     flask_app.jinja_env.trim_blocks = True
     flask_app.jinja_env.lstrip_blocks = True
+    flask_app.jinja_env.globals.update(
+        _build_auth_code_flow=build_auth_code_flow
+    )  # Used in template
 
     # Initialise logging
     logging.init_app(flask_app)
 
-    # Configure Swagger
-    options = {
-        "swagger_path": Config.FLASK_ROOT + "/swagger/dist",
-        "swagger_url": "/docs",
-        "swagger_ui_template_arguments": {},
-    }
-
-    connexion_app.add_api(
-        get_bundled_specs(Config.FLASK_ROOT + "/openapi/api.yml"),
-        validate_responses=True,
-        resolver=MethodViewResolver("api"),
-    )
-
+    # Initialise Sessions
     session = Session()
     session.init_app(flask_app)
 
+    # Initialise Redis Magic Links Store
     redis_mlinks.init_app(flask_app)
 
+    # Configure Talisman Security Settings
     talisman = Talisman(
         flask_app,
         strict_transport_security=Config.HSTS_HEADERS,
@@ -96,10 +93,6 @@ def create_app() -> Flask:
 
     flask_app.wsgi_app = ProxyFix(flask_app.wsgi_app, x_proto=1, x_host=1)
 
-    flask_app.jinja_env.globals.update(
-        _build_auth_code_flow=build_auth_code_flow
-    )  # Used in template
-
     # Disable strict talisman on swagger docs pages
     @flask_app.before_request
     def before_request_modifier():

config/__init__.py

@@ -5,25 +5,17 @@
 
 match FLASK_ENV:
     case "development":
-        from config.environments.development import (
-            DevelopmentConfig as Config,
-        )
+        from config.envs.development import DevelopmentConfig as Config
     case "dev":
-        from config.environments.dev import (
-            DevConfig as Config,
-        )
+        from config.envs.dev import DevConfig as Config
     case "test":
-        from config.environments.test import TestConfig as Config
+        from config.envs.test import TestConfig as Config
     case "unit_test":
-        from config.environments.unit_test import (
-            UnitTestConfig as Config,
-        )
+        from config.envs.unit_test import UnitTestConfig as Config
     case "production":
-        from config.environments.production import (
-            ProductionConfig as Config,
-        )
+        from config.envs.production import ProductionConfig as Config
     case _:
-        from config.environments.default import DefaultConfig as Config
+        from config.envs.default import DefaultConfig as Config
 
 try:
     Config.pretty_print()

config/environments/dev.py → config/envs/dev.py

@@ -3,7 +3,7 @@
 from os import environ
 
 import redis
-from config.environments.default import DefaultConfig as Config
+from config.envs.default import DefaultConfig as Config
 from config.utils import VcapServices
 from fsd_utils import configclass
 

config/environments/development.py → config/envs/development.py

@@ -3,7 +3,7 @@
 from os import getenv
 
 import redis
-from config.environments.default import DefaultConfig as Config
+from config.envs.default import DefaultConfig as Config
 from fsd_utils import configclass
 
 
@@ -55,16 +55,20 @@ class DevelopmentConfig(Config):
         RSA256_PUBLIC_KEY = public_key_file.read()
 
     # Redis
-    
+
     REDIS_INSTANCE_URI = getenv("REDIS_INSTANCE_URI", "redis://localhost:6379")
     REDIS_MLINKS_URL = f"{REDIS_INSTANCE_URI}/0"
     REDIS_SESSIONS_URL = f"{REDIS_INSTANCE_URI}/1"
     SESSION_REDIS = redis.from_url(REDIS_SESSIONS_URL)
 
     # APIs
-    APPLICATION_STORE_API_HOST =  getenv("APPLICATION_STORE_API_HOST", "application_store")
+    APPLICATION_STORE_API_HOST = getenv(
+        "APPLICATION_STORE_API_HOST", "application_store"
+    )
     ACCOUNT_STORE_API_HOST = getenv("ACCOUNT_STORE_API_HOST", "account_store")
-    NOTIFICATION_SERVICE_HOST = getenv("NOTIFICATION_SERVICE_HOST", "notification_service")
+    NOTIFICATION_SERVICE_HOST = getenv(
+        "NOTIFICATION_SERVICE_HOST", "notification_service"
+    )
 
     # Security
     FORCE_HTTPS = False

config/environments/production.py → config/envs/production.py

@@ -2,7 +2,7 @@
 from os import environ
 
 import redis
-from config.environments.default import DefaultConfig as Config
+from config.envs.default import DefaultConfig as Config
 from config.utils import VcapServices
 from fsd_utils import configclass
 

config/environments/test.py → config/envs/test.py

@@ -3,7 +3,7 @@
 from os import environ
 
 import redis
-from config.environments.default import DefaultConfig as Config
+from config.envs.default import DefaultConfig as Config
 from config.utils import VcapServices
 from fsd_utils import configclass
 

config/environments/unit_test.py → config/envs/unit_test.py

@@ -2,7 +2,7 @@
 import logging
 
 import redis
-from config.environments.default import DefaultConfig as Config
+from config.envs.default import DefaultConfig as Config
 from fsd_utils import configclass
 
 

manifest-dev.yml

@@ -4,7 +4,7 @@ applications:
   memory: 64M
   buildpacks:
   - https://github.com/cloudfoundry/python-buildpack.git
-  command: gunicorn wsgi:app -c run/gunicorn/config.devtest.py
+  command: gunicorn wsgi:app -c run/gunicorn/devtest.py
   env:
     AUTHENTICATOR_HOST: https://funding-service-design-authenticator-dev.london.cloudapps.digital
     ACCOUNT_STORE_API_HOST: https://funding-service-design-account-store-dev.london.cloudapps.digital

manifest-test.yml

@@ -4,7 +4,7 @@ applications:
   memory: 64M
   buildpacks:
   - https://github.com/cloudfoundry/python-buildpack.git
-  command: gunicorn wsgi:app -c run/gunicorn/config.devtest.py
+  command: gunicorn wsgi:app -c run/gunicorn/devtest.py
   env:
     AUTHENTICATOR_HOST: https://funding-service-design-authenticator-test.london.cloudapps.digital
     ACCOUNT_STORE_API_HOST: https://funding-service-design-account-store-test.london.cloudapps.digital

models/account.py

@@ -2,6 +2,7 @@
 from typing import List
 
 from config import Config
+from flask import current_app
 from models.application import Application
 from models.application import ApplicationMethods
 from models.data import get_data
@@ -130,6 +131,10 @@ def get_magic_link(
                 notification_content,
             )
             return True
+
+        current_app.logger.error(
+            f"Could not create an account ({account}) for email '{email}'"
+        )
         raise AccountError(
             message=(
                 "Sorry, we couldn't create an account for this email, please"

models/magic_link.py

@@ -10,6 +10,7 @@
 from typing import TYPE_CHECKING
 
 from config import Config
+from flask import current_app
 from flask_redis import FlaskRedis
 from security.utils import create_token
 
@@ -176,6 +177,7 @@ def create_magic_link(
         :param redirect_url: (str, optional) An optional redirect_url
         :return:
         """
+        current_app.logger.info(f"Creating magic link for {account}")
         self._clear_existing_user_link(account)
         if not redirect_url:
             redirect_url = self._create_redirect_url(account)
@@ -201,6 +203,10 @@ def create_magic_link(
                     "link": magic_link_url,
                 }
             )
+            current_app.logger.info(f"Magic link created for {account}")
             return new_link_json
 
+        current_app.logger.error(
+            f"Magic link for account {account} could not be created"
+        )
         raise MagicLinkError(message="Could not create a magic link")

requirements-dev.in

@@ -0,0 +1,16 @@
+-r requirements.txt
+bandit
+safety==1.10.3
+black
+deepdiff
+pre-commit
+pytest
+pytest-env
+pytest-flask
+pytest-mock==3.7.0
+pytest-selenium==2.0.1
+pip-tools
+axe-selenium-python==2.1.6
+json2html==1.3.0
+selenium==4.2.0
+webdriver-manager==3.7.0