Merge pull request #640 from communitiesuk/FS-4004-optimise-fund-dash…

…board

FS4004 Optimise fund dashboard endpoint

app/blueprints/assessments/helpers.py

@@ -16,7 +16,6 @@
 from app.blueprints.assessments.models.common import OptionGroup
 from app.blueprints.services.aws import generate_url
 from app.blueprints.services.aws import list_files_by_prefix
-from app.blueprints.services.data_services import get_tag_types
 from app.blueprints.services.models.flag import FlagType
 from app.blueprints.services.models.fund import Fund
 from app.blueprints.shared.filters import utc_to_bst
@@ -99,8 +98,7 @@ def set_application_status_in_overview(application_overviews):
     return application_overviews
 
 
-def get_tag_map_and_tag_options(fund_round_tags, post_processed_overviews):
-    tag_types = get_tag_types()
+def get_tag_map_and_tag_options(tag_types, fund_round_tags, post_processed_overviews):
     tag_option_groups = [
         OptionGroup(
             label=", ".join(p.capitalize() for p in purposes),
@@ -166,7 +164,9 @@ def generate_csv_of_application(
             if not answers:
                 answers = "Not provided"
 
-            writer.writerow([section_title, questions, format_answer(answers)])
+            writer.writerow(
+                [section_title, questions, format_answer(answer=answers, language="en")]
+            )
     return output.getvalue()
 
 

app/blueprints/assessments/models/full_application.py

@@ -55,7 +55,7 @@ def from_data(cls, args):
                     questions_and_answers=[
                         QuestionAndAnswer(
                             question=i["question"],
-                            answer=format_answer(i.get("answer")),
+                            answer=format_answer(i.get("answer"), language="en"),
                             number="",
                         )
                         for i in args.all_uploaded_documents

app/blueprints/assessments/routes.py

@@ -1,3 +1,5 @@
+import concurrent.futures
+import contextvars
 import io
 import time
 import zipfile
@@ -16,6 +18,7 @@
 from flask import request
 from flask import url_for
 from fsd_utils import extract_questions_and_answers
+from fsd_utils.sqs_scheduler.context_aware_executor import ContextAwareExecutor
 
 from app.blueprints.assessments.activity_trail import AssociatedTags
 from app.blueprints.assessments.activity_trail import CheckboxForm
@@ -91,6 +94,7 @@
 from app.blueprints.services.data_services import get_score_and_justification
 from app.blueprints.services.data_services import get_sub_criteria
 from app.blueprints.services.data_services import get_sub_criteria_theme_answers_all
+from app.blueprints.services.data_services import get_tag_types
 from app.blueprints.services.data_services import get_tags_for_fund_round
 from app.blueprints.services.data_services import match_comment_to_theme
 from app.blueprints.services.data_services import submit_comment
@@ -134,6 +138,21 @@
 )
 
 
+def run_with_context(func, *args, **kwargs):
+    with current_app.app_context():
+        return func(*args, **kwargs)
+
+
+class ContextThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):
+    def __init__(self, *args, **kwargs):
+        self.context = contextvars.copy_context()
+        super().__init__(*args, **kwargs, initializer=self._set_child_context)
+
+    def _set_child_context(self):
+        for var, value in self.context.items():
+            var.set(value)
+
+
 def _handle_all_uploaded_documents(application_id):
     flags_list = get_flags(application_id)
     flag_status = determine_flag_status(flags_list)
@@ -237,16 +256,6 @@ def fund_dashboard(fund_short_name: str, round_short_name: str):
     if has_devolved_authority_validation(fund_id=fund_id):
         countries = get_countries_from_roles(fund.short_name)
 
-    # This call is to get the location data such as country, region and local_authority
-    # from all the existing applications.
-    all_applications_metadata = get_application_overviews(
-        fund_id, round_id, search_params=""
-    )
-    # note, we are not sending search parameters here as we don't want to filter
-    # the stats at all.  see https://dluhcdigital.atlassian.net/browse/FS-3249
-    unfiltered_stats = process_assessments_stats(all_applications_metadata)
-    all_application_locations = LocationData.from_json_blob(all_applications_metadata)
-
     search_params = {
         **search_params,
         "countries": ",".join(countries),
@@ -255,8 +264,43 @@ def fund_dashboard(fund_short_name: str, round_short_name: str):
     # matches the query parameters provided in the search and filter form
     search_params, show_clear_filters = match_search_params(search_params, request.args)
 
-    # request all the application overviews based on the search parameters
-    application_overviews = get_application_overviews(fund_id, round_id, search_params)
+    thread_executor = ContextAwareExecutor(
+        max_workers=4,
+        thread_name_prefix="fund-dashboard-request",
+        flask_app=current_app,
+    )
+
+    # The first call is to get the location data such as country, region and local_authority
+    # from all the existing applications (i.e withou search parameters as we don't want to filter
+    # the stats at all).  see https://dluhcdigital.atlassian.net/browse/FS-3249
+    future_all_applications_metadata = thread_executor.submit(
+        get_application_overviews, fund_id, round_id, ""
+    )
+
+    # The second call is with the search parameters
+    future_application_overviews = thread_executor.submit(
+        get_application_overviews,
+        fund_id,
+        round_id,
+        search_params,
+    )
+
+    future_active_fund_round_tags = thread_executor.submit(
+        get_tags_for_fund_round,
+        fund_id,
+        round_id,
+        {"tag_status": "True"},
+    )
+
+    future_tag_types = thread_executor.submit(get_tag_types)
+
+    all_applications_metadata = future_all_applications_metadata.result()
+    application_overviews = future_application_overviews.result()
+    active_fund_round_tags = future_active_fund_round_tags.result()
+    tag_types = future_tag_types.result()
+
+    unfiltered_stats = process_assessments_stats(all_applications_metadata)
+    all_application_locations = LocationData.from_json_blob(all_applications_metadata)
 
     teams_flag_stats = get_team_flag_stats(application_overviews)
 
@@ -285,11 +329,8 @@ def fund_dashboard(fund_short_name: str, round_short_name: str):
         post_processed_overviews
     )
 
-    active_fund_round_tags = get_tags_for_fund_round(
-        fund_id, round_id, {"tag_status": "True"}
-    )
     tags_in_application_map, tag_option_groups = get_tag_map_and_tag_options(
-        active_fund_round_tags, post_processed_overviews
+        tag_types, active_fund_round_tags, post_processed_overviews
     )
 
     def get_sorted_application_overviews(application_overviews, column, reverse=False):

requirements-dev.txt

@@ -1,8 +1,8 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile requirements-dev.in
+#    pip-compile --output-file=requirements-dev.txt requirements-dev.in
 #
 alembic==1.10.3
     # via
@@ -124,6 +124,8 @@ docopt==0.6.2
     #   num2words
 email-validator==2.1.1
     # via -r requirements.txt
+exceptiongroup==1.2.1
+    # via pytest
 filelock==3.9.0
     # via virtualenv
 flake8==7.0.0
@@ -173,14 +175,10 @@ flipper-client==1.3.2
     # via
     #   -r requirements.txt
     #   funding-service-design-utils
-funding-service-design-utils==2.0.32
+funding-service-design-utils==2.0.51
     # via -r requirements.txt
 govuk-frontend-jinja==2.8.0
     # via -r requirements.txt
-greenlet==3.0.0
-    # via
-    #   -r requirements.txt
-    #   sqlalchemy
 gunicorn==20.1.0
     # via
     #   -r requirements.txt
@@ -485,10 +483,18 @@ tinycss2==1.2.1
     #   -r requirements.txt
     #   cssselect2
     #   svglib
+tomli==2.0.1
+    # via
+    #   black
+    #   build
+    #   flake8-pyproject
+    #   pytest
+    #   pytest-env
 typing-extensions==4.5.0
     # via
     #   -r requirements.txt
     #   alembic
+    #   black
     #   qrcode
     #   sqlalchemy
 tzlocal==5.0.1

requirements.in

@@ -1,7 +1,7 @@
 #-----------------------------------
 #   FSD Utils
 #-----------------------------------
-funding-service-design-utils>=2.0.27,<2.1.0
+funding-service-design-utils>=2.0.51,<2.1.0
 
 
 requests

requirements.txt

@@ -1,8 +1,8 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile requirements.in
+#    pip-compile --output-file=requirements.txt requirements.in
 #
 alembic==1.10.3
     # via flask-migrate
@@ -111,12 +111,10 @@ flask-wtf==1.1.1
     # via -r requirements.in
 flipper-client==1.3.2
     # via funding-service-design-utils
-funding-service-design-utils==2.0.32
+funding-service-design-utils==2.0.51
     # via -r requirements.in
 govuk-frontend-jinja==2.8.0
     # via -r requirements.in
-greenlet==3.0.0
-    # via sqlalchemy
 gunicorn==20.1.0
     # via funding-service-design-utils
 html5lib==1.1

tests/conftest.py

@@ -832,7 +832,7 @@ def mock_get_tag_for_fund_round(mocker):
 def mock_get_tag_types(mocker):
     for function_module_path in [
         "app.blueprints.tagging.routes.get_tag_types",
-        "app.blueprints.assessments.helpers.get_tag_types",
+        "app.blueprints.services.data_services.get_tag_types",
     ]:
         mocker.patch(
             function_module_path,