From 26e600548c62426b054ac3ed4b09a76ae62e7e87 Mon Sep 17 00:00:00 2001 From: Kishan Patel Date: Tue, 11 Jun 2024 11:55:41 +0100 Subject: [PATCH 1/4] Fetch data concurrently in fund_dashboard --- app/blueprints/assessments/helpers.py | 4 +- app/blueprints/assessments/routes.py | 66 ++++++++++++++++++++------- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/app/blueprints/assessments/helpers.py b/app/blueprints/assessments/helpers.py index b4df0616..396d7941 100644 --- a/app/blueprints/assessments/helpers.py +++ b/app/blueprints/assessments/helpers.py @@ -16,7 +16,6 @@ from app.blueprints.assessments.models.common import OptionGroup from app.blueprints.services.aws import generate_url from app.blueprints.services.aws import list_files_by_prefix -from app.blueprints.services.data_services import get_tag_types from app.blueprints.services.models.flag import FlagType from app.blueprints.services.models.fund import Fund from app.blueprints.shared.filters import utc_to_bst @@ -99,8 +98,7 @@ def set_application_status_in_overview(application_overviews): return application_overviews -def get_tag_map_and_tag_options(fund_round_tags, post_processed_overviews): - tag_types = get_tag_types() +def get_tag_map_and_tag_options(tag_types, fund_round_tags, post_processed_overviews): tag_option_groups = [ OptionGroup( label=", ".join(p.capitalize() for p in purposes), diff --git a/app/blueprints/assessments/routes.py b/app/blueprints/assessments/routes.py index 56070ef8..99930583 100644 --- a/app/blueprints/assessments/routes.py +++ b/app/blueprints/assessments/routes.py @@ -1,3 +1,5 @@ +import concurrent.futures +import contextvars import io import time import zipfile @@ -91,6 +93,7 @@ from app.blueprints.services.data_services import get_score_and_justification from app.blueprints.services.data_services import get_sub_criteria from app.blueprints.services.data_services import get_sub_criteria_theme_answers_all +from app.blueprints.services.data_services import get_tag_types from app.blueprints.services.data_services import get_tags_for_fund_round from app.blueprints.services.data_services import match_comment_to_theme from app.blueprints.services.data_services import submit_comment @@ -134,6 +137,21 @@ ) +def run_with_context(func, *args, **kwargs): + with current_app.app_context(): + return func(*args, **kwargs) + + +class ContextThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor): + def __init__(self, *args, **kwargs): + self.context = contextvars.copy_context() + super().__init__(*args, **kwargs, initializer=self._set_child_context) + + def _set_child_context(self): + for var, value in self.context.items(): + var.set(value) + + def _handle_all_uploaded_documents(application_id): flags_list = get_flags(application_id) flag_status = determine_flag_status(flags_list) @@ -237,16 +255,6 @@ def fund_dashboard(fund_short_name: str, round_short_name: str): if has_devolved_authority_validation(fund_id=fund_id): countries = get_countries_from_roles(fund.short_name) - # This call is to get the location data such as country, region and local_authority - # from all the existing applications. - all_applications_metadata = get_application_overviews( - fund_id, round_id, search_params="" - ) - # note, we are not sending search parameters here as we don't want to filter - # the stats at all. see https://dluhcdigital.atlassian.net/browse/FS-3249 - unfiltered_stats = process_assessments_stats(all_applications_metadata) - all_application_locations = LocationData.from_json_blob(all_applications_metadata) - search_params = { **search_params, "countries": ",".join(countries), @@ -255,8 +263,37 @@ def fund_dashboard(fund_short_name: str, round_short_name: str): # matches the query parameters provided in the search and filter form search_params, show_clear_filters = match_search_params(search_params, request.args) - # request all the application overviews based on the search parameters - application_overviews = get_application_overviews(fund_id, round_id, search_params) + with ContextThreadPoolExecutor(max_workers=4) as executor: + # The first call is to get the location data such as country, region and local_authority + # from all the existing applications (i.e withou search parameters as we don't want to filter + # the stats at all). see https://dluhcdigital.atlassian.net/browse/FS-3249 + future_all_applications_metadata = executor.submit( + run_with_context, get_application_overviews, fund_id, round_id, "" + ) + # The second call is with the search parameters + future_application_overviews = executor.submit( + run_with_context, + get_application_overviews, + fund_id, + round_id, + search_params, + ) + future_active_fund_round_tags = executor.submit( + run_with_context, + get_tags_for_fund_round, + fund_id, + round_id, + {"tag_status": "True"}, + ) + future_tag_types = executor.submit(run_with_context, get_tag_types) + + all_applications_metadata = future_all_applications_metadata.result() + application_overviews = future_application_overviews.result() + active_fund_round_tags = future_active_fund_round_tags.result() + tag_types = future_tag_types.result() + + unfiltered_stats = process_assessments_stats(all_applications_metadata) + all_application_locations = LocationData.from_json_blob(all_applications_metadata) teams_flag_stats = get_team_flag_stats(application_overviews) @@ -285,11 +322,8 @@ def fund_dashboard(fund_short_name: str, round_short_name: str): post_processed_overviews ) - active_fund_round_tags = get_tags_for_fund_round( - fund_id, round_id, {"tag_status": "True"} - ) tags_in_application_map, tag_option_groups = get_tag_map_and_tag_options( - active_fund_round_tags, post_processed_overviews + tag_types, active_fund_round_tags, post_processed_overviews ) def get_sorted_application_overviews(application_overviews, column, reverse=False): From beb41f04fa195492be0979afff0c58800be0dcd8 Mon Sep 17 00:00:00 2001 From: Kishan Patel Date: Tue, 11 Jun 2024 13:11:04 +0100 Subject: [PATCH 2/4] Fix tests by updating patch location for get_tag_types --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 524b7059..8b63a1cc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -832,7 +832,7 @@ def mock_get_tag_for_fund_round(mocker): def mock_get_tag_types(mocker): for function_module_path in [ "app.blueprints.tagging.routes.get_tag_types", - "app.blueprints.assessments.helpers.get_tag_types", + "app.blueprints.services.data_services.get_tag_types", ]: mocker.patch( function_module_path, From d85638e41fc853f908c579942f538038dfd4bc62 Mon Sep 17 00:00:00 2001 From: Kishan Patel Date: Fri, 14 Jun 2024 10:58:17 +0100 Subject: [PATCH 3/4] Update fsd-utils and use existing ContextAwareExecutor --- app/blueprints/assessments/routes.py | 61 ++++++++++++++++------------ requirements-dev.txt | 20 +++++---- requirements.in | 2 +- requirements.txt | 8 ++-- 4 files changed, 51 insertions(+), 40 deletions(-) diff --git a/app/blueprints/assessments/routes.py b/app/blueprints/assessments/routes.py index 99930583..515b9a34 100644 --- a/app/blueprints/assessments/routes.py +++ b/app/blueprints/assessments/routes.py @@ -18,6 +18,7 @@ from flask import request from flask import url_for from fsd_utils import extract_questions_and_answers +from fsd_utils.sqs_scheduler.context_aware_executor import ContextAwareExecutor from app.blueprints.assessments.activity_trail import AssociatedTags from app.blueprints.assessments.activity_trail import CheckboxForm @@ -263,34 +264,40 @@ def fund_dashboard(fund_short_name: str, round_short_name: str): # matches the query parameters provided in the search and filter form search_params, show_clear_filters = match_search_params(search_params, request.args) - with ContextThreadPoolExecutor(max_workers=4) as executor: - # The first call is to get the location data such as country, region and local_authority - # from all the existing applications (i.e withou search parameters as we don't want to filter - # the stats at all). see https://dluhcdigital.atlassian.net/browse/FS-3249 - future_all_applications_metadata = executor.submit( - run_with_context, get_application_overviews, fund_id, round_id, "" - ) - # The second call is with the search parameters - future_application_overviews = executor.submit( - run_with_context, - get_application_overviews, - fund_id, - round_id, - search_params, - ) - future_active_fund_round_tags = executor.submit( - run_with_context, - get_tags_for_fund_round, - fund_id, - round_id, - {"tag_status": "True"}, - ) - future_tag_types = executor.submit(run_with_context, get_tag_types) + thread_executor = ContextAwareExecutor( + max_workers=4, + thread_name_prefix="fund-dashboard-request", + flask_app=current_app, + ) + + # The first call is to get the location data such as country, region and local_authority + # from all the existing applications (i.e withou search parameters as we don't want to filter + # the stats at all). see https://dluhcdigital.atlassian.net/browse/FS-3249 + future_all_applications_metadata = thread_executor.submit( + get_application_overviews, fund_id, round_id, "" + ) + + # The second call is with the search parameters + future_application_overviews = thread_executor.submit( + get_application_overviews, + fund_id, + round_id, + search_params, + ) + + future_active_fund_round_tags = thread_executor.submit( + get_tags_for_fund_round, + fund_id, + round_id, + {"tag_status": "True"}, + ) + + future_tag_types = thread_executor.submit(get_tag_types) - all_applications_metadata = future_all_applications_metadata.result() - application_overviews = future_application_overviews.result() - active_fund_round_tags = future_active_fund_round_tags.result() - tag_types = future_tag_types.result() + all_applications_metadata = future_all_applications_metadata.result() + application_overviews = future_application_overviews.result() + active_fund_round_tags = future_active_fund_round_tags.result() + tag_types = future_tag_types.result() unfiltered_stats = process_assessments_stats(all_applications_metadata) all_application_locations = LocationData.from_json_blob(all_applications_metadata) diff --git a/requirements-dev.txt b/requirements-dev.txt index 8cbc5a01..4e704b20 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile requirements-dev.in +# pip-compile --output-file=requirements-dev.txt requirements-dev.in # alembic==1.10.3 # via @@ -124,6 +124,8 @@ docopt==0.6.2 # num2words email-validator==2.1.1 # via -r requirements.txt +exceptiongroup==1.2.1 + # via pytest filelock==3.9.0 # via virtualenv flake8==7.0.0 @@ -173,14 +175,10 @@ flipper-client==1.3.2 # via # -r requirements.txt # funding-service-design-utils -funding-service-design-utils==2.0.32 +funding-service-design-utils==2.0.51 # via -r requirements.txt govuk-frontend-jinja==2.8.0 # via -r requirements.txt -greenlet==3.0.0 - # via - # -r requirements.txt - # sqlalchemy gunicorn==20.1.0 # via # -r requirements.txt @@ -485,10 +483,18 @@ tinycss2==1.2.1 # -r requirements.txt # cssselect2 # svglib +tomli==2.0.1 + # via + # black + # build + # flake8-pyproject + # pytest + # pytest-env typing-extensions==4.5.0 # via # -r requirements.txt # alembic + # black # qrcode # sqlalchemy tzlocal==5.0.1 diff --git a/requirements.in b/requirements.in index 58547a6e..553ff8c9 100644 --- a/requirements.in +++ b/requirements.in @@ -1,7 +1,7 @@ #----------------------------------- # FSD Utils #----------------------------------- -funding-service-design-utils>=2.0.27,<2.1.0 +funding-service-design-utils>=2.0.51,<2.1.0 requests diff --git a/requirements.txt b/requirements.txt index 34dcc4f8..8579bd88 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile requirements.in +# pip-compile --output-file=requirements.txt requirements.in # alembic==1.10.3 # via flask-migrate @@ -111,12 +111,10 @@ flask-wtf==1.1.1 # via -r requirements.in flipper-client==1.3.2 # via funding-service-design-utils -funding-service-design-utils==2.0.32 +funding-service-design-utils==2.0.51 # via -r requirements.in govuk-frontend-jinja==2.8.0 # via -r requirements.in -greenlet==3.0.0 - # via sqlalchemy gunicorn==20.1.0 # via funding-service-design-utils html5lib==1.1 From d3c675998205083468bb68903b36e0921d8d75c9 Mon Sep 17 00:00:00 2001 From: Kishan Patel Date: Fri, 14 Jun 2024 11:38:56 +0100 Subject: [PATCH 4/4] Specify (default) language for format_answer --- app/blueprints/assessments/helpers.py | 4 +++- app/blueprints/assessments/models/full_application.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/blueprints/assessments/helpers.py b/app/blueprints/assessments/helpers.py index 396d7941..e9ef4ee1 100644 --- a/app/blueprints/assessments/helpers.py +++ b/app/blueprints/assessments/helpers.py @@ -164,7 +164,9 @@ def generate_csv_of_application( if not answers: answers = "Not provided" - writer.writerow([section_title, questions, format_answer(answers)]) + writer.writerow( + [section_title, questions, format_answer(answer=answers, language="en")] + ) return output.getvalue() diff --git a/app/blueprints/assessments/models/full_application.py b/app/blueprints/assessments/models/full_application.py index 8c235178..7c126c3e 100644 --- a/app/blueprints/assessments/models/full_application.py +++ b/app/blueprints/assessments/models/full_application.py @@ -55,7 +55,7 @@ def from_data(cls, args): questions_and_answers=[ QuestionAndAnswer( question=i["question"], - answer=format_answer(i.get("answer")), + answer=format_answer(i.get("answer"), language="en"), number="", ) for i in args.all_uploaded_documents