From ec1d5ca3863726a12e789bd74979d6f3db5710fa Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Mon, 6 Nov 2023 10:44:54 -0800 Subject: [PATCH] Feature/user-cache-dir: Add User Preference for the Cache Directory (#5621) * add user preference for storing cached data * ruff * fix tests? * fix test key error..? * etf integration test fix * add cache dir to SEC * black --- .../core/openbb_core/app/model/preferences.py | 1 + .../platform/core/openbb_core/app/utils.py | 19 +++++++++++++++ .../biztoc/openbb_biztoc/utils/helpers.py | 23 +++++++++++-------- .../cboe/openbb_cboe/utils/helpers.py | 16 ++++--------- .../providers/sec/openbb_sec/utils/helpers.py | 9 +++++--- 5 files changed, 44 insertions(+), 24 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/preferences.py b/openbb_platform/platform/core/openbb_core/app/model/preferences.py index a323ae10bdc8..afb0963fe2a3 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/preferences.py +++ b/openbb_platform/platform/core/openbb_core/app/model/preferences.py @@ -8,6 +8,7 @@ class Preferences(BaseModel): data_directory: str = str(Path.home() / "OpenBBUserData") export_directory: str = str(Path.home() / "OpenBBUserData" / "exports") user_styles_directory: str = str(Path.home() / "OpenBBUserData" / "styles" / "user") + cache_directory: str = str(Path.home() / "OpenBBUserData" / "cache") charting_extension: Literal["openbb_charting"] = "openbb_charting" chart_style: Literal["dark", "light"] = "dark" plot_enable_pywry: bool = True diff --git a/openbb_platform/platform/core/openbb_core/app/utils.py b/openbb_platform/platform/core/openbb_core/app/utils.py index eaafa1bc3402..c7e8dc9edc14 100644 --- a/openbb_platform/platform/core/openbb_core/app/utils.py +++ b/openbb_platform/platform/core/openbb_core/app/utils.py @@ -5,6 +5,9 @@ import pandas as pd from openbb_provider.abstract.data import Data +from openbb_core.app.model.preferences import Preferences +from openbb_core.app.model.system_settings import SystemSettings + def basemodel_to_df( data: Union[List[Data], Data], @@ -69,3 +72,19 @@ def get_target_columns(df: pd.DataFrame, target_columns: List[str]) -> pd.DataFr for target in target_columns: df_result[target] = get_target_column(df, target).to_frame() return df_result + + +def get_user_cache_directory() -> str: + file = SystemSettings().model_dump()["user_settings_path"] + with open(file) as settings_file: + contents = settings_file.read() + try: + settings = json.loads(contents)["preferences"] + except KeyError: + settings = None + cache_dir = ( + settings["cache_directory"] + if settings and "cache_directory" in settings + else Preferences().cache_directory + ) + return cache_dir diff --git a/openbb_platform/providers/biztoc/openbb_biztoc/utils/helpers.py b/openbb_platform/providers/biztoc/openbb_biztoc/utils/helpers.py index c618915eec14..63b64c739dc4 100644 --- a/openbb_platform/providers/biztoc/openbb_biztoc/utils/helpers.py +++ b/openbb_platform/providers/biztoc/openbb_biztoc/utils/helpers.py @@ -1,16 +1,25 @@ +"""Biztoc Helpers""" + from datetime import timedelta from typing import Dict, List, Literal import requests import requests_cache +from openbb_core.app.utils import get_user_cache_directory + +cache_dir = get_user_cache_directory() + +biztoc_session_tags = requests_cache.CachedSession( + f"{cache_dir}/http/biztoc_tags", expire_after=timedelta(days=1) +) +biztoc_session_sources = requests_cache.CachedSession( + f"{cache_dir}/http/biztoc_sources", expire_after=timedelta(days=3) +) def get_sources(api_key: str) -> List[Dict]: """Valid sources for Biztoc queries.""" - biztoc_session_sources = requests_cache.CachedSession( - "OpenBB_Biztoc_Pages", expire_after=timedelta(days=3), use_cache_dir=True - ) headers = { "X-RapidAPI-Key": f"{api_key}", "X-RapidAPI-Host": "biztoc.p.rapidapi.com", @@ -27,16 +36,13 @@ def get_sources(api_key: str) -> List[Dict]: def get_pages(api_key: str) -> List[str]: """Valid pages for Biztoc queries.""" - biztoc_session_pages = requests_cache.CachedSession( - "OpenBB_Biztoc_Pages", expire_after=timedelta(days=3), use_cache_dir=True - ) headers = { "X-RapidAPI-Key": f"{api_key}", "X-RapidAPI-Host": "biztoc.p.rapidapi.com", "Accept": "application/json", "Accept-Encoding": "gzip", } - pages = biztoc_session_pages.get( + pages = biztoc_session_sources.get( "https://biztoc.p.rapidapi.com/pages", headers=headers, timeout=10 ) @@ -46,9 +52,6 @@ def get_pages(api_key: str) -> List[str]: def get_tags_by_page(page_id: str, api_key: str) -> List[str]: """Valid tags required for Biztoc queries.""" - biztoc_session_tags = requests_cache.CachedSession( - "OpenBB_Biztoc_Tags", expire_after=timedelta(days=1), use_cache_dir=True - ) headers = { "X-RapidAPI-Key": f"{api_key}", "X-RapidAPI-Host": "biztoc.p.rapidapi.com", diff --git a/openbb_platform/providers/cboe/openbb_cboe/utils/helpers.py b/openbb_platform/providers/cboe/openbb_cboe/utils/helpers.py index 861eaf51f18e..48d271358fb5 100644 --- a/openbb_platform/providers/cboe/openbb_cboe/utils/helpers.py +++ b/openbb_platform/providers/cboe/openbb_cboe/utils/helpers.py @@ -7,15 +7,16 @@ import pandas as pd import requests import requests_cache +from openbb_core.app.utils import get_user_cache_directory from openbb_provider.utils.helpers import to_snake_case -TICKER_EXCEPTIONS = ["NDX", "RUT"] -# This will cache the import requests for 7 days. Ideally to speed up subsequent imports. -# Only used on functions run on import +cache_dir = get_user_cache_directory() cboe_session = requests_cache.CachedSession( - "OpenBB_CBOE", expire_after=timedelta(days=7), use_cache_dir=True + f"{cache_dir}/http/cboe_directories", expire_after=timedelta(days=7) ) +TICKER_EXCEPTIONS = ["NDX", "RUT"] + US_INDEX_COLUMNS = { "symbol": "symbol", "name": "name", @@ -64,13 +65,6 @@ } -# This will cache certain requests for 7 days. Ideally to speed up subsequent queries. -# Only used on functions with static values like symbol directories. -cboe_session = requests_cache.CachedSession( - "OpenBB_CBOE", expire_after=timedelta(days=7), use_cache_dir=True -) - - def get_cboe_directory(**kwargs) -> pd.DataFrame: """Get the US Listings Directory for the CBOE. diff --git a/openbb_platform/providers/sec/openbb_sec/utils/helpers.py b/openbb_platform/providers/sec/openbb_sec/utils/helpers.py index 39bc07626173..1c26e7e1bdc9 100644 --- a/openbb_platform/providers/sec/openbb_sec/utils/helpers.py +++ b/openbb_platform/providers/sec/openbb_sec/utils/helpers.py @@ -8,15 +8,18 @@ import pandas as pd import requests import requests_cache +from openbb_core.app.utils import get_user_cache_directory from openbb_sec.utils.definitions import HEADERS, QUARTERS, SEC_HEADERS, TAXONOMIES +cache_dir = get_user_cache_directory() + sec_session_companies = requests_cache.CachedSession( - "OpenBB_SEC_Companies", expire_after=timedelta(days=7), use_cache_dir=True + f"{cache_dir}/http/sec_companies", expire_after=timedelta(days=7) ) sec_session_frames = requests_cache.CachedSession( - "OpenBB_SEC_Frames", expire_after=timedelta(days=30), use_cache_dir=True + f"{cache_dir}/http/sec_frames", expire_after=timedelta(days=5) ) -sec_session_ftd = requests_cache.CachedSession("OpenBB_SEC_FTD", use_cache_dir=True) +sec_session_ftd = requests_cache.CachedSession(f"{cache_dir}/http/sec_ftd") def get_all_companies(use_cache: bool = True) -> pd.DataFrame: