Feature/user-cache-dir: Add User Preference for the Cache Directory (#…

…5621)

* add user preference for storing cached data

* ruff

* fix tests?

* fix test key error..?

* etf integration test fix

* add cache dir to SEC

* black

openbb_platform/platform/core/openbb_core/app/model/preferences.py

@@ -8,6 +8,7 @@ class Preferences(BaseModel):
     data_directory: str = str(Path.home() / "OpenBBUserData")
     export_directory: str = str(Path.home() / "OpenBBUserData" / "exports")
     user_styles_directory: str = str(Path.home() / "OpenBBUserData" / "styles" / "user")
+    cache_directory: str = str(Path.home() / "OpenBBUserData" / "cache")
     charting_extension: Literal["openbb_charting"] = "openbb_charting"
     chart_style: Literal["dark", "light"] = "dark"
     plot_enable_pywry: bool = True

openbb_platform/platform/core/openbb_core/app/utils.py

@@ -5,6 +5,9 @@
 import pandas as pd
 from openbb_provider.abstract.data import Data
 
+from openbb_core.app.model.preferences import Preferences
+from openbb_core.app.model.system_settings import SystemSettings
+
 
 def basemodel_to_df(
     data: Union[List[Data], Data],
@@ -69,3 +72,19 @@ def get_target_columns(df: pd.DataFrame, target_columns: List[str]) -> pd.DataFr
     for target in target_columns:
         df_result[target] = get_target_column(df, target).to_frame()
     return df_result
+
+
+def get_user_cache_directory() -> str:
+    file = SystemSettings().model_dump()["user_settings_path"]
+    with open(file) as settings_file:
+        contents = settings_file.read()
+    try:
+        settings = json.loads(contents)["preferences"]
+    except KeyError:
+        settings = None
+    cache_dir = (
+        settings["cache_directory"]
+        if settings and "cache_directory" in settings
+        else Preferences().cache_directory
+    )
+    return cache_dir

openbb_platform/providers/biztoc/openbb_biztoc/utils/helpers.py

@@ -1,16 +1,25 @@
+"""Biztoc Helpers"""
+
 from datetime import timedelta
 from typing import Dict, List, Literal
 
 import requests
 import requests_cache
+from openbb_core.app.utils import get_user_cache_directory
+
+cache_dir = get_user_cache_directory()
+
+biztoc_session_tags = requests_cache.CachedSession(
+    f"{cache_dir}/http/biztoc_tags", expire_after=timedelta(days=1)
+)
+biztoc_session_sources = requests_cache.CachedSession(
+    f"{cache_dir}/http/biztoc_sources", expire_after=timedelta(days=3)
+)
 
 
 def get_sources(api_key: str) -> List[Dict]:
     """Valid sources for Biztoc queries."""
 
-    biztoc_session_sources = requests_cache.CachedSession(
-        "OpenBB_Biztoc_Pages", expire_after=timedelta(days=3), use_cache_dir=True
-    )
     headers = {
         "X-RapidAPI-Key": f"{api_key}",
         "X-RapidAPI-Host": "biztoc.p.rapidapi.com",
@@ -27,16 +36,13 @@ def get_sources(api_key: str) -> List[Dict]:
 def get_pages(api_key: str) -> List[str]:
     """Valid pages for Biztoc queries."""
 
-    biztoc_session_pages = requests_cache.CachedSession(
-        "OpenBB_Biztoc_Pages", expire_after=timedelta(days=3), use_cache_dir=True
-    )
     headers = {
         "X-RapidAPI-Key": f"{api_key}",
         "X-RapidAPI-Host": "biztoc.p.rapidapi.com",
         "Accept": "application/json",
         "Accept-Encoding": "gzip",
     }
-    pages = biztoc_session_pages.get(
+    pages = biztoc_session_sources.get(
         "https://biztoc.p.rapidapi.com/pages", headers=headers, timeout=10
     )
 
@@ -46,9 +52,6 @@ def get_pages(api_key: str) -> List[str]:
 def get_tags_by_page(page_id: str, api_key: str) -> List[str]:
     """Valid tags required for Biztoc queries."""
 
-    biztoc_session_tags = requests_cache.CachedSession(
-        "OpenBB_Biztoc_Tags", expire_after=timedelta(days=1), use_cache_dir=True
-    )
     headers = {
         "X-RapidAPI-Key": f"{api_key}",
         "X-RapidAPI-Host": "biztoc.p.rapidapi.com",

openbb_platform/providers/cboe/openbb_cboe/utils/helpers.py

@@ -7,15 +7,16 @@
 import pandas as pd
 import requests
 import requests_cache
+from openbb_core.app.utils import get_user_cache_directory
 from openbb_provider.utils.helpers import to_snake_case
 
-TICKER_EXCEPTIONS = ["NDX", "RUT"]
-# This will cache the import requests for 7 days.  Ideally to speed up subsequent imports.
-# Only used on functions run on import
+cache_dir = get_user_cache_directory()
 cboe_session = requests_cache.CachedSession(
-    "OpenBB_CBOE", expire_after=timedelta(days=7), use_cache_dir=True
+    f"{cache_dir}/http/cboe_directories", expire_after=timedelta(days=7)
 )
 
+TICKER_EXCEPTIONS = ["NDX", "RUT"]
+
 US_INDEX_COLUMNS = {
     "symbol": "symbol",
     "name": "name",
@@ -64,13 +65,6 @@
 }
 
 
-# This will cache certain requests for 7 days.  Ideally to speed up subsequent queries.
-# Only used on functions with static values like symbol directories.
-cboe_session = requests_cache.CachedSession(
-    "OpenBB_CBOE", expire_after=timedelta(days=7), use_cache_dir=True
-)
-
-
 def get_cboe_directory(**kwargs) -> pd.DataFrame:
     """Get the US Listings Directory for the CBOE.
 

openbb_platform/providers/sec/openbb_sec/utils/helpers.py

@@ -8,15 +8,18 @@
 import pandas as pd
 import requests
 import requests_cache
+from openbb_core.app.utils import get_user_cache_directory
 from openbb_sec.utils.definitions import HEADERS, QUARTERS, SEC_HEADERS, TAXONOMIES
 
+cache_dir = get_user_cache_directory()
+
 sec_session_companies = requests_cache.CachedSession(
-    "OpenBB_SEC_Companies", expire_after=timedelta(days=7), use_cache_dir=True
+    f"{cache_dir}/http/sec_companies", expire_after=timedelta(days=7)
 )
 sec_session_frames = requests_cache.CachedSession(
-    "OpenBB_SEC_Frames", expire_after=timedelta(days=30), use_cache_dir=True
+    f"{cache_dir}/http/sec_frames", expire_after=timedelta(days=5)
 )
-sec_session_ftd = requests_cache.CachedSession("OpenBB_SEC_FTD", use_cache_dir=True)
+sec_session_ftd = requests_cache.CachedSession(f"{cache_dir}/http/sec_ftd")
 
 
 def get_all_companies(use_cache: bool = True) -> pd.DataFrame: