Merge branch 'feat/scrapers-config' into v4.4

mov_cli/cli/__main__.py

@@ -84,7 +84,7 @@ def mov_cli(
 
         http_client = HTTPClient(config)
 
-        selected_scraper = select_scraper(plugins, config.fzf_enabled, config.default_scraper)
+        selected_scraper = select_scraper(plugins, config.scrapers, config.fzf_enabled, config.default_scraper)
 
         if selected_scraper is None:
             mov_cli_logger.error(
@@ -93,7 +93,9 @@ def mov_cli(
             )
             return False
 
-        chosen_scraper = use_scraper(selected_scraper, config, http_client, scrape_options)
+        selected_scraper[2].update(scrape_options)
+
+        chosen_scraper = use_scraper(selected_scraper, config, http_client)
 
         choice = search(query, auto_select, chosen_scraper, config.fzf_enabled)
 

mov_cli/cli/scraper.py

@@ -6,9 +6,9 @@
 
     from .plugins import PluginsDataT
 
-    from ..config import Config
     from ..media import Metadata, Media
     from ..http_client import HTTPClient
+    from ..config import Config, ScrapersConfigT
     from ..utils.episode_selector import EpisodeSelector
     from ..scraper import Scraper, ScraperOptionsT
 
@@ -32,12 +32,11 @@ def scrape(choice: Metadata, episode: EpisodeSelector, scraper: Scraper) -> Medi
     return media
 
 def use_scraper(
-    selected_scraper: Tuple[str, Type[Scraper]], 
+    selected_scraper: Tuple[str, Type[Scraper], ScraperOptionsT], 
     config: Config, 
-    http_client: HTTPClient,
-    scraper_options: ScraperOptionsT
+    http_client: HTTPClient
 ) -> Scraper:
-    scraper_name, scraper_class = selected_scraper
+    scraper_name, scraper_class, scraper_options = selected_scraper
 
     mov_cli_logger.info(f"Using '{Colours.BLUE.apply(scraper_name)}' scraper...")
 
@@ -48,11 +47,11 @@ def use_scraper(
 
     return chosen_scraper
 
-def select_scraper(plugins: Dict[str, str], fzf_enabled: bool, default_scraper: Optional[str] = None) -> Optional[Tuple[str, Type[Scraper]]]:
+def select_scraper(plugins: Dict[str, str], scrapers: ScrapersConfigT, fzf_enabled: bool, default_scraper: Optional[str] = None) -> Optional[Tuple[str, Type[Scraper], ScraperOptionsT]]:
     plugins_data = get_plugins_data(plugins)
 
     if default_scraper is not None:
-        scraper_name, scraper_or_available_scrapers = get_scraper(default_scraper, plugins_data)
+        scraper_name, scraper_or_available_scrapers, scraper_options = get_scraper(default_scraper, plugins_data, scrapers)
 
         if scraper_name is None:
             mov_cli_logger.error(
@@ -72,7 +71,7 @@ def select_scraper(plugins: Dict[str, str], fzf_enabled: bool, default_scraper:
 
             return None
 
-        return scraper_name, scraper_or_available_scrapers
+        return scraper_name, scraper_or_available_scrapers, scraper_options
 
     chosen_plugin = prompt(
         "Select a plugin", 
@@ -96,7 +95,7 @@ def select_scraper(plugins: Dict[str, str], fzf_enabled: bool, default_scraper:
 
         scraper_name, scraper = chosen_scraper
 
-        return f"{plugin_namespace}.{scraper_name}".lower(), scraper
+        return f"{plugin_namespace}.{scraper_name}".lower(), scraper, {}
 
     return None
 
@@ -135,26 +134,35 @@ def steal_scraper_args(query: List[str]) -> ScraperOptionsT:
 
     return dict(scraper_options_args)
 
-def get_scraper(scraper_id: str, plugins_data: PluginsDataT) -> Tuple[str, Type[Scraper] | Tuple[None, List[str]]]:
+def get_scraper(scraper_id: str, plugins_data: PluginsDataT, user_defined_scrapers: ScrapersConfigT) -> Tuple[str, Type[Scraper] | Tuple[None, List[str]], ScraperOptionsT]:
+    scraper_options = {}
     available_scrapers = []
 
+    # scraper namespace override.
+    for scraper_namespace, scraper_data in user_defined_scrapers.items():
+
+        if scraper_id.lower() == scraper_namespace.lower():
+            mov_cli_logger.debug(f"Using the scraper overridden namespace '{scraper_namespace}'...")
+            scraper_id = scraper_data["namespace"]
+            scraper_options = scraper_data["options"]
+
     platform = what_platform().upper()
 
     for plugin_namespace, _, plugin in plugins_data:
-        scrapers = plugin.hook_data["scrapers"]
+        plugin_scrapers = plugin.hook_data["scrapers"]
 
-        if scraper_id.lower() == plugin_namespace.lower() and f"{platform}.DEFAULT" in scrapers:
-            return f"{plugin_namespace}.{platform}.DEFAULT", scrapers[f"{platform}.DEFAULT"]
+        if scraper_id.lower() == plugin_namespace.lower() and f"{platform}.DEFAULT" in plugin_scrapers:
+            return f"{plugin_namespace}.{platform}.DEFAULT", plugin_scrapers[f"{platform}.DEFAULT"], scraper_options
 
-        elif scraper_id.lower() == plugin_namespace.lower() and "DEFAULT" in scrapers:
-            return f"{plugin_namespace}.DEFAULT", scrapers["DEFAULT"]
+        elif scraper_id.lower() == plugin_namespace.lower() and "DEFAULT" in plugin_scrapers:
+            return f"{plugin_namespace}.DEFAULT", plugin_scrapers["DEFAULT"], scraper_options
 
-        for scraper_name, scraper in scrapers.items():
+        for scraper_name, scraper in plugin_scrapers.items():
             id = f"{plugin_namespace}.{scraper_name}".lower()
 
             available_scrapers.append(id)
 
             if scraper_id.lower() == id:
-                return id, scraper
+                return id, scraper, scraper_options
 
-    return None, available_scrapers
+    return None, available_scrapers, scraper_options

mov_cli/config.py

@@ -4,10 +4,16 @@
 
 if TYPE_CHECKING:
     from .players import Player
-    from typing import Dict, Union, Literal, Any, Optional
+    from typing import Dict, Literal, Any, Optional
 
-    JSON_VALUES = Union[str, bool, int, dict]
-    SUPPORTED_PARSERS = Literal["lxml", "html.parser"]
+    SupportedParsersT = Literal["lxml", "html.parser"]
+
+    @final
+    class ScraperData(TypedDict):
+        namespace: str
+        options: Dict[str, str | bool]
+
+    ScrapersConfigT = Dict[Literal["default"], str] | Dict[str, ScraperData]
 
 import os
 import toml
@@ -20,7 +26,7 @@
 from .logger import mov_cli_logger
 from .utils import get_appdata_directory
 
-__all__ = ("Config", )
+__all__ = ("Config",)
 
 @final
 class ConfigUIData(TypedDict):
@@ -34,20 +40,17 @@ class ConfigHTTPData(TypedDict):
 class ConfigDownloadsData(TypedDict):
     save_path: str
 
-@final
-class ScrapersData(TypedDict):
-    default: str
-
 @final
 class ConfigData(TypedDict):
     version: int
     debug: bool
     player: str
-    parser: SUPPORTED_PARSERS
+    editor: str
+    parser: SupportedParsersT
     ui: ConfigUIData
     http: ConfigHTTPData
     downloads: ConfigDownloadsData
-    scrapers: ScrapersData
+    scrapers: ScrapersConfigT | Dict[str, str]
     plugins: Dict[str, str]
     resolution: int
 
@@ -109,10 +112,33 @@ def player(self) -> Player:
     def plugins(self) -> Dict[str, str]:
         return self.data.get("plugins", {"test": "mov-cli-test"})
 
+    @property
+    def scrapers(self) -> ScrapersConfigT:
+        scrapers = self.data.get("scrapers", {})
+
+        consistent_scrapers: Dict[str, ScraperData] = {}
+
+        for scraper, plugin_namespace_or_dict in scrapers.items():
+
+            if scraper == "default":
+                consistent_scrapers["default"] = plugin_namespace_or_dict
+
+            elif isinstance(plugin_namespace_or_dict, str):
+                consistent_scrapers[scraper] = {"namespace": plugin_namespace_or_dict, "options": {}}
+
+            else:
+                dict = plugin_namespace_or_dict
+                consistent_scrapers[scraper] = {
+                    "namespace": dict["namespace"], 
+                    "options": dict["options"]
+                }
+
+        return consistent_scrapers
+
     @property
     def editor(self) -> Optional[str]:
         """Returns the editor that should be opened while editing."""
-        return self.data.get("editor")
+        return self.data.get("editor", None)
 
     @property
     def skip_update_checker(self) -> bool:
@@ -129,7 +155,7 @@ def fzf_enabled(self) -> bool:
         return self.data.get("ui", {}).get("fzf", True if shutil.which("fzf") is not None else False)
 
     @property
-    def parser(self) -> SUPPORTED_PARSERS | Any:
+    def parser(self) -> SupportedParsersT | Any:
         """Returns the parser type configured by the user else it just returns the default."""
         default_parser = "lxml" if find_spec("lxml") else "html.parser"
         return self.data.get("parser", default_parser)

mov_cli/config.template.toml

@@ -12,8 +12,9 @@ skip_update_checker = false
 [mov-cli.plugins] # E.g: namespace = "package-name"
 test = "mov-cli-test"
 
-# [mov-cli.scrapers]
+[mov-cli.scrapers]
 # default = "films"
+test = "test.DEFAULT"
 
 # [mov-cli.http] # Don't mess with it if you don't know what you are doing!
 # headers = { User-Agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/117.0" }