From 7b3f043ead9bb9ef926502d84763c56e1bb96d32 Mon Sep 17 00:00:00 2001 From: granitosaurus Date: Sat, 26 Apr 2025 14:40:59 +0700 Subject: [PATCH] add rendering_stage parameter to ScrapeConfig --- scrapfly/__init__.py | 2 +- scrapfly/scrape_config.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapfly/__init__.py b/scrapfly/__init__.py index dfeff07..3761519 100644 --- a/scrapfly/__init__.py +++ b/scrapfly/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.8.22' +__version__ = '0.8.23' from typing import Tuple from .errors import ScrapflyError diff --git a/scrapfly/scrape_config.py b/scrapfly/scrape_config.py index 04e01c1..7154033 100644 --- a/scrapfly/scrape_config.py +++ b/scrapfly/scrape_config.py @@ -4,7 +4,7 @@ from enum import Enum from urllib.parse import urlencode, quote_plus from base64 import urlsafe_b64encode -from typing import Optional, List, Dict, Iterable, Union, Set +from typing import Literal, Optional, List, Dict, Iterable, Union, Set from requests.structures import CaseInsensitiveDict from .api_config import BaseApiConfig @@ -92,6 +92,7 @@ class ScrapeConfig(BaseApiConfig): headers: Optional[CaseInsensitiveDict] = None js: str = None rendering_wait: int = None + rendering_stage: Literal["complete", "domcontentloaded"] = "complete" wait_for_selector: Optional[str] = None session_sticky_proxy:bool = True screenshots:Optional[Dict]=None @@ -136,6 +137,7 @@ def __init__( headers: Optional[Union[CaseInsensitiveDict, Dict[str, str]]] = None, js: str = None, rendering_wait: int = None, + rendering_stage: Literal["complete", "domcontentloaded"] = "complete", wait_for_selector: Optional[str] = None, screenshots:Optional[Dict]=None, screenshot_flags: Optional[List[ScreenshotFlag]] = None, @@ -186,6 +188,7 @@ def __init__( self.data = data self.js = js self.rendering_wait = rendering_wait + self.rendering_stage = rendering_stage self.raise_on_upstream_error = raise_on_upstream_error self.screenshots = screenshots self.screenshot_flags = screenshot_flags @@ -270,6 +273,9 @@ def to_api_params(self, key:str) -> Dict: if self.rendering_wait: params['rendering_wait'] = self.rendering_wait + + if self.rendering_stage: + params['rendering_stage'] = self.rendering_stage if self.screenshots is not None: for name, element in self.screenshots.items():